From 40a273b99aa82a71a6fee4f57ff7843bef19221f Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 7 Oct 2026 11:21:14 +0530 Subject: [PATCH 01/43] chore: add /dev_docs to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2df82949..7ab00aa2 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,7 @@ rendered/ # reaches a `git add -A`. secrets.json *-secrets.json - +/dev_docs # Python bytecode __pycache__/ *.pyc From aaca20446c06267c08c3ab30ca135b24a99ed905 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 12:46:46 +0530 Subject: [PATCH 02/43] feat(quick-start): add unified network layer mode alongside the multi-adapter stack Run the consumer, network and provider tiers as modules of ONE adapter (network-layer-adapter) with in-process (inproc://) tier hops, next to the existing three-adapter stack, which is unchanged. - docker-compose.unified-network-layer.yml: full stack with the three adapters replaced by network-layer-adapter (port 9210 on 127.0.0.1) - docker-compose.unified-network-layer-http.yml: optional overlay that sends tier hops over http://localhost instead (any image, HTTP=1) - config/adapters/unified-network-layer/: overrides.yaml (module names, paths, routing file, extended schema, appName, port) plus routing-inproc.yaml (default) and routing-http.yaml - bin/setup.py: builds unified-network-layer.yaml from the three tier templates plus overrides.yaml, so tier changes reach both modes; each module keeps its own tier identity; PyYAML imported lazily - bin/stack.sh, Makefile: make up-/down-unified-network-layer [HTTP=1], failing early when the inproc image is missing - bin/unified-network-layer-up.sh: build the inproc adapter image, start the stack and smoke-test publish, discover and select - .env.example: UNIFIED_NETWORK_LAYER_ADAPTER_PORT and _IMAGE - README: "Unified network layer" section - .gitignore: rendered config, local demo/bench scripts, Apple-silicon override, .bench/ results, local Postman exports --- .gitignore | 6 + quick-start/.env.example | 8 + quick-start/.gitignore | 11 + quick-start/Makefile | 4 +- quick-start/README.md | 46 + quick-start/bin/setup.py | 191 +++- quick-start/bin/stack.sh | 50 + quick-start/bin/unified-network-layer-up.sh | 88 ++ .../unified-network-layer/overrides.yaml | 46 + .../unified-network-layer/routing-http.yaml | 36 + .../unified-network-layer/routing-inproc.yaml | 39 + ...ker-compose.unified-network-layer-http.yml | 11 + .../docker-compose.unified-network-layer.yml | 928 ++++++++++++++++++ 13 files changed, 1410 insertions(+), 54 deletions(-) create mode 100755 quick-start/bin/unified-network-layer-up.sh create mode 100644 quick-start/config/adapters/unified-network-layer/overrides.yaml create mode 100644 quick-start/config/adapters/unified-network-layer/routing-http.yaml create mode 100644 quick-start/config/adapters/unified-network-layer/routing-inproc.yaml create mode 100644 quick-start/docker-compose.unified-network-layer-http.yml create mode 100644 quick-start/docker-compose.unified-network-layer.yml diff --git a/.gitignore b/.gitignore index 7ab00aa2..ceaee1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,12 @@ rendered/ # reaches a `git add -A`. secrets.json *-secrets.json + +# Generated local Postman exports for the unified network layer; keep them out of source +# control even though they are tracked alongside the repo's API collection files. +api-collection/OpenAgriNet.unified-network-layer.collection.json +api-collection/OpenAgriNet.unified-network-layer.environment.json + /dev_docs # Python bytecode __pycache__/ diff --git a/quick-start/.env.example b/quick-start/.env.example index 3b6d9a10..f1beeafd 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,6 +39,14 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 +# Unified network layer (make up-unified-network-layer): ONE adapter for the +# consumer, network and provider tiers. Host port, 127.0.0.1 only. +UNIFIED_NETWORK_LAYER_ADAPTER_PORT=9210 +# Image with in-process routing (inproc://), built by bin/unified-network-layer-up.sh +# from network-adapter feat/55-unified-adapter-inproc-routing. Needed for the +# default inproc:// hops; with HTTP=1 (hops over http://localhost) any image +# works, and empty falls back to ADAPTER_IMAGE. +UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=network-adapter:inproc # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 diff --git a/quick-start/.gitignore b/quick-start/.gitignore index ac5fe84d..70bdeef9 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -16,6 +16,14 @@ config/adapters/experience.yaml config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml +config/adapters/unified-network-layer/unified-network-layer.yaml +bin/demo-unified-network-layer.sh +bin/bench-unified-network-layer.sh +poc-flow.py +bin/poc +# Apple-silicon-only override that forces amd64 emulation for images with no +# arm64 build. A local workaround, not part of the stack. +docker-compose.arm64.yml # A local override for the discovery service, if you make one. config/discovery/instance.yaml @@ -33,3 +41,6 @@ config/discovery/instance.yaml # certificates in the npm-data and npm-letsencrypt volumes, so there is # nothing here to ignore -- and nothing here to review either. Back those # volumes up; they are the only copy. + +# bin/bench-unified-network-layer.sh scratch: generated bodies and results +.bench/ \ No newline at end of file diff --git a/quick-start/Makefile b/quick-start/Makefile index 1aae6475..739a95e2 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core down destroy setup reverse-proxy observability pull restart restart-edge ps logs +.PHONY: help up up-core up-unified-network-layer down-unified-network-layer down destroy setup reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -25,6 +25,8 @@ help: up: ; @$(STACK) up up-core: ; @$(STACK) up-core down: ; @$(STACK) down +up-unified-network-layer: ; @HTTP=$(HTTP) $(STACK) up-unified-network-layer +down-unified-network-layer: ; @$(STACK) down-unified-network-layer destroy: ; @$(STACK) destroy # --------------------------------------------- the optional tiers, on their own diff --git a/quick-start/README.md b/quick-start/README.md index 1165f014..aedced2a 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -189,6 +189,52 @@ collection. → Appendix N. --- +## Unified network layer (optional) + +The same network can run two ways. Both use the same registry, keys and data: + +| | Multi-adapter (default) | Unified network layer | +|---|---|---| +| What runs | `consumer-adapter`, `network-adapter`, `provider-adapter` | one `network-layer-adapter` (the unified single adapter) | +| Compose file | `docker-compose.yml` | `docker-compose.unified-network-layer.yml` | +| Start / stop | `make up` / `make down` | `make up-unified-network-layer` / `make down-unified-network-layer` | +| Tier-to-tier hops | HTTP between containers | `inproc://`: a function call inside the process (default). `HTTP=1` sends them over `http://localhost` inside the container instead | +| Port | 9202 / 9201 / 9200 | `127.0.0.1:${UNIFIED_NETWORK_LAYER_ADAPTER_PORT}` (9210) | + +Use one or the other, not both: they share the project name and data volumes. + +**Image.** In-process hops need an adapter image built with in-process routing +(`network-adapter` branch `feat/55-unified-adapter-inproc-routing`). Set it in +`.env` as `UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE` (default `network-adapter:inproc`), +or let `bin/unified-network-layer-up.sh` build it and bring everything up. + +**Config.** There is no hand-written config for this mode. `bin/setup.py` builds +`config/adapters/unified-network-layer/unified-network-layer.yaml` from the three +tier templates plus `unified-network-layer/overrides.yaml`. That file holds only +the module names and paths, the one routing file, extended schema, appName and +port. A change to a tier template reaches both modes on the next `setup.py`. +Each module keeps its own tier identity. + +| Module | Path | From | +|---|---|---| +| `consumer` | `/consumer/` | `consumer.yaml.tmpl` | +| `network` | `/network/` | `network.yaml.tmpl` | +| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | +| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | + +Routing: `unified-network-layer/routing-inproc.yaml` (default). Discover and +publish go to discovery. Select, init, confirm and status go to +`inproc://provider`. Provider publish goes to `inproc://network/publish`. +`make up-unified-network-layer HTTP=1` mounts `routing-http.yaml` instead, with +the same targets on `http://localhost:9200/...`. It works with any adapter image. + +Tools: +- `bin/unified-network-layer-up.sh [--no-build]`: build the image, start, smoke test. +- `bin/demo-unified-network-layer.sh`: step-by-step live demo. +- `bin/bench-unified-network-layer.sh`: multi vs unified network layer, parallel curl, JSON results under `.bench/`. +- `bin/poc-flow.py --mode multi|unified-network-layer`: quick publish/discover/select check. +- Postman: `api-collection/OpenAgriNet.unified-network-layer.*.json`. + # Part 2 — Run it on a VM Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 541ef06e..18554818 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -556,8 +556,8 @@ def key_osids(identities): ROLES = ("consumer", "network", "provider") -def render(identities): - print("configs:") +def _shared_substitutions(): + """Placeholders that are the same in every adapter config.""" mausamgram_binding = (f"{env('MAUSAMGRAM_PARTICIPANT_ID')}" f"|{env('MAUSAMGRAM_CAPABILITY')}") agmarknet_binding = (f"{env('AGMARKNET_PARTICIPANT_ID')}" @@ -565,59 +565,144 @@ def render(identities): vistaar_binding = (f"{env('VISTAAR_PARTICIPANT_ID')}" f"|{env('VISTAAR_CAPABILITY')}") pocra_binding = f"{env('POCRA_PARTICIPANT_ID')}|{env('POCRA_CAPABILITY')}" + return ( + ("__MAUSAMGRAM_BINDING_KEY__", mausamgram_binding), + ("__AGMARKNET_BINDING_KEY__", agmarknet_binding), + ("__VISTAAR_BINDING_KEY__", vistaar_binding), + ("__VISTAAR_TOKEN_URL__", env("VISTAAR_TOKEN_URL")), + # Agmarknet's own token endpoint. A deployment fact, not a + # credential, so it is rendered directly -- the access name and + # password stay as variable NAMES in the config and reach the + # adapter through its environment. + ("__AGMARKNET_TOKEN_URL__", env("AGMARKNET_TOKEN_URL")), + # Auth is per provider, so the participant id is a YAML KEY in + # the adapter config, not only half of a binding key. + ("__POCRA_BINDING_KEY__", pocra_binding), + ("__MAUSAMGRAM_PARTICIPANT_ID__", env("MAUSAMGRAM_PARTICIPANT_ID")), + ("__AGMARKNET_PARTICIPANT_ID__", env("AGMARKNET_PARTICIPANT_ID")), + ("__VISTAAR_PARTICIPANT_ID__", env("VISTAAR_PARTICIPANT_ID")), + ("__POCRA_PARTICIPANT_ID__", env("POCRA_PARTICIPANT_ID")), + # Telemetry. One switch drives all three signals: with every + # one false the plugin builds no exporter and never dials, so + # a stack running without the observability profile stays + # quiet instead of logging a refused connection on a loop. + ("__OTEL_ENABLED__", env("ADAPTER_OTEL_ENABLED", "true")), + ("__OTLP_ENDPOINT__", env("OTLP_ENDPOINT", "hyperdx:4317")), + ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))) + + +def _role_substitutions(role, identity): + """The six per-role placeholders: id, key id, and the four keys.""" + prefix = role.upper() + return ( + (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), + (f"__{prefix}_KEY_ID__", identity["keyOsid"]), + (f"__{prefix}_SIGNING_PRIVATE__", identity["signingPrivate"]), + (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), + (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), + (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"])) + + +def _fill(name, template, substitutions): + """Fill one template's placeholders; refuse if any survive.""" + for placeholder, value in substitutions: + template = template.replace(placeholder, value) + if "__" in template: + sys.exit(f"setup: {name}.yaml still has unrendered placeholders") + return template + + +def _write(rel, text): + """Write config/adapters/ (mode 600: it holds private keys).""" + out = ADAPTERS / rel + # A bare `docker compose up -d` before this script runs starts the + # adapters too, and Docker creates a DIRECTORY at a bind-mount source + # that does not exist. Writing would then fail with a bare + # IsADirectoryError that says nothing about the cause. + if out.is_dir(): + sys.exit( + f"setup: {out} is a directory, not a file.\n" + f" Docker created it, which means the adapters were started before this\n" + f" script ran. Bring them down, remove the empty directories and retry:\n" + f" docker compose down\n" + f" rmdir {out}\n" + f" make up") + out.write_text(text) + out.chmod(0o600) # holds a private key + print(f" config/adapters/{rel}") + + +def _render_one(name, template, substitutions): + """Fill one template and write config/adapters/.yaml (mode 600).""" + _write(f"{name}.yaml", _fill(name, template, substitutions)) + + +# ----------------------------------------------------- unified network layer +# +# The unified network layer runs the consumer, network and provider tiers as +# modules of ONE adapter (the unified single adapter). Its config is not a +# template of its own: it is BUILT from the three tier templates above, plus +# the few differences in config/adapters/unified-network-layer/overrides.yaml +# (module names and paths, the one routing file, extended schema, appName, +# port). A change to a tier template therefore reaches both stacks on the next +# run, and there is no second copy of any module to drift. +# +# Each module keeps its own tier identity, so both stacks use the same keys and +# registry rows. +UNIFIED_NETWORK_LAYER = ADAPTERS / "unified-network-layer" + + +def _render_unified_network_layer(identities, shared): + try: + import yaml # only this mode needs it; the multi-adapter stack does not + except ImportError: + print(" config/adapters/unified-network-layer/unified-network-layer.yaml" + " SKIPPED: needs PyYAML (pip install pyyaml, or apt install python3-yaml)") + return + overrides = yaml.safe_load((UNIFIED_NETWORK_LAYER / "overrides.yaml").read_text()) + tiers = {} + for role in ROLES: + text = _fill(role, (ADAPTERS / f"{role}.yaml.tmpl").read_text(), + _role_substitutions(role, identities[role]) + shared) + tiers[role] = yaml.safe_load(text) + + modules = [] + for spec in overrides["modules"]: + module = tiers[spec["from"]]["modules"][spec["index"]] + module["name"], module["path"] = spec["name"], spec["path"] + plugins = module["handler"].get("plugins", {}) + if "router" in plugins: + plugins["router"]["config"]["routingConfig"] = overrides["routingConfig"] + if "schemaValidator" in plugins: + plugins["schemaValidator"]["config"]["extendedSchema_enabled"] = \ + overrides["extendedSchemaEnabled"] + modules.append(module) + + # Process-level settings come from the consumer template; only appName, + # the port and the telemetry service name change. + doc = {k: v for k, v in tiers["consumer"].items() if k != "modules"} + doc["appName"] = overrides["appName"] + doc["http"]["port"] = overrides["port"] + otel = doc.get("plugins", {}).get("otelsetup") + if otel: + otel["config"]["serviceName"] = overrides["otelServiceName"] + doc["modules"] = modules + + header = ("# GENERATED by bin/setup.py from consumer/network/provider.yaml.tmpl\n" + "# and unified-network-layer/overrides.yaml. Do not edit: edit those and\n" + "# re-run setup. Holds private keys (mode 600, gitignored).\n") + _write("unified-network-layer/unified-network-layer.yaml", + header + yaml.safe_dump(doc, sort_keys=False, width=1000)) + + +def render(identities): + print("configs:") + shared = _shared_substitutions() for role in ROLES: - identity = identities[role] template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() - prefix = role.upper() - for placeholder, value in ( - (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), - (f"__{prefix}_KEY_ID__", identity["keyOsid"]), - (f"__{prefix}_SIGNING_PRIVATE__", identity["signingPrivate"]), - (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), - (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), - (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"]), - ("__MAUSAMGRAM_BINDING_KEY__", mausamgram_binding), - ("__AGMARKNET_BINDING_KEY__", agmarknet_binding), - ("__VISTAAR_BINDING_KEY__", vistaar_binding), - ("__VISTAAR_TOKEN_URL__", env("VISTAAR_TOKEN_URL")), - # Agmarknet's own token endpoint. A deployment fact, not a - # credential, so it is rendered directly -- the access name and - # password stay as variable NAMES in the config and reach the - # adapter through its environment. - ("__AGMARKNET_TOKEN_URL__", env("AGMARKNET_TOKEN_URL")), - # Auth is per provider, so the participant id is a YAML KEY in - # the adapter config, not only half of a binding key. - ("__POCRA_BINDING_KEY__", pocra_binding), - ("__MAUSAMGRAM_PARTICIPANT_ID__", env("MAUSAMGRAM_PARTICIPANT_ID")), - ("__AGMARKNET_PARTICIPANT_ID__", env("AGMARKNET_PARTICIPANT_ID")), - ("__VISTAAR_PARTICIPANT_ID__", env("VISTAAR_PARTICIPANT_ID")), - ("__POCRA_PARTICIPANT_ID__", env("POCRA_PARTICIPANT_ID")), - # Telemetry. One switch drives all three signals: with every - # one false the plugin builds no exporter and never dials, so - # a stack running without the observability profile stays - # quiet instead of logging a refused connection on a loop. - ("__OTEL_ENABLED__", env("ADAPTER_OTEL_ENABLED", "true")), - ("__OTLP_ENDPOINT__", env("OTLP_ENDPOINT", "hyperdx:4317")), - ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))): - template = template.replace(placeholder, value) - if "__" in template: - sys.exit(f"setup: {role}.yaml still has unrendered placeholders") - out = ADAPTERS / f"{role}.yaml" - # A bare `docker compose up -d` before this script runs starts the - # adapters too, and Docker creates a DIRECTORY at a bind-mount source - # that does not exist. Writing would then fail with a bare - # IsADirectoryError that says nothing about the cause. - if out.is_dir(): - sys.exit( - f"setup: {out} is a directory, not a file.\n" - f" Docker created it, which means the adapters were started before this\n" - f" script ran. Bring them down, remove the empty directories and retry:\n" - f" docker compose down\n" - f" rmdir config/adapters/*.yaml\n" - f" make up") - out.write_text(template) - out.chmod(0o600) # holds a private key - print(f" config/adapters/{role}.yaml") + _render_one(role, template, + _role_substitutions(role, identities[role]) + shared) + _render_unified_network_layer(identities, shared) if __name__ == "__main__": diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 1e7ac6e1..661f0513 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -165,6 +165,50 @@ done_banner() { NEXT } +# ----------------------------------------------- unified network layer + +# The unified network layer: ONE adapter runs the consumer, network and +# provider tiers as modules of one process (the unified single adapter), with +# tier-to-tier hops in-process (inproc://). The multi-adapter containers are +# NOT started. Mocks are not started either, as in `up`. +# +# Its stack is its own complete compose file (docker-compose.unified-network-layer.yml), +# so docker-compose.yml is not touched. HTTP=1 adds the overlay that sends tier +# hops over http://localhost instead (any image). Passing -f overrides +# COMPOSE_FILE, so the Apple-silicon override is added here when it exists. +unified_network_layer_compose() { + local files=(-f docker-compose.unified-network-layer.yml) + [ -f docker-compose.arm64.yml ] && files+=(-f docker-compose.arm64.yml) + [ "${HTTP:-}" = 1 ] && files+=(-f docker-compose.unified-network-layer-http.yml) + docker compose "${files[@]}" "$@" +} + +up_unified_network_layer() { + preflight + local image + image="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=' .env | cut -d= -f2- || true)" + if [ "${HTTP:-}" != 1 ]; then + # inproc:// needs an image built with in-process routing. + [ -n "$image" ] || die "UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE is empty in .env: set it to an image with in-process routing (network-adapter:inproc), or run with HTTP=1" + docker image inspect "$image" >/dev/null 2>&1 \ + || die "image $image not found. Build it: bin/unified-network-layer-up.sh (or run with HTTP=1)" + fi + step 1 3 "registry and discovery (unified network layer compose file)" + unified_network_layer_compose up -d sunbird-registry-service discovery-service + up_setup 3 + step 3 3 "network-layer-adapter (consumer + network + provider in one process, $([ "${HTTP:-}" = 1 ] && echo 'http' || echo 'inproc') hops)" + unified_network_layer_compose up -d --force-recreate network-layer-adapter + printf '\n\033[1;32m==> unified network layer is up\033[0m on 127.0.0.1:%s\n' \ + "$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_PORT=' .env | cut -d= -f2 || true)" +} + +# Stops only the unified network layer adapter; registry, discovery and data stay. +down_unified_network_layer() { + step 1 1 "stopping network-layer-adapter only" + unified_network_layer_compose stop network-layer-adapter + unified_network_layer_compose rm -f network-layer-adapter +} + # ----------------------------------------------------------------- down # Containers and networks go; named volumes stay. So the registry's Postgres @@ -327,6 +371,10 @@ bin/stack.sh up the whole stack: registry+discovery -> setup.py -> adapters -> gateway (public, 80/443) -> hyperdx (wants 16 GB) up-core steps 1-3 only. Nothing public, no ClickHouse. + up-unified-network-layer docker-compose.unified-network-layer.yml: registry+discovery + -> setup.py -> network-layer-adapter (inproc hops; + HTTP=1 for http hops) + down-unified-network-layer stop and remove network-layer-adapter only down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only @@ -344,6 +392,8 @@ USAGE case "${1:-}" in up) up ;; up-core) up_core ;; + up-unified-network-layer) up_unified_network_layer ;; + down-unified-network-layer) down_unified_network_layer ;; down) down ;; destroy) destroy ;; setup) setup ;; diff --git a/quick-start/bin/unified-network-layer-up.sh b/quick-start/bin/unified-network-layer-up.sh new file mode 100755 index 00000000..f14e7041 --- /dev/null +++ b/quick-start/bin/unified-network-layer-up.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Build the adapter image with in-process (inproc://) routing and run the +# unified network layer on it -- ONE adapter for the consumer, network and +# provider tiers (the unified single adapter) -- then prove it works. +# +# bin/unified-network-layer-up.sh build image, start stack, smoke test +# bin/unified-network-layer-up.sh --no-build reuse the existing image +# HTTP=1 bin/unified-network-layer-up.sh tier hops over http://localhost instead +# +# Env: +# ADAPTER_SRC network-adapter checkout to build from (default ../../network-adapter) +# IMAGE image tag to build and run (default: UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE +# from .env, else network-adapter:inproc) +# +# Uses docker-compose.unified-network-layer.yml (and docker-compose.arm64.yml +# when present). Leaves the unified network layer RUNNING. +# Stop it: make down-unified-network-layer +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +BUILD=1 +[ "${1:-}" = "--no-build" ] && BUILD=0 +ADAPTER_SRC="${ADAPTER_SRC:-$ROOT/../../network-adapter}" +ENV_IMAGE="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=' .env 2>/dev/null | cut -d= -f2-)" +IMAGE="${IMAGE:-${ENV_IMAGE:-network-adapter:inproc}}" + +COMPOSE=(-f docker-compose.unified-network-layer.yml) +[ -f docker-compose.arm64.yml ] && COMPOSE+=(-f docker-compose.arm64.yml) +# HTTP=1: tier hops over http://localhost instead of inproc:// (any image). +[ "${HTTP:-}" = 1 ] && COMPOSE+=(-f docker-compose.unified-network-layer-http.yml) +PORT="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_PORT=' .env 2>/dev/null | cut -d= -f2)"; PORT="${PORT:-9210}" + +step() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; } +die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } + +[ -f .env ] || die ".env missing: cp .env.example .env and fill it in" + +# --- 1. build --------------------------------------------------------------- +if [ "$BUILD" = 1 ]; then + step "1/5 build $IMAGE from $ADAPTER_SRC" + [ -f "$ADAPTER_SRC/core/module/handler/inproc.go" ] \ + || die "$ADAPTER_SRC has no core/module/handler/inproc.go: check out a branch with in-process routing" + BRANCH="$(git -C "$ADAPTER_SRC" rev-parse --abbrev-ref HEAD)" + COMMIT="$(git -C "$ADAPTER_SRC" rev-parse --short HEAD)" + DIRTY="clean"; [ -n "$(git -C "$ADAPTER_SRC" status --porcelain)" ] && DIRTY="dirty" + echo " source: $BRANCH @ $COMMIT ($DIRTY)" + docker build --progress=plain -f "$ADAPTER_SRC/Dockerfile.adapter-with-plugins" \ + --build-arg ONIX_VERSION="$BRANCH" --build-arg GIT_COMMIT="$COMMIT" \ + --build-arg GIT_TREE_STATE="$DIRTY" --build-arg BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -t "$IMAGE" -t "network-adapter:inproc-$COMMIT" "$ADAPTER_SRC" +else + step "1/5 reuse $IMAGE" + docker image inspect "$IMAGE" >/dev/null 2>&1 || die "$IMAGE not found; run without --no-build" +fi + +# --- 2. infrastructure ------------------------------------------------------ +step "2/5 registry, discovery, mocks" +docker compose "${COMPOSE[@]}" up -d sunbird-registry-service discovery-service mock-imd mock-agmarknet + +# --- 3. config -------------------------------------------------------------- +step "3/5 keys, registry entries, rendered config (bin/setup.py)" +python3 bin/setup.py >/dev/null +echo " config/adapters/unified-network-layer/unified-network-layer.yaml rendered" + +# --- 4. adapter on the inproc image ----------------------------------------- +step "4/5 unified network layer on $IMAGE (inproc routing)" +# The multi-adapter containers share this project; only one topology at a time. +docker compose -f docker-compose.yml stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true +UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE="$IMAGE" docker compose "${COMPOSE[@]}" up -d --force-recreate network-layer-adapter +for i in $(seq 1 60); do + curl -sf "http://127.0.0.1:$PORT/health" >/dev/null && break + [ "$i" = 60 ] && { docker logs --tail 30 network-layer-adapter; die "adapter not healthy after 60s"; } + sleep 1 +done +echo " image: $(docker inspect network-layer-adapter --format '{{.Config.Image}}')" +echo " health: $(curl -s "http://127.0.0.1:$PORT/health")" + +# --- 5. smoke test ---------------------------------------------------------- +step "5/5 publish, discover, select through the unified network layer" +python3 bin/poc-flow.py --mode unified-network-layer --n 3 +echo +echo " in-process hops seen in the adapter log:" +docker logs --since 2m network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' + +printf '\n\033[1;32mUnified network layer running on %s (inproc routing). Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" +echo "Demo: bin/demo-unified-network-layer.sh Postman: api-collection/OpenAgriNet.unified-network-layer.collection.json" diff --git a/quick-start/config/adapters/unified-network-layer/overrides.yaml b/quick-start/config/adapters/unified-network-layer/overrides.yaml new file mode 100644 index 00000000..ac685f56 --- /dev/null +++ b/quick-start/config/adapters/unified-network-layer/overrides.yaml @@ -0,0 +1,46 @@ +# Unified network layer: the consumer, network and provider tiers as modules of +# ONE adapter process (the unified single adapter). +# +# This file holds only what differs from the three tier templates beside it +# (consumer.yaml.tmpl, network.yaml.tmpl, provider.yaml.tmpl). bin/setup.py +# builds unified-network-layer.yaml from those templates plus this file, so a +# change to a tier template reaches both the multi-adapter stack and this one +# on the next run -- there is no second copy of any module to keep in step. +# +# Every module keeps its own tier identity (__CONSUMER_*__, __NETWORK_*__, +# __PROVIDER_*__): the same keys and registry rows the multi-adapter stack +# uses, so either stack can run against the same registry. + +appName: network-layer-adapter +otelServiceName: network-layer-adapter +# Inside the container. The host port is UNIFIED_NETWORK_LAYER_ADAPTER_PORT. +port: 9200 + +# Every module's router reads this one file. Compose mounts routing-inproc.yaml +# here (default), or routing-http.yaml with HTTP=1. +routingConfig: /app/config/routing-unified-network-layer.yaml + +# Extended schema validation on for every module that validates schemas. +extendedSchemaEnabled: "true" + +# Which tier-template module becomes which module here. `index` is the +# module's position in that template's `modules:` list. +modules: + - from: consumer + index: 0 + name: consumer + path: /consumer/ + - from: network + index: 0 + name: network + path: /network/ + - from: provider + index: 0 + name: provider + path: /provider/ + # The provider's catalogue publishing. An exact path: stripping it leaves the + # empty endpoint, which the routing file keys on with excludeAction. + - from: provider + index: 1 + name: provider-publish + path: /provider/publish diff --git a/quick-start/config/adapters/unified-network-layer/routing-http.yaml b/quick-start/config/adapters/unified-network-layer/routing-http.yaml new file mode 100644 index 00000000..af047b51 --- /dev/null +++ b/quick-start/config/adapters/unified-network-layer/routing-http.yaml @@ -0,0 +1,36 @@ +# Routing for the unified network layer: tier hops over HTTP to this same +# container (http://localhost), instead of in-process. Works with any adapter +# image. Selected with HTTP=1 (make up-unified-network-layer HTTP=1), which adds +# the overlay docker-compose.unified-network-layer-http.yml. +# +# localhost:9200 is the container's own listener -- it must match `port` in +# overrides.yaml. Discovery is a separate container either way. +routingRules: + # consumer discover, and publish arriving at /network/publish + - version: "2.0.0" + targetType: "url" + target: + url: "http://discovery-service:8080" + endpoints: + - discover + - publish + + # consumer -> provider tier, over loopback HTTP + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/provider" + endpoints: + - select + - init + - confirm + - status + + # provider publish module (/provider/publish, endpoint "") -> network publish, over loopback HTTP + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network/publish" + excludeAction: true + endpoints: + - "" diff --git a/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml b/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml new file mode 100644 index 00000000..d3ff6cc0 --- /dev/null +++ b/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml @@ -0,0 +1,39 @@ +# Routing for the unified network layer: tier hops IN-PROCESS (the default). +# +# One file for every module: the router keys rules on (version, endpoint) only, +# and no endpoint needs two targets -- consumer discover goes straight to +# discovery, and the network module only receives publish. +# +# inproc:// hands the request to that module of this same process as a +# function call: no socket, no port, no TCP. Needs an adapter image built with +# in-process routing (network-adapter feat/55-unified-adapter-inproc-routing). +# For any other image use HTTP=1 (routing-http.yaml). +routingRules: + # consumer discover, and publish arriving at /network/publish + - version: "2.0.0" + targetType: "url" + target: + url: "http://discovery-service:8080" + endpoints: + - discover + - publish + + # consumer -> provider tier + - version: "2.0.0" + targetType: "url" + target: + url: "inproc://provider" + endpoints: + - select + - init + - confirm + - status + + # provider publish module (/provider/publish, endpoint "") -> network publish + - version: "2.0.0" + targetType: "url" + target: + url: "inproc://network/publish" + excludeAction: true + endpoints: + - "" diff --git a/quick-start/docker-compose.unified-network-layer-http.yml b/quick-start/docker-compose.unified-network-layer-http.yml new file mode 100644 index 00000000..3df651b7 --- /dev/null +++ b/quick-start/docker-compose.unified-network-layer-http.yml @@ -0,0 +1,11 @@ +# Overlay on docker-compose.unified-network-layer.yml: tier-to-tier hops over +# HTTP to the container's own listener (http://localhost:9200/...) instead of +# in-process (inproc://). Works with any adapter image. +# +# make up-unified-network-layer HTTP=1 +# docker compose -f docker-compose.unified-network-layer.yml \ +# -f docker-compose.unified-network-layer-http.yml up -d +services: + network-layer-adapter: + volumes: + - ./config/adapters/unified-network-layer/routing-http.yaml:/app/config/routing-unified-network-layer.yaml:ro diff --git a/quick-start/docker-compose.unified-network-layer.yml b/quick-start/docker-compose.unified-network-layer.yml new file mode 100644 index 00000000..08a5eeb1 --- /dev/null +++ b/quick-start/docker-compose.unified-network-layer.yml @@ -0,0 +1,928 @@ +# The whole OAN stack in one file, UNIFIED NETWORK LAYER variant: registry, +# discovery, ONE adapter running the consumer, network and provider tiers as +# modules of one process (the unified single adapter), the telemetry stack and +# the public edge. +# +# This is docker-compose.yml with the three adapter services replaced by one +# `network-layer-adapter`. Everything else is the same, so use ONE of +# the two files, not both: they share the project name, container names and +# data volumes. +# +# make up-unified-network-layer (or: bin/unified-network-layer-up.sh) +# docker compose -f docker-compose.unified-network-layer.yml up -d +# +# Tier-to-tier hops are IN-PROCESS by default (inproc://, routing-inproc.yaml), +# which needs an adapter image built with in-process routing -- see +# UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE in .env (bin/unified-network-layer-up.sh +# builds it). The overlay docker-compose.unified-network-layer-http.yml +# (make up-unified-network-layer HTTP=1) sends them over http://localhost +# instead, with any image. +# +# The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among +# them authenticates nothing but that registry, and shares its postgres. +# +# Read it in tiers -- the banner comments below are the structure: +# +# registry sunbird-registry-postgres, sunbird-registry-keycloak, +# sunbird-registry-service +# discovery discovery-postgres, discovery-service +# adapter network-layer-adapter (consumer + network + provider) +# observability hyperdx profile: observability +# edge nginx-proxy-manager profile: reverse-proxy +# +# The last two are behind profiles because neither is needed to exercise the +# stack, and HyperDX is the heaviest thing here. +# +# ---------------------------------------------------------------- bringing up +# +# 1. cp .env.example .env change every credential -- they are +# shipped defaults +# 2. docker compose -f docker-compose.unified-network-layer.yml up -d sunbird-registry-service discovery-service +# everything EXCEPT the adapter +# 3. python3 bin/setup.py keys, the three tier identities, and +# config/adapters/unified-network-layer/unified-network-layer.yaml +# 4. docker compose -f docker-compose.unified-network-layer.yml up -d now the adapter +# 5. docker compose --profile reverse-proxy --profile observability up -d +# the edge and the telemetry stack +# +# Naming registry and discovery in step 2 is not tidiness. An adapter config +# is a bind-mounted FILE, and Docker creates a DIRECTORY at any bind-mount +# source that does not exist yet -- so starting an adapter before step 3 both +# wedges that container on a directory it cannot parse and leaves a directory +# where step 3 needs to write a file. +# +# Nothing is built here. Every image is pulled from the tags named in .env. +# +# ------------------------------------------------------------- what is public +# +# Exactly one container publishes on a routable interface: Nginx Proxy +# Manager, on 80 and 443. Everything else -- both Postgres instances, +# Keycloak, the registry, discovery, the HyperDX UI, and NPM's own admin UI on +# 81 -- publishes on 127.0.0.1 and is reachable only through an SSH tunnel: +# +# The HyperDX UI is the one item on that list with a second way in: it also has +# a proxy host, so it is readable at https:/// with the +# Access List's username and password. Its loopback publish is unchanged -- +# routing it did not open a port here. +# +# ssh -L 81:127.0.0.1:81 -L 8080:127.0.0.1:8080 \ +# -L 8081:127.0.0.1:8081 -L 8085:127.0.0.1:8085 -N you@the-vm +# +# The loopback binds are written literally rather than taken from a BIND_ADDR +# variable, which is what they used to be. One variable that moves the whole +# stack onto a public interface is a footgun once an edge exists to do that +# job deliberately for the one tier that should be reachable: behind those +# ports are a Keycloak whose admin password ships as a default and a registry +# whose write token anyone who has read .env.example can mint. +# +# ------------------------------------------------------------------- networks +# +# Two, and with a UI-configured proxy at the edge the split stops being +# decoration and becomes the actual control. NPM's "Forward Hostname" is a +# free-text field, so anyone with the admin password can point a public route +# at `sunbird-registry-service` or `sunbird-registry-keycloak`. NPM sits only on +# oan-edge, where neither name resolves and neither address is routable -- the three adapters and hyperdx +# are the only containers it shares a network with. Those four straddle both +# networks; everything else is internal-only. +# +# Neither is `internal: true`: the adapters and discovery fetch the Beckn spec +# from raw.githubusercontent.com at boot, the provider adapter calls an +# upstream API that lives outside this VM, and NPM has to reach Let's Encrypt. +# Cutting egress would break all three. + +# The stack's name, pinned rather than derived from the directory. Compose +# labels every container with it and prefixes every volume with it, so without +# this line the deployment is named after whatever the checkout is called: +# `quick-start_registry-data` names a directory, not a network. Pinning it also +# means the directory can be renamed without orphaning the registry's data. +name: openagrinet + +networks: + oan-internal: + name: oan-internal + oan-edge: + name: oan-edge + +volumes: + sunbird-registry-postgres-data: + discovery-postgres-data: + hyperdx-clickhouse-data: + nginx-proxy-manager-data: + nginx-proxy-manager-letsencrypt: + +x-logging: &logging + # Every container's stdout, tagged so the log console can name it. + # + # The driver is unchanged -- json-file is what Docker was already using. The + # two additions are: + # + # `labels`, which makes the json-file driver write the compose service name + # into each line's `attrs` field. Compose sets that label on every container + # already; without this opt it stays in the container's metadata, where the + # collector reading the log FILE cannot see it. config/observability/filelog.yaml + # turns it into service.name, and it is the only reason HyperDX can show + # "discovery-service" rather than 64 hex characters. + # + # `max-size`/`max-file`, which bound the files those logs accumulate in. + # Docker's default is unbounded, so today a chatty service fills the disk and + # takes Postgres down with it. Unrelated to telemetry and fixed here because + # this is the change that made the log files matter. + driver: json-file + options: + labels: com.docker.compose.service + max-size: "10m" + max-file: "3" + +x-adapter: &adapter + # Pulled, never built. Set ADAPTER_IMAGE in .env to the tag published for + # this environment -- `docker compose up -d` is the whole deployment step. + image: ${ADAPTER_IMAGE} + pull_policy: missing + restart: unless-stopped + # The adapters are the only services on oan-edge, and therefore the only + # ones NPM can reach. They still need oan-internal to read the registry and + # to call discovery, so they sit on both. + networks: [oan-internal, oan-edge] + logging: *logging + +x-adapter-env: &adapter-env + CONFIG_FILE: /app/config/adapter.yaml + # OpenTelemetry. These ARE read: the otelsetup plugin builds gRPC exporters + # and the SDK applies its OTEL_EXPORTER_OTLP_* environment config before any + # option the plugin passes. + # + # OTEL_EXPORTER_OTLP_INSECURE IS LOAD-BEARING, and the reason is worth + # knowing because the symptom names TLS and the cause does not. + # + # The SDK derives transport security from the endpoint's SCHEME: + # + # switch u.Scheme { case "http", "unix": WithInsecure(); default: WithSecure() } + # + # "default" includes an EMPTY scheme. OTLP_ENDPOINT is a bare host:port, + # because that is what the plugin's own otlpEndpoint config field takes -- + # so without the scheme prefix below, or this flag, the exporter dials TLS + # at hyperdx's plaintext 4317 and logs, once per export interval: + # + # failed to upload metrics: ... authentication handshake failed: + # tls: first record does not look like a TLS handshake + # + # The flag is set explicitly rather than left to the scheme, so a future + # edit to the URL cannot silently turn TLS back on. + # + # This does not weaken anything that was encrypted: the collector is on + # oan-internal, reachable from nowhere else, and serves plaintext OTLP. + # + # There is also a bug underneath, in the plugin rather than here: it passes + # insecure credentials via WithDialOption, and NewGRPCConfig appends its own + # default TLS credentials AFTER user options, so gRPC takes the TLS ones and + # the plugin's setting is silently discarded. The env var is what actually + # controls this until that is fixed upstream. + # + # OTEL_EXPORTER_OTLP_PROTOCOL is deliberately absent. The plugin constructs + # gRPC exporters in code, so a protocol preference here is ignored and only + # suggests the transport is configurable from this file. + OTEL_EXPORTER_OTLP_ENDPOINT: http://${OTLP_ENDPOINT:-hyperdx:4317} + OTEL_EXPORTER_OTLP_INSECURE: "true" + +services: + # ========================================================================== + # registry -- who is on the network, their public keys, and which upstream + # API answers which capability. + # ========================================================================== + sunbird-registry-postgres: + image: postgres:14 + container_name: sunbird-registry-postgres + restart: unless-stopped + logging: *logging + networks: [oan-internal] + environment: + POSTGRES_DB: ${REGISTRY_DB_NAME} + POSTGRES_USER: ${REGISTRY_DB_USER} + POSTGRES_PASSWORD: ${REGISTRY_DB_PASSWORD} + volumes: + - sunbird-registry-postgres-data:/var/lib/postgresql/data + # No published port at all, not even on loopback: psql runs through + # `docker compose exec sunbird-registry-postgres`, so publishing one would only widen + # the surface without adding a way in. + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${REGISTRY_DB_USER}"] + interval: 5s + timeout: 5s + retries: 20 + + sunbird-registry-keycloak: + image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest + container_name: sunbird-registry-keycloak + restart: unless-stopped + logging: *logging + networks: [oan-internal] + volumes: + - ./config/registry/imports:/opt/jboss/keycloak/imports + environment: + - DB_VENDOR=postgres + - DB_ADDR=sunbird-registry-postgres + - DB_PORT=5432 + - DB_DATABASE=${REGISTRY_DB_NAME} + - DB_USER=${REGISTRY_DB_USER} + - DB_PASSWORD=${REGISTRY_DB_PASSWORD} + - KEYCLOAK_USER=${KEYCLOAK_ADMIN_USER} + - KEYCLOAK_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD} + - KEYCLOAK_IMPORT=/opt/jboss/keycloak/imports/realm-export.json + - PROXY_ADDRESS_FORWARDING=true + ports: + # Loopback only. This is an admin console with an imported realm, and + # the tunnel is how you reach it. + - "127.0.0.1:${KEYCLOAK_PORT}:8080" + - "127.0.0.1:${KEYCLOAK_ADMIN_PORT}:9990" + depends_on: + sunbird-registry-postgres: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9990/ || exit 1"] + interval: 10s + timeout: 10s + retries: 30 + + sunbird-registry-service: + image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_IMAGE_TAG} + container_name: sunbird-registry-service + restart: unless-stopped + logging: *logging + # oan-internal ONLY, and that is the decision, not an omission. NPM sits + # on oan-edge alone, so it cannot resolve the name `registry` -- a proxy + # host pointed here fails to start rather than quietly working. Attaching + # oan-edge is the one edit that would make the registry publishable, so it + # is the one edit to refuse. + # + # Why not route it. POST /api/v1/Participant/search takes NO token, which + # makes it tempting: it is the one call a network peer needs. But + # SunbirdRC uses POST for both reads and writes -- /Participant/search + # reads, /Participant creates -- so no method rule separates them, and a + # route publishes the write surface the moment anything can mint a token. + # Today nothing can, for one reason only: Keycloak publishes on + # 127.0.0.1. That is a decision forty lines up, and a route here would + # silently depend on it. + # + # What this costs: nothing outside the VM can write a registry row, which + # is why bin/setup.py seeds all seven participants and all four bindings from + # inside. It is also why the Postman collection has no registry request -- + # there is no way for one to work. + # + # Reaching it for debugging is an SSH tunnel to ${REGISTRY_PORT}. + networks: [oan-internal] + volumes: + # Schemas are read at startup, so a change here needs this service + # restarted before the registry will honour it. + - ./config/registry/schemas:/home/sunbirdrc/config/public/_schemas + environment: + - connectionInfo_uri=jdbc:postgresql://sunbird-registry-postgres:5432/${REGISTRY_DB_NAME} + - connectionInfo_username=${REGISTRY_DB_USER} + - connectionInfo_password=${REGISTRY_DB_PASSWORD} + - search_providerName=dev.sunbirdrc.registry.service.NativeSearchService + - authentication_enabled=true + - sunbird_sso_realm=${KEYCLOAK_REALM} + - sunbird_sso_url=http://sunbird-registry-keycloak:8080/auth + # Without this the service resolves its issuer to localhost -- which + # inside the container is itself -- and the security filter chain fails + # to build, so the whole registry never starts. + - OAUTH2_RESOURCES_0_URI=http://sunbird-registry-keycloak:8080/auth/realms/${KEYCLOAK_REALM} + - OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH=realm_access.roles + - identity_provider=dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + - sunbird_sso_admin_client_id=${KEYCLOAK_ADMIN_CLIENT_ID} + - sunbird_sso_client_id=${KEYCLOAK_CLIENT_ID} + - sunbird_sso_admin_client_secret=${KEYCLOAK_ADMIN_CLIENT_SECRET} + - sunbird_keycloak_user_set_password=true + - sunbird_keycloak_user_password=${KEYCLOAK_DEFAULT_USER_PASSWORD} + - encryption_enabled=false + - event_enabled=false + - idgen_enabled=false + - claims_enabled=false + - did_enabled=false + - signature_enabled=false + - certificate_enabled=false + - filestorage_enabled=false + - notification_enabled=false + - notification_async_enabled=false + - async_enabled=false + - webhook_enabled=false + - registry_base_apis_enable=false + - manager_type=DefinitionsManager + - expand_reference=false + - swagger_title=OAN Registry + - logging.level.root=INFO + ports: + # Loopback only, and bin/setup.py talks to exactly this: it derives + # http://localhost:${REGISTRY_PORT} and runs on the VM. + - "127.0.0.1:${REGISTRY_PORT}:8081" + depends_on: + sunbird-registry-postgres: + condition: service_healthy + sunbird-registry-keycloak: + condition: service_healthy + healthcheck: + # This image ships no curl, wget, nc or bash, so the check reads the + # kernel's own table instead: 1F91 is 8081 in hex, and the service binds + # v6, hence tcp6. It proves the port is accepting connections, which is + # what everything downstream waits for. + test: ["CMD-SHELL", "grep -q ':1F91' /proc/net/tcp6"] + interval: 5s + timeout: 5s + retries: 60 + start_period: 40s + + # ========================================================================== + # discovery -- catalogue search, and its own Postgres. + # ========================================================================== + discovery-postgres: + # pgvector rather than plain postgres: the discovery service's HNSW index + # options arrived in 0.8. + image: pgvector/pgvector:0.8.0-pg16 + container_name: discovery-postgres + restart: unless-stopped + logging: *logging + networks: [oan-internal] + environment: + # Not read from .env, unlike the registry's pair, and so not covered by + # the credential rotation .env.example asks for. That is tolerable only + # because this database publishes no port and sits on an internal + # network; if it ever needs to be reachable, these move to .env first. + POSTGRES_USER: discovery + POSTGRES_PASSWORD: discovery + POSTGRES_DB: discovery + volumes: + - discovery-postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U discovery -d discovery"] + interval: 5s + timeout: 5s + retries: 20 + + discovery-service: + image: ${DISCOVERY_IMAGE} + pull_policy: missing + + # THE ONLY SERVICE HERE THAT CAN BE BUILT, and the exception is earned. It + # is never built by `make up` -- `image:` above still decides what runs, and + # a build only happens when you ask for one: + # + # DISCOVERY_IMAGE=discovery-service:local docker compose build discovery-service + # + # Why this one and not the adapters or the mocks: the published tag went + # stale in a way no error reports. Measured on 2026-09-10, + # ghcr.io/nisargabd/discovery-service:latest was built before the telemetry + # package existed -- its embedded build info lists no go.opentelemetry.io + # dependency at all. Against that image every OTEL_* setting below is + # accepted, nothing is exported, and the stack looks entirely healthy. A + # build you can run locally is what makes that falsifiable. + # + # The context defaults to a sibling checkout because that is where it sits + # for anyone who has both repos; point DISCOVERY_CONTEXT elsewhere if yours + # differs. Without that checkout the build fails on a missing directory, + # which is the correct failure -- there is no source to build. + build: + context: ${DISCOVERY_CONTEXT:-../../discovery-service} + + # PUBLIC BASE IMAGES, deliberately overriding the Dockerfile's own + # defaults. It defaults to dhi.io/* -- Docker Hardened Images, what CI + # and every shipped image use -- and those 401 on an anonymous pull. A + # quick-start reader is precisely the person without that subscription, + # so the public equivalents the Dockerfile documents are the right + # default HERE and the wrong one there. INSTALL_BUILD_TOOLS comes with + # them and is not optional: h3-go is a cgo binding around Uber's C + # library, DHI's -dev variant bundles a C toolchain and golang:alpine + # does not. + args: + BUILD_IMAGE: ${DISCOVERY_BUILD_IMAGE:-golang:1.27-alpine} + RUNTIME_IMAGE: ${DISCOVERY_RUNTIME_IMAGE:-gcr.io/distroless/static-debian12:nonroot} + INSTALL_BUILD_TOOLS: "true" + + # Bare names, so each is passed through from the environment if set and + # left to the Dockerfile's own default if not. No values invented here: + # compose cannot run `git describe`, and a stamp this file made up would + # be worse than an absent one. + # + # Unset, the binary reports `dev unknown 1970-01-01T00:00:00Z unknown` + # and its telemetry Resource carries that verbatim in build.commit, + # build.date and build.tree_state. Honest for a local build. To stamp + # one, build it in the discovery-service repo instead -- `make docker` + # is what derives all four -- or set them in the environment first. + # + # A key with NO VALUE is compose's pass-through form. It is not the same + # as `VERSION: ""`, which would pass an empty string and defeat the + # Dockerfile's default. + VERSION: + COMMIT: + BUILD_DATE: + TREE_STATE: + + container_name: discovery-service + restart: unless-stopped + logging: *logging + networks: [oan-internal] + environment: + DATABASE_URL: postgres://discovery:discovery@discovery-postgres:5432/discovery?sslmode=disable + DATABASE_AUTO_MIGRATE: "true" + # APP_NETWORK_ID is the discovery image's own variable, so the left side + # cannot be renamed with the rest -- only the .env name it reads from. + APP_NETWORK_ID: ${BECKN_NETWORK_ID} + SERVER_PORT: 8080 + VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} + + # ---- the publish geometry budget -------------------------------------- + # + # How many GeoJSON shapes ONE catalog may contribute to the spatial index. + # Past it the publish still answers 200, with status PARTIAL and a fault + # per shape naming its path -- so nothing is dropped in silence, but those + # shapes are not searchable. Set here rather than left to the image's own + # 256 because it is the one discovery setting whose right value depends + # entirely on what you publish, and the failure it causes reads as missing + # data rather than as configuration. + # + # Count SHAPES, not resources: the walker recognises GeoJSON by shape and + # not by field name, at any depth. A mandi catalog carrying a Point under + # both coverageAreas and market.location spends TWO per market -- 273 + # Maharashtra markets published that way need 546, and under 256 only the + # first 128 markets are findable by place. + GEO_MAX_GEOMETRIES_PER_CATALOG: ${DISCOVERY_MAX_GEOMETRIES_PER_CATALOG:-256} + + # ---- search ----------------------------------------------------------- + # + # OFF, against the image's own default of true. With it off a search + # matches on the structured index alone -- domain, capability and + # geometry -- and free-text terms stop contributing to what comes back. + # SEARCH_ENABLE_TEXT_SEARCH is the image's variable name, so the left + # side is its contract and only the value here is ours. + SEARCH_ENABLE_TEXT_SEARCH: "false" + + # ---- rate limit ------------------------------------------------------- + # + # EFFECTIVELY OFF, against the image's own default of 20 rps / burst 40. + # That default is right for a shared deployment and wrong for this one: + # a quick-start is read by someone running a scripted sweep -- setup.py, + # a Postman collection, a bulk publish, a benchmark loop -- and 20 rps is + # under all of them. What they get is AUT_RATE_LIMITED, and a 429 NACK + # carries no `catalogs`, so a throttled sweep reads as an empty result + # rather than as backpressure. Measured on 2026-09-12: a plain 20-request + # loop against /discover tripped it. + # + # NOT ZERO, and it cannot be: the boot requires burst >= rps > 0, on the + # same argument as the geometry budget above -- zero reads either as + # "unlimited" or as "refuse everything", and the config layer will not + # guess which. A number nothing reaches is the honest way to say off. + RATE_LIMIT_RPS: ${DISCOVERY_RATE_LIMIT_RPS:-100000} + RATE_LIMIT_BURST: ${DISCOVERY_RATE_LIMIT_BURST:-100000} + + # ---- telemetry -------------------------------------------------------- + # + # This block used to say the exporter was parsed and unread, and that + # `none` was therefore the honest default. That stopped being true when + # the telemetry package landed: the service exports spans and two pgxpool + # instruments over OTLP/gRPC, straight to hyperdx like the adapters. + # + # WHAT DOES NOT ARRIVE, so nobody goes looking for it: the five OAN + # `metric.code` streams -- the two *_total_count and the three *_percent. + # They are not instrumented in the service at all; they are derived from + # the exported span stream by a collector running the discovery-service + # repo's otel/collector.yaml, and this stack does not run one. Traces + # arrive, everything looks healthy, and those five are simply absent. + # That is a quick-start simplification, not the deployment shape -- see + # Appendix H. + OTEL_EXPORTER: ${DISCOVERY_OTEL_EXPORTER:-otlp} + + # NO SCHEME, and that is the opposite of the adapters two screens up -- + # worth reading beside the note on the x-adapter anchor, because the same + # bare host:port means opposite things on either side. + # + # This service does not consult the SDK's env lookup; it passes the + # endpoint in explicitly, having branched on whether it has a scheme: + # + # if hasScheme(endpoint) { WithEndpointURL(endpoint) } + # else { WithEndpoint(endpoint), WithInsecure() } + # + # So a bare host:port is plaintext HERE, where on the adapters a bare + # host:port is what silently selects TLS. Adding http:// to this line + # would not make it more explicit; it would route through the URL branch. + # OTEL_EXPORTER_OTLP_INSECURE is absent for the same reason: unread. + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-hyperdx:4317} + + # BOTH ARE REQUIRED once the exporter is otlp, and their absence fails + # the boot rather than degrading -- the service refuses to export a + # stream it cannot attribute. That refusal is why they are here: before + # this, flipping DISCOVERY_OTEL_EXPORTER=otlp would have stopped the container + # starting, naming a variable this file never set. + # + # APP_SUBSCRIBER_ID is the Resource's `producer` and every span's + # recipient.id. Empty means the whole stream lands under no participant, + # which is worse than no stream: an empty grouping key reads as data + # rather than as an absence. + # + # APP_DOMAIN is checked against the sector list the build declares, so a + # typo fails the boot instead of splitting this deployment's telemetry + # into a group of one. + APP_SUBSCRIBER_ID: ${DISCOVERY_SUBSCRIBER_ID} + APP_DOMAIN: ${DISCOVERY_DOMAIN:-Agriculture} + + # OTEL_SERVICE_NAME is deliberately absent, and it WAS SET HERE on both + # sides of this merge -- to `oan-discovery` before the container rename + # and to `discovery` after it. The service reads neither: `service.name` + # is a compile-time constant, `discovery-service`, because it says what + # SOFTWARE this is and so must not vary by deployment. Confirmed against + # a live stack on 2026-09-10 -- spans arrived as `discovery-service` with + # nothing set. The variable was worse than inert: it named a value you + # would then go hunting for in HyperDX and never find, and the rename + # gave it a second wrong value to hunt for. What varies per deployment is + # `producer`, from APP_SUBSCRIBER_ID above. + # config/common.yaml is baked into the image and is the reviewed default. + # To override a setting, copy config/discovery/instance.yaml.example to + # config/discovery/instance.yaml and uncomment the mount below -- compose + # creates a DIRECTORY if you mount a file that does not exist, which the + # service then fails to parse, so the file has to be there first. + # volumes: + # - ./config/discovery/instance.yaml:/app/config/instance.yaml:ro + ports: + # Loopback only. Discovery is reached by the network adapter over + # oan-internal; this publish exists for curl through the tunnel. + - "127.0.0.1:${DISCOVERY_PORT}:8080" + depends_on: + discovery-postgres: + condition: service_healthy + + # ========================================================================== + # adapters -- consumer, network and provider. Same image, three configs. + # + # Their configs are bind-mounted FILES that do not exist until bin/setup.py + # renders them from the .tmpl beside them, which is what the step ordering + # above is about. + # ========================================================================== + + # ========================================================================== + # mock upstreams -- stand-ins for the real provider APIs, so the stack can be + # exercised without their credentials. + # + # Pulled like everything else. The sources are in mock-server/ to be built and + # published once, not built here -- see mock-server/README.md. + # + # oan-internal only: an upstream is called BY the provider adapter and is + # never reached from outside, so NPM has no business seeing it. The published + # ports are for looking at the mock directly while debugging. + # + # Both reproduce the awkward parts of the services they stand in for on + # purpose. A tidy mock would let a mapping pass here and fail in the real + # deployment. + # ========================================================================== + mock-imd: + image: ${MOCKIMD_IMAGE} + pull_policy: missing + container_name: mock-imd + restart: unless-stopped + logging: *logging + networks: [oan-internal] + command: + - "-addr=:9100" + # How many forecast days it answers with. The mapping reads however many + # arrive, so this is the knob for checking that it does. + - "-days=${MOCKIMD_DAYS:-3}" + ports: + - "127.0.0.1:${MOCKIMD_PORT}:9100" + + mock-agmarknet: + image: ${MOCKAGMARKNET_IMAGE} + pull_policy: missing + container_name: mock-agmarknet + restart: unless-stopped + logging: *logging + networks: [oan-internal] + command: + - "-addr=:9101" + # It answers 401 without this token, which is what proves the adapter + # sent one rather than the call happening to work anyway. + - "-token=${MOCKAGMARKNET_TOKEN:-local-mandi-token}" + - "-days=${MOCKAGMARKNET_DAYS:-2}" + ports: + - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" + + # The unified network layer: the consumer, network and provider tiers as + # modules of ONE adapter process (the unified single adapter), on ONE config + # built by bin/setup.py from the three tier templates + # (config/adapters/unified-network-layer/unified-network-layer.yaml), with ONE + # routing file. Tier-to-tier hops are inproc:// -- a function call inside the + # process -- so there is no consumer-adapter, network-adapter or + # provider-adapter here. + # + # :9200 inside, ${UNIFIED_NETWORK_LAYER_ADAPTER_PORT} on 127.0.0.1 + # /consumer/* the caller's entry (bap), takes unsigned requests + # /network/* hands publish on to discovery + # /provider/* calls the upstream, answers synchronously + # /provider/publish sends the provider's catalogue to /network/publish + # + # Each module signs as its own tier identity (consumer, network, provider): + # the same keys and registry rows as the multi-adapter stack. + network-layer-adapter: + <<: *adapter + container_name: network-layer-adapter + # inproc:// needs an image built with in-process routing; empty falls back + # to ADAPTER_IMAGE, which then needs HTTP=1. + image: ${UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE:-${ADAPTER_IMAGE}} + depends_on: + sunbird-registry-service: + condition: service_healthy + discovery-service: + condition: service_started + environment: + <<: *adapter-env + OTEL_SERVICE_NAME: network-layer-adapter + # The weather upstream's credential, as ONE ready-made header value -- + # "Basic ", prefix included. Named here and read at call time, so + # the value is in neither a config file nor the registry. + MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} + # The mandi upstream's credentials, exchanged for a token at call time. + AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} + AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} + # Read only when MandiPrice is pointed back at mock-agmarknet, which takes + # a static token instead of issuing one. + MOCKAGMARKNET_TOKEN: ${MOCKAGMARKNET_TOKEN:-local-mandi-token} + # The knowledge provider's OAuth2 client credentials. The step fails fast + # and permanently when either is unset. + VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} + VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} + volumes: + - ./config/adapters/unified-network-layer/unified-network-layer.yaml:/app/config/adapter.yaml:ro + # Default: tier hops in-process. HTTP=1 (the -http overlay) mounts routing-http.yaml here instead. + - ./config/adapters/unified-network-layer/routing-inproc.yaml:/app/config/routing-unified-network-layer.yaml:ro + ports: + - "127.0.0.1:${UNIFIED_NETWORK_LAYER_ADAPTER_PORT:-9210}:9200" + + # ========================================================================== + # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI + # over it. + # + # docker compose --profile observability up -d + # + # Sizing: ClickHouse alone wants 2-4 GB. With this profile on, the VM needs + # 16 GB (t3.xlarge or m6i.xlarge); without it, 8 GB is workable. + # ========================================================================== + hyperdx: + # clickstack-local, not clickstack-all-in-one: local runs single-user with + # no team to create and no ingestion API key to mint, which is what makes + # `up -d` the whole setup step. It is also why it has NO login of its own + # -- so once this is routed (see networks, below) the Access List on the + # proxy host is the only authentication in front of every trace, log and + # metric the stack has emitted. It is not an optional hardening step. + image: clickhouse/clickstack-local:latest + container_name: hyperdx + restart: unless-stopped + logging: *logging + profiles: ["observability"] + + # oan-edge in addition to oan-internal, so NPM can resolve `hyperdx` and + # serve the UI on a hostname instead of an `ssh -L`. oan-internal stays, + # because that is the network the in-stack senders reach 4317/4318 on. + # + # This widens what a wrong click in NPM can reach, and the widening is + # worth stating rather than discovering: this container is also ClickHouse, + # so `hyperdx` is now a resolvable Forward Hostname for port 8123 -- an + # HTTP interface clickstack-local leaves open, over the same volume. The + # proxy host for the UI pins 8080 and is fine; nothing in NPM stops someone + # typing 8123 into a second one. Registry and Keycloak remain off oan-edge, + # so that bound is unchanged -- this is a deliberate trade of one name for + # not having to tunnel to read telemetry. + networks: [oan-internal, oan-edge] + env_file: + # Optional. Present so an existing .env.docker keeps working; compose + # errors on a missing env_file unless it is declared this way. + - path: .env.docker + required: false + environment: + # Self-instrumentation off: the telemetry stack reporting on itself is + # noise in the same tables the stack under test writes to, and at debug + # level it is most of the volume. + OTEL_SDK_DISABLED: "true" + HYPERDX_LOG_LEVEL: error + HYPERDX_USAGE_STATS_ENABLED: "false" + HYPERDX_USAGE_STATS_COLLECTION_ENABLED: "false" + OTEL_LOG_LEVEL: none + OTEL_TRACES_EXPORTER: none + OTEL_METRICS_EXPORTER: none + OTEL_LOGS_EXPORTER: none + USAGE_STATS_ENABLED: "false" + + # Container stdout -> this container's own collector. The supervisor + # appends this file to /etc/otelcol-contrib/config.yaml rather than + # replacing it (/etc/otel/supervisor.yaml.tmpl:27), so it adds a pipeline + # and changes nothing HyperDX configures for itself. The reasoning is in + # the file; the short version is that traces, metrics and AUDIT records + # arrive over OTLP and the console log did not arrive at all. + CUSTOM_OTELCOL_CONFIG_FILE: /etc/otelcol-contrib/filelog.yaml + ports: + # 8085, not the 8080 this image serves the UI on and not the 8081 it is + # usually published as: 8081 is REGISTRY_PORT here, and two services + # cannot claim one host port. + # + # Still 127.0.0.1 now that there is a route: the public path is NPM -> + # hyperdx:8080 across oan-edge, which is what makes the Access List + # unavoidable. Rebinding this to 0.0.0.0 would open a second door on the + # same UI with no credential on it, and the security group is the only + # thing that would be holding it shut. + - "127.0.0.1:${HYPERDX_UI_PORT:-8085}:8080" + # OTLP. Published on loopback only so a reverse tunnel can carry + # telemetry from a provider API running on someone's laptop; the + # in-stack senders reach hyperdx:4317/4318 over oan-internal and do not + # need these at all. + - "127.0.0.1:${HYPERDX_OTLP_GRPC_PORT:-4317}:4317" + - "127.0.0.1:${HYPERDX_OTLP_HTTP_PORT:-4318}:4318" + + # ClickHouse's HTTP interface, and it is published for one specific + # reason: clickstack-local is LOCAL mode, where the UI queries ClickHouse + # straight from the BROWSER with no API server in between. So the Host + # field in Team Settings is dialled by your laptop, not by the container + # -- its `http://localhost:8123` default means *your* machine, so with + # nothing listening there it fails with "Error connecting to ClickHouse + # server" however the proxy is set up. The route and the Access List are + # not implicated; the query never leaves the browser's own stack. + # + # A tunnel to 8123 is all it needs, and the UI can still be the public + # host -- the origin serving the UI and the address it queries are + # independent: + # + # ssh -L 8123:127.0.0.1:8123 -N you@the-vm + # then https://hyperdx./ with connection Host + # http://localhost:8123 (add -L 8085 if you want the UI local too) + # + # The catch is that this is PER VIEWER: a teammate who opens the public + # host without that tunnel gets the same error, and connections live in + # browser local storage, so each origin needs its own set up by hand. + # + # 127.0.0.1 and NOT negotiable, unlike the UI on 8085. This interface + # executes arbitrary SQL and clickstack-local leaves it open, so a + # 0.0.0.0 bind or an NPM route publishes the whole telemetry store -- + # and its own auth uses the same `Authorization: Basic` header an NPM + # Access List needs, so the two cannot be stacked. + # + # The real fix is clickstack-all-in-one: a server-side API queries + # ClickHouse, connections stop being browser state, and this publish and + # the tunnel both become unnecessary. See README Appendix H. + - "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123" + volumes: + - hyperdx-clickhouse-data:/var/lib/clickhouse + - ./config/observability/filelog.yaml:/etc/otelcol-contrib/filelog.yaml:ro + + # Docker's own log files, READ-ONLY. This is the container that reads + # every other container's stdout, which is a real widening of what a + # compromise here reaches -- stated rather than buried, because it is the + # cost of the feature. + # + # The path is the daemon's, not the host's. On Docker Desktop for macOS + # it lives inside the LinuxKit VM and the bind still resolves (verified + # on 28.2.2). On rootless Docker, Podman or colima it does not, and the + # collector simply finds no files -- the stack comes up, the pipeline + # runs, and the log console stays empty. + # + # "No logs arrive" has TWO causes that look identical from outside, so + # check them in this order -- the second is the more common one and the + # cheaper test: + # + # 1. The collector never built the pipeline, because something in + # filelog.yaml did not parse or names a component the merged config + # no longer supplies. The container stays UP and the UI keeps + # serving; only the collector is down. `docker compose ps` says + # unhealthy, because the healthcheck below probes 13133 as well + # as 8080. Then: docker compose logs hyperdx | grep -i error + # + # 2. This mount resolved to nothing, so the receiver has no files to + # read. The container is HEALTHY -- the collector is fine, it just + # has no input -- which is what makes this one silent: + # + # docker exec hyperdx ls /var/lib/docker/containers | head + # + # Healthy-but-empty is (2). Unhealthy-and-empty is (1). + - /var/lib/docker/containers:/var/lib/docker/containers:ro + healthcheck: + # wget, not curl: this image has busybox. And a GET, not `--spider`, + # which sends HEAD -- Next.js answers HEAD / with a 404 while serving + # GET / perfectly, so the probe failed against a UI that was working. + # + # 127.0.0.1 rather than localhost, because busybox resolves localhost to + # ::1 first and these listeners are v4. + # + # Two checks, because this container is two things. 8080 is the Next.js + # UI -- what 8085 publishes and what you actually open. 13133 is the + # collector's health_check extension, and the collector is what receives + # telemetry on 4317/4318: it can die while the UI keeps serving, which + # loses data silently and is the failure worth catching. + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/ && wget -q -O /dev/null http://127.0.0.1:13133/"] + interval: 10s + timeout: 5s + # ClickHouse creates its system tables on a cold volume, which takes + # minutes rather than the 80s the previous 5/30s pair allowed. Docker + # does not re-evaluate once a container has failed past `retries`, so a + # window that is merely tight leaves a permanent wrong label. + retries: 10 + start_period: 180s + + # ========================================================================== + # edge -- Nginx Proxy Manager. + # + # docker compose --profile reverse-proxy up -d + # + # NPM rather than a hand-written nginx.conf because it owns the part that is + # genuinely tedious to do by hand -- ACME. It requests, installs and renews + # Let's Encrypt certificates from a UI rather than from a certbot invocation + # someone has to remember. In exchange, the routing table stops being a file + # in this repo and becomes rows in NPM's SQLite database under the nginx-proxy-manager-data + # volume: reviewable only by opening the UI, and restorable only from that + # volume. See README for the proxy hosts to create. + # ========================================================================== + nginx-proxy-manager: + image: jc21/nginx-proxy-manager:latest + container_name: nginx-proxy-manager + restart: unless-stopped + logging: *logging + profiles: ["reverse-proxy"] + + # oan-edge ONLY, and this is the whole security argument for putting a + # UI-configured proxy in front of this stack. + # + # NPM's "Forward Hostname" is a free-text field. Anyone with the admin + # password can type `sunbird-registry-service`, `sunbird-registry-keycloak` or `discovery-postgres` into it. On + # this network none of those names resolve and none of those addresses are + # routable from here -- the three adapters, and hyperdx when the + # observability profile is up, are the only containers NPM shares a network + # with. So the blast radius of a wrong click, or of the admin UI being + # reached by someone who should not have it, is bounded to that set. + # + # Putting NPM on oan-internal to "make things easier" would remove that + # bound entirely and put the registry's write API one form field away from + # the internet. Do not. + networks: [oan-edge] + + ports: + # Public. Both of these must be reachable from anywhere, not just from + # your address: Let's Encrypt validates HTTP-01 by fetching + # http:///.well-known/acme-challenge/... from its own servers, and + # those come from addresses you do not get to enumerate. An SG rule + # scoped to your IP will make certificate issuance fail with a challenge + # timeout. Use DNS-01 instead if 80 must stay closed -- see README. + - "0.0.0.0:${REVERSE_PROXY_HTTP_PORT:-80}:80" + - "0.0.0.0:${REVERSE_PROXY_HTTPS_PORT:-443}:443" + + # The admin UI, on loopback -- deliberately not the "publish it and + # restrict it in the firewall" arrangement. Two reasons. It ships with + # a known default login (admin@example.com / changeme) that is live from + # first boot until someone changes it, and it is the one surface here + # that can mint certificates and re-point every public route. A security + # group is a second system to keep in step with that; a loopback bind is + # not. Reach it over the tunnel: + # + # ssh -L 81:127.0.0.1:81 -N you@the-vm then http://127.0.0.1:81 + - "127.0.0.1:${NPM_ADMIN_PORT:-81}:81" + + volumes: + - nginx-proxy-manager-data:/data + - nginx-proxy-manager-letsencrypt:/etc/letsencrypt + + # Version-controlled nginx that NPM includes on its own. /data is a + # named volume and this bind mounts over one directory inside it, which + # works because the more specific mount wins. + # + # Only what is genuinely context-wide belongs here -- see the README for + # which of these NPM loads automatically and which is a paste job. + # + # NOT :ro, however much it wants to be. NPM's s6 init runs 50-ipv6.sh + # over everything under /data/nginx, which writes a .tmp beside each file + # and then chowns it. On a read-only mount both fail, the `prepare` + # service exits 1, s6 aborts the rest of the chain, and nginx never + # starts -- the container still reports Up, because s6 itself is alive, + # so the symptom is an empty reply on 81 and a healthcheck that never + # passes rather than anything that says "permissions". + # + # What NPM actually rewrites is `listen` directives, and neither file + # here has one, so the pass is a no-op on content. It does take + # ownership of the files, which is cosmetic and not tracked by git. + - ./config/reverse-proxy/npm-custom:/data/nginx/custom + + depends_on: + # Ordering only; the adapter image has no healthcheck. + # + # And ordering is not the whole problem. NPM writes a literal + # proxy_pass hostname per proxy host, which nginx resolves at reload and + # then caches -- so if you RECREATE an adapter (not merely restart it; + # a restart keeps the address) NPM keeps proxying to an address nothing + # answers on. The fix is one command, and it is worth knowing before you + # spend an afternoon on a 502: + # + # docker compose restart nginx-proxy-manager + # + # hyperdx is deliberately absent even though it is routed: it is behind + # the `observability` profile, and a depends_on would make it start + # whenever NPM does. That means the restart above is not optional after + # `make observability` -- NPM was started before the name existed, so the + # HyperDX host 502s until it re-resolves. + network-layer-adapter: + condition: service_started + + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:81/api/ || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s From bb5fea5d8d00cedf460e92e85fe014b99d9f4f85 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 13:02:07 +0530 Subject: [PATCH 03/43] refactor(quick-start): unified network layer uses loopback HTTP and per-module routing Tier hops in the unified network layer (the unified single adapter) now go over loopback HTTP to the container's own listener instead of inproc://, so any adapter image works and the config has the same shape as the multi-adapter stack -- split across containers, only the hosts change. - Routing: one file per module, as in the multi-adapter stack - consumer: single-network-layer-consumer-routing.yaml (discover -> localhost:9200/network, select/init/confirm/status -> localhost:9200/provider) - network: reuses the multi-adapter routing-network.yaml unchanged (discover, publish -> discovery-service) - provider-publish: single-network-layer-publish-routing.yaml (-> localhost:9200/network/publish) - overrides.yaml: routingConfig per module; setup.py fails if a module with a router has none - Remove routing-inproc.yaml, routing-http.yaml, the -http compose overlay and HTTP=1 - .env/.env.example: drop UNIFIED_NETWORK_LAYER_ADAPTER_PORT/_IMAGE; the stack uses ADAPTER_IMAGE like multi mode - Service and container renamed to network-layer-adapter (appName and OTEL service name match); host port 127.0.0.1:9200 - make up stops network-layer-adapter first, and the unified paths stop the multi adapters first, since both publish 9200 - bin/unified-network-layer-up.sh: no image build; starts the stack and smoke-tests publish, discover and select - README: unified network layer section updated --- quick-start/.env.example | 8 --- quick-start/Makefile | 2 +- quick-start/README.md | 36 ++++------ quick-start/bin/setup.py | 8 ++- quick-start/bin/stack.sh | 30 +++----- quick-start/bin/unified-network-layer-up.sh | 69 ++++++------------- .../unified-network-layer/overrides.yaml | 15 ++-- .../unified-network-layer/routing-http.yaml | 36 ---------- .../unified-network-layer/routing-inproc.yaml | 39 ----------- ...single-network-layer-consumer-routing.yaml | 35 ++++++++++ .../single-network-layer-publish-routing.yaml | 18 +++++ ...ker-compose.unified-network-layer-http.yml | 11 --- .../docker-compose.unified-network-layer.yml | 29 ++++---- 13 files changed, 129 insertions(+), 207 deletions(-) delete mode 100644 quick-start/config/adapters/unified-network-layer/routing-http.yaml delete mode 100644 quick-start/config/adapters/unified-network-layer/routing-inproc.yaml create mode 100644 quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml create mode 100644 quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml delete mode 100644 quick-start/docker-compose.unified-network-layer-http.yml diff --git a/quick-start/.env.example b/quick-start/.env.example index f1beeafd..3b6d9a10 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,14 +39,6 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 -# Unified network layer (make up-unified-network-layer): ONE adapter for the -# consumer, network and provider tiers. Host port, 127.0.0.1 only. -UNIFIED_NETWORK_LAYER_ADAPTER_PORT=9210 -# Image with in-process routing (inproc://), built by bin/unified-network-layer-up.sh -# from network-adapter feat/55-unified-adapter-inproc-routing. Needed for the -# default inproc:// hops; with HTTP=1 (hops over http://localhost) any image -# works, and empty falls back to ADAPTER_IMAGE. -UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=network-adapter:inproc # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 diff --git a/quick-start/Makefile b/quick-start/Makefile index 739a95e2..e68de4e0 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -25,7 +25,7 @@ help: up: ; @$(STACK) up up-core: ; @$(STACK) up-core down: ; @$(STACK) down -up-unified-network-layer: ; @HTTP=$(HTTP) $(STACK) up-unified-network-layer +up-unified-network-layer: ; @$(STACK) up-unified-network-layer down-unified-network-layer: ; @$(STACK) down-unified-network-layer destroy: ; @$(STACK) destroy diff --git a/quick-start/README.md b/quick-start/README.md index aedced2a..af55c2b9 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -198,38 +198,32 @@ The same network can run two ways. Both use the same registry, keys and data: | What runs | `consumer-adapter`, `network-adapter`, `provider-adapter` | one `network-layer-adapter` (the unified single adapter) | | Compose file | `docker-compose.yml` | `docker-compose.unified-network-layer.yml` | | Start / stop | `make up` / `make down` | `make up-unified-network-layer` / `make down-unified-network-layer` | -| Tier-to-tier hops | HTTP between containers | `inproc://`: a function call inside the process (default). `HTTP=1` sends them over `http://localhost` inside the container instead | -| Port | 9202 / 9201 / 9200 | `127.0.0.1:${UNIFIED_NETWORK_LAYER_ADAPTER_PORT}` (9210) | +| Tier-to-tier hops | HTTP between containers | HTTP over loopback (`http://localhost:9200`) inside the one container | +| Port | 9202 / 9201 / 9200 | `127.0.0.1:9200` (so only one mode runs at a time) | Use one or the other, not both: they share the project name and data volumes. - -**Image.** In-process hops need an adapter image built with in-process routing -(`network-adapter` branch `feat/55-unified-adapter-inproc-routing`). Set it in -`.env` as `UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE` (default `network-adapter:inproc`), -or let `bin/unified-network-layer-up.sh` build it and bring everything up. +Both use `ADAPTER_IMAGE` from `.env`; nothing else needs setting. **Config.** There is no hand-written config for this mode. `bin/setup.py` builds `config/adapters/unified-network-layer/unified-network-layer.yaml` from the three tier templates plus `unified-network-layer/overrides.yaml`. That file holds only -the module names and paths, the one routing file, extended schema, appName and -port. A change to a tier template reaches both modes on the next `setup.py`. +the module names and paths, each module's routing file, extended schema, +appName and port. A change to a tier template reaches both modes on the next `setup.py`. Each module keeps its own tier identity. -| Module | Path | From | -|---|---|---| -| `consumer` | `/consumer/` | `consumer.yaml.tmpl` | -| `network` | `/network/` | `network.yaml.tmpl` | -| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | -| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | +| Module | Path | From | Routing file | Routes | +|---|---|---|---|---| +| `consumer` | `/consumer/` | `consumer.yaml.tmpl` | `unified-network-layer/single-network-layer-consumer-routing.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | +| `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (multi file, reused) | discover, publish to discovery | +| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | +| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `unified-network-layer/single-network-layer-publish-routing.yaml` | to `localhost:9200/network/publish` | -Routing: `unified-network-layer/routing-inproc.yaml` (default). Discover and -publish go to discovery. Select, init, confirm and status go to -`inproc://provider`. Provider publish goes to `inproc://network/publish`. -`make up-unified-network-layer HTTP=1` mounts `routing-http.yaml` instead, with -the same targets on `http://localhost:9200/...`. It works with any adapter image. +The hops between modules are loopback HTTP calls to the same container. In the +multi-adapter stack only the hosts differ. 9200 is the container's own listener +and must match `port` in `overrides.yaml`. Tools: -- `bin/unified-network-layer-up.sh [--no-build]`: build the image, start, smoke test. +- `bin/unified-network-layer-up.sh`: start and smoke-test. - `bin/demo-unified-network-layer.sh`: step-by-step live demo. - `bin/bench-unified-network-layer.sh`: multi vs unified network layer, parallel curl, JSON results under `.bench/`. - `bin/poc-flow.py --mode multi|unified-network-layer`: quick publish/discover/select check. diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 18554818..25fe1639 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -643,8 +643,8 @@ def _render_one(name, template, substitutions): # modules of ONE adapter (the unified single adapter). Its config is not a # template of its own: it is BUILT from the three tier templates above, plus # the few differences in config/adapters/unified-network-layer/overrides.yaml -# (module names and paths, the one routing file, extended schema, appName, -# port). A change to a tier template therefore reaches both stacks on the next +# (module names and paths, each module's routing file, extended schema, +# appName, port). A change to a tier template therefore reaches both stacks on the next # run, and there is no second copy of any module to drift. # # Each module keeps its own tier identity, so both stacks use the same keys and @@ -672,7 +672,9 @@ def _render_unified_network_layer(identities, shared): module["name"], module["path"] = spec["name"], spec["path"] plugins = module["handler"].get("plugins", {}) if "router" in plugins: - plugins["router"]["config"]["routingConfig"] = overrides["routingConfig"] + if "routingConfig" not in spec: + sys.exit(f"setup: overrides.yaml module {spec['name']} has a router but no routingConfig") + plugins["router"]["config"]["routingConfig"] = spec["routingConfig"] if "schemaValidator" in plugins: plugins["schemaValidator"]["config"]["extendedSchema_enabled"] = \ overrides["extendedSchemaEnabled"] diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 661f0513..7ef6121e 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -143,6 +143,8 @@ up_adapters() { # new participant id and base URL in .env plus a setup.py re-run, because # the registry cannot repoint an existing row. step 3 "$1" "adapters (provider, network, consumer)" + # The unified network layer adapter also publishes 9200: stop it first. + docker compose -f docker-compose.unified-network-layer.yml stop network-layer-adapter >/dev/null 2>&1 || true docker compose up -d provider-adapter network-adapter consumer-adapter } @@ -169,37 +171,28 @@ NEXT # The unified network layer: ONE adapter runs the consumer, network and # provider tiers as modules of one process (the unified single adapter), with -# tier-to-tier hops in-process (inproc://). The multi-adapter containers are -# NOT started. Mocks are not started either, as in `up`. +# tier-to-tier hops over loopback HTTP inside the container. The multi-adapter +# containers are NOT started. Mocks are not started either, as in `up`. # # Its stack is its own complete compose file (docker-compose.unified-network-layer.yml), -# so docker-compose.yml is not touched. HTTP=1 adds the overlay that sends tier -# hops over http://localhost instead (any image). Passing -f overrides -# COMPOSE_FILE, so the Apple-silicon override is added here when it exists. +# so docker-compose.yml is not touched. Passing -f overrides COMPOSE_FILE, so +# the Apple-silicon override is added here when it exists. unified_network_layer_compose() { local files=(-f docker-compose.unified-network-layer.yml) [ -f docker-compose.arm64.yml ] && files+=(-f docker-compose.arm64.yml) - [ "${HTTP:-}" = 1 ] && files+=(-f docker-compose.unified-network-layer-http.yml) docker compose "${files[@]}" "$@" } up_unified_network_layer() { preflight - local image - image="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=' .env | cut -d= -f2- || true)" - if [ "${HTTP:-}" != 1 ]; then - # inproc:// needs an image built with in-process routing. - [ -n "$image" ] || die "UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE is empty in .env: set it to an image with in-process routing (network-adapter:inproc), or run with HTTP=1" - docker image inspect "$image" >/dev/null 2>&1 \ - || die "image $image not found. Build it: bin/unified-network-layer-up.sh (or run with HTTP=1)" - fi step 1 3 "registry and discovery (unified network layer compose file)" unified_network_layer_compose up -d sunbird-registry-service discovery-service up_setup 3 - step 3 3 "network-layer-adapter (consumer + network + provider in one process, $([ "${HTTP:-}" = 1 ] && echo 'http' || echo 'inproc') hops)" + step 3 3 "network-layer-adapter (consumer + network + provider in one process)" + # The multi-adapter containers share this project; one topology at a time. + docker compose stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true unified_network_layer_compose up -d --force-recreate network-layer-adapter - printf '\n\033[1;32m==> unified network layer is up\033[0m on 127.0.0.1:%s\n' \ - "$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_PORT=' .env | cut -d= -f2 || true)" + printf '\n\033[1;32m==> unified network layer is up\033[0m on 127.0.0.1:9200\n' } # Stops only the unified network layer adapter; registry, discovery and data stay. @@ -372,8 +365,7 @@ bin/stack.sh -> gateway (public, 80/443) -> hyperdx (wants 16 GB) up-core steps 1-3 only. Nothing public, no ClickHouse. up-unified-network-layer docker-compose.unified-network-layer.yml: registry+discovery - -> setup.py -> network-layer-adapter (inproc hops; - HTTP=1 for http hops) + -> setup.py -> network-layer-adapter down-unified-network-layer stop and remove network-layer-adapter only down stop everything, keep the data destroy stop everything and DELETE every volume diff --git a/quick-start/bin/unified-network-layer-up.sh b/quick-start/bin/unified-network-layer-up.sh index f14e7041..4446129f 100755 --- a/quick-start/bin/unified-network-layer-up.sh +++ b/quick-start/bin/unified-network-layer-up.sh @@ -1,17 +1,11 @@ #!/usr/bin/env bash -# Build the adapter image with in-process (inproc://) routing and run the -# unified network layer on it -- ONE adapter for the consumer, network and -# provider tiers (the unified single adapter) -- then prove it works. +# Start the unified network layer -- ONE adapter for the consumer, network and +# provider tiers (the unified single adapter), tier hops over loopback HTTP -- +# then prove it works. # -# bin/unified-network-layer-up.sh build image, start stack, smoke test -# bin/unified-network-layer-up.sh --no-build reuse the existing image -# HTTP=1 bin/unified-network-layer-up.sh tier hops over http://localhost instead -# -# Env: -# ADAPTER_SRC network-adapter checkout to build from (default ../../network-adapter) -# IMAGE image tag to build and run (default: UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE -# from .env, else network-adapter:inproc) +# bin/unified-network-layer-up.sh # +# Image: ADAPTER_IMAGE from .env (the same as the multi-adapter stack). Port 9200. # Uses docker-compose.unified-network-layer.yml (and docker-compose.arm64.yml # when present). Leaves the unified network layer RUNNING. # Stop it: make down-unified-network-layer @@ -20,55 +14,32 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" -BUILD=1 -[ "${1:-}" = "--no-build" ] && BUILD=0 -ADAPTER_SRC="${ADAPTER_SRC:-$ROOT/../../network-adapter}" -ENV_IMAGE="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE=' .env 2>/dev/null | cut -d= -f2-)" -IMAGE="${IMAGE:-${ENV_IMAGE:-network-adapter:inproc}}" - COMPOSE=(-f docker-compose.unified-network-layer.yml) [ -f docker-compose.arm64.yml ] && COMPOSE+=(-f docker-compose.arm64.yml) -# HTTP=1: tier hops over http://localhost instead of inproc:// (any image). -[ "${HTTP:-}" = 1 ] && COMPOSE+=(-f docker-compose.unified-network-layer-http.yml) -PORT="$(grep -E '^UNIFIED_NETWORK_LAYER_ADAPTER_PORT=' .env 2>/dev/null | cut -d= -f2)"; PORT="${PORT:-9210}" +envv() { grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-; } +PORT=9200 +IMAGE="$(envv ADAPTER_IMAGE)" step() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; } die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } [ -f .env ] || die ".env missing: cp .env.example .env and fill it in" +[ -n "$IMAGE" ] || die "no image: set ADAPTER_IMAGE in .env" -# --- 1. build --------------------------------------------------------------- -if [ "$BUILD" = 1 ]; then - step "1/5 build $IMAGE from $ADAPTER_SRC" - [ -f "$ADAPTER_SRC/core/module/handler/inproc.go" ] \ - || die "$ADAPTER_SRC has no core/module/handler/inproc.go: check out a branch with in-process routing" - BRANCH="$(git -C "$ADAPTER_SRC" rev-parse --abbrev-ref HEAD)" - COMMIT="$(git -C "$ADAPTER_SRC" rev-parse --short HEAD)" - DIRTY="clean"; [ -n "$(git -C "$ADAPTER_SRC" status --porcelain)" ] && DIRTY="dirty" - echo " source: $BRANCH @ $COMMIT ($DIRTY)" - docker build --progress=plain -f "$ADAPTER_SRC/Dockerfile.adapter-with-plugins" \ - --build-arg ONIX_VERSION="$BRANCH" --build-arg GIT_COMMIT="$COMMIT" \ - --build-arg GIT_TREE_STATE="$DIRTY" --build-arg BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - -t "$IMAGE" -t "network-adapter:inproc-$COMMIT" "$ADAPTER_SRC" -else - step "1/5 reuse $IMAGE" - docker image inspect "$IMAGE" >/dev/null 2>&1 || die "$IMAGE not found; run without --no-build" -fi - -# --- 2. infrastructure ------------------------------------------------------ -step "2/5 registry, discovery, mocks" +# --- 1. infrastructure ------------------------------------------------------ +step "1/4 registry, discovery, mocks" docker compose "${COMPOSE[@]}" up -d sunbird-registry-service discovery-service mock-imd mock-agmarknet -# --- 3. config -------------------------------------------------------------- -step "3/5 keys, registry entries, rendered config (bin/setup.py)" +# --- 2. config -------------------------------------------------------------- +step "2/4 keys, registry entries, rendered config (bin/setup.py)" python3 bin/setup.py >/dev/null echo " config/adapters/unified-network-layer/unified-network-layer.yaml rendered" -# --- 4. adapter on the inproc image ----------------------------------------- -step "4/5 unified network layer on $IMAGE (inproc routing)" +# --- 3. adapter ------------------------------------------------------------- +step "3/4 network-layer-adapter on $IMAGE (loopback tier hops)" # The multi-adapter containers share this project; only one topology at a time. docker compose -f docker-compose.yml stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true -UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE="$IMAGE" docker compose "${COMPOSE[@]}" up -d --force-recreate network-layer-adapter +docker compose "${COMPOSE[@]}" up -d --force-recreate network-layer-adapter for i in $(seq 1 60); do curl -sf "http://127.0.0.1:$PORT/health" >/dev/null && break [ "$i" = 60 ] && { docker logs --tail 30 network-layer-adapter; die "adapter not healthy after 60s"; } @@ -77,12 +48,12 @@ done echo " image: $(docker inspect network-layer-adapter --format '{{.Config.Image}}')" echo " health: $(curl -s "http://127.0.0.1:$PORT/health")" -# --- 5. smoke test ---------------------------------------------------------- -step "5/5 publish, discover, select through the unified network layer" +# --- 4. smoke test ---------------------------------------------------------- +step "4/4 publish, discover, select through the unified network layer" python3 bin/poc-flow.py --mode unified-network-layer --n 3 echo -echo " in-process hops seen in the adapter log:" +echo " tier hops seen in the adapter log:" docker logs --since 2m network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' -printf '\n\033[1;32mUnified network layer running on %s (inproc routing). Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" +printf '\n\033[1;32mUnified network layer running on %s. Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" echo "Demo: bin/demo-unified-network-layer.sh Postman: api-collection/OpenAgriNet.unified-network-layer.collection.json" diff --git a/quick-start/config/adapters/unified-network-layer/overrides.yaml b/quick-start/config/adapters/unified-network-layer/overrides.yaml index ac685f56..02392862 100644 --- a/quick-start/config/adapters/unified-network-layer/overrides.yaml +++ b/quick-start/config/adapters/unified-network-layer/overrides.yaml @@ -13,27 +13,31 @@ appName: network-layer-adapter otelServiceName: network-layer-adapter -# Inside the container. The host port is UNIFIED_NETWORK_LAYER_ADAPTER_PORT. +# Inside the container; published on 127.0.0.1:9200 (docker-compose.unified-network-layer.yml). port: 9200 -# Every module's router reads this one file. Compose mounts routing-inproc.yaml -# here (default), or routing-http.yaml with HTTP=1. -routingConfig: /app/config/routing-unified-network-layer.yaml # Extended schema validation on for every module that validates schemas. extendedSchemaEnabled: "true" # Which tier-template module becomes which module here. `index` is the -# module's position in that template's `modules:` list. +# module's position in that template's `modules:` list. `routingConfig` is the +# path (inside the container) of that module's routing file; compose mounts: +# consumer unified-network-layer/single-network-layer-consumer-routing.yaml +# network routing-network.yaml (the multi-adapter file, unchanged) +# provider-publish unified-network-layer/single-network-layer-publish-routing.yaml +# The provider module has no router: it answers requests itself. modules: - from: consumer index: 0 name: consumer path: /consumer/ + routingConfig: /app/config/single-network-layer-consumer-routing.yaml - from: network index: 0 name: network path: /network/ + routingConfig: /app/config/routing-network.yaml - from: provider index: 0 name: provider @@ -44,3 +48,4 @@ modules: index: 1 name: provider-publish path: /provider/publish + routingConfig: /app/config/single-network-layer-publish-routing.yaml diff --git a/quick-start/config/adapters/unified-network-layer/routing-http.yaml b/quick-start/config/adapters/unified-network-layer/routing-http.yaml deleted file mode 100644 index af047b51..00000000 --- a/quick-start/config/adapters/unified-network-layer/routing-http.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Routing for the unified network layer: tier hops over HTTP to this same -# container (http://localhost), instead of in-process. Works with any adapter -# image. Selected with HTTP=1 (make up-unified-network-layer HTTP=1), which adds -# the overlay docker-compose.unified-network-layer-http.yml. -# -# localhost:9200 is the container's own listener -- it must match `port` in -# overrides.yaml. Discovery is a separate container either way. -routingRules: - # consumer discover, and publish arriving at /network/publish - - version: "2.0.0" - targetType: "url" - target: - url: "http://discovery-service:8080" - endpoints: - - discover - - publish - - # consumer -> provider tier, over loopback HTTP - - version: "2.0.0" - targetType: "url" - target: - url: "http://localhost:9200/provider" - endpoints: - - select - - init - - confirm - - status - - # provider publish module (/provider/publish, endpoint "") -> network publish, over loopback HTTP - - version: "2.0.0" - targetType: "url" - target: - url: "http://localhost:9200/network/publish" - excludeAction: true - endpoints: - - "" diff --git a/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml b/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml deleted file mode 100644 index d3ff6cc0..00000000 --- a/quick-start/config/adapters/unified-network-layer/routing-inproc.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Routing for the unified network layer: tier hops IN-PROCESS (the default). -# -# One file for every module: the router keys rules on (version, endpoint) only, -# and no endpoint needs two targets -- consumer discover goes straight to -# discovery, and the network module only receives publish. -# -# inproc:// hands the request to that module of this same process as a -# function call: no socket, no port, no TCP. Needs an adapter image built with -# in-process routing (network-adapter feat/55-unified-adapter-inproc-routing). -# For any other image use HTTP=1 (routing-http.yaml). -routingRules: - # consumer discover, and publish arriving at /network/publish - - version: "2.0.0" - targetType: "url" - target: - url: "http://discovery-service:8080" - endpoints: - - discover - - publish - - # consumer -> provider tier - - version: "2.0.0" - targetType: "url" - target: - url: "inproc://provider" - endpoints: - - select - - init - - confirm - - status - - # provider publish module (/provider/publish, endpoint "") -> network publish - - version: "2.0.0" - targetType: "url" - target: - url: "inproc://network/publish" - excludeAction: true - endpoints: - - "" diff --git a/quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml b/quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml new file mode 100644 index 00000000..662f73dd --- /dev/null +++ b/quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml @@ -0,0 +1,35 @@ +# What the consumer layer does with each action, in the unified network layer +# (the unified single adapter: consumer, network and provider as modules of one +# adapter). +# +# Both targets are THIS SAME adapter on another path. In the multi-adapter +# stack they are two other containers; here the hop is a loopback call from one +# module to another. It stays a real HTTP hop rather than an in-process call +# because that keeps the two shapes the same config: split these across +# containers and only the host in these URLs changes. +# +# targetType "url" appends the action to the path, so the base here is the +# module mount point and /discover or /select is added to it. localhost:9200 is +# the container's own listener and must match `port` in overrides.yaml. +routingRules: + # Discovery is answered from this network's own discovery service, which the + # network module fronts. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network" + endpoints: + - discover + + # Everything transactional goes to the provider layer, and this layer decides + # nothing about where it ends up. context.bppUri travels onward in the body + # untouched. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/provider" + endpoints: + - select + - init + - confirm + - status diff --git a/quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml b/quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml new file mode 100644 index 00000000..fd566ec0 --- /dev/null +++ b/quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml @@ -0,0 +1,18 @@ +# This network's catalogue system publishing inward, in the unified network +# layer (the unified single adapter). +# +# It goes to the NETWORK module rather than straight to discovery, so a publish +# takes the same path a publish from anywhere else takes -- one place that +# decides what reaches discovery, rather than two. +# +# The publish module is mounted at the exact path /provider/publish, so the +# endpoint left after stripping it is "" -- which is what this rule keys on, +# with excludeAction so nothing is appended to the target. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network/publish" + excludeAction: true + endpoints: + - "" diff --git a/quick-start/docker-compose.unified-network-layer-http.yml b/quick-start/docker-compose.unified-network-layer-http.yml deleted file mode 100644 index 3df651b7..00000000 --- a/quick-start/docker-compose.unified-network-layer-http.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Overlay on docker-compose.unified-network-layer.yml: tier-to-tier hops over -# HTTP to the container's own listener (http://localhost:9200/...) instead of -# in-process (inproc://). Works with any adapter image. -# -# make up-unified-network-layer HTTP=1 -# docker compose -f docker-compose.unified-network-layer.yml \ -# -f docker-compose.unified-network-layer-http.yml up -d -services: - network-layer-adapter: - volumes: - - ./config/adapters/unified-network-layer/routing-http.yaml:/app/config/routing-unified-network-layer.yaml:ro diff --git a/quick-start/docker-compose.unified-network-layer.yml b/quick-start/docker-compose.unified-network-layer.yml index 08a5eeb1..015c4f9f 100644 --- a/quick-start/docker-compose.unified-network-layer.yml +++ b/quick-start/docker-compose.unified-network-layer.yml @@ -11,12 +11,10 @@ # make up-unified-network-layer (or: bin/unified-network-layer-up.sh) # docker compose -f docker-compose.unified-network-layer.yml up -d # -# Tier-to-tier hops are IN-PROCESS by default (inproc://, routing-inproc.yaml), -# which needs an adapter image built with in-process routing -- see -# UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE in .env (bin/unified-network-layer-up.sh -# builds it). The overlay docker-compose.unified-network-layer-http.yml -# (make up-unified-network-layer HTTP=1) sends them over http://localhost -# instead, with any image. +# Tier-to-tier hops go over loopback HTTP to the container's own listener +# (http://localhost:9200): they never leave the container. One routing file per +# module, as in the multi-adapter stack -- only the hosts differ. Any adapter +# image works. # # The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among # them authenticates nothing but that registry, and shares its postgres. @@ -605,11 +603,12 @@ services: # modules of ONE adapter process (the unified single adapter), on ONE config # built by bin/setup.py from the three tier templates # (config/adapters/unified-network-layer/unified-network-layer.yaml), with ONE - # routing file. Tier-to-tier hops are inproc:// -- a function call inside the - # process -- so there is no consumer-adapter, network-adapter or + # routing file. Tier-to-tier hops go over loopback (http://localhost:9200), + # inside the container, so there is no consumer-adapter, network-adapter or # provider-adapter here. # - # :9200 inside, ${UNIFIED_NETWORK_LAYER_ADAPTER_PORT} on 127.0.0.1 + # :9200 inside and on 127.0.0.1 (ADAPTER_IMAGE, same as the multi-adapter stack). + # The multi-adapter provider-adapter uses 9200 too: run one stack at a time. # /consumer/* the caller's entry (bap), takes unsigned requests # /network/* hands publish on to discovery # /provider/* calls the upstream, answers synchronously @@ -620,9 +619,6 @@ services: network-layer-adapter: <<: *adapter container_name: network-layer-adapter - # inproc:// needs an image built with in-process routing; empty falls back - # to ADAPTER_IMAGE, which then needs HTTP=1. - image: ${UNIFIED_NETWORK_LAYER_ADAPTER_IMAGE:-${ADAPTER_IMAGE}} depends_on: sunbird-registry-service: condition: service_healthy @@ -647,10 +643,13 @@ services: VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} volumes: - ./config/adapters/unified-network-layer/unified-network-layer.yaml:/app/config/adapter.yaml:ro - # Default: tier hops in-process. HTTP=1 (the -http overlay) mounts routing-http.yaml here instead. - - ./config/adapters/unified-network-layer/routing-inproc.yaml:/app/config/routing-unified-network-layer.yaml:ro + # One routing file per module, as in the multi-adapter stack. The network + # module reuses the multi-adapter routing-network.yaml unchanged. + - ./config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml:/app/config/single-network-layer-consumer-routing.yaml:ro + - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro + - ./config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml:/app/config/single-network-layer-publish-routing.yaml:ro ports: - - "127.0.0.1:${UNIFIED_NETWORK_LAYER_ADAPTER_PORT:-9210}:9200" + - "127.0.0.1:9200:9200" # ========================================================================== # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI From a31dcc20a6aef8177a728248817af9d32d611f8e Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 13:32:53 +0530 Subject: [PATCH 04/43] fix(quick-start): address review findings on the unified network layer - unified-network-layer-up.sh: self-contained publish/discover/select smoke test instead of the untracked poc-flow.py; stop hiding setup errors; confirm the rendered config exists before starting - setup.py: missing PyYAML is fatal when rendering the unified network layer (UNIFIED_NETWORK_LAYER=1), still a skip for multi mode - stack.sh: PyYAML preflight and a rendered-config file check before starting network-layer-adapter, so compose never creates a directory at the bind-mount path - bootstrap-ubuntu.sh and README: install/list PyYAML; drop untracked tools from the README - docker-compose.unified-network-layer.yml: -f on profile commands; correct stale comments (routing files, adapter count, mocks) It covers the 6 changed files: README.md, bin/bootstrap-ubuntu.sh, bin/setup.py, bin/stack.sh, bin/unified-network-layer-up.sh and docker-compose.unified-network-layer.yml. --- quick-start/README.md | 9 ++-- quick-start/bin/bootstrap-ubuntu.sh | 4 +- quick-start/bin/setup.py | 6 ++- quick-start/bin/stack.sh | 18 +++++-- quick-start/bin/unified-network-layer-up.sh | 47 +++++++++++++++++-- .../docker-compose.unified-network-layer.yml | 31 ++++++------ 6 files changed, 83 insertions(+), 32 deletions(-) diff --git a/quick-start/README.md b/quick-start/README.md index af55c2b9..6d276142 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -55,6 +55,7 @@ below follow startup order, so they work top to bottom. - `git` - Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` - `python3` with `cryptography` — `pip install cryptography` +- `PyYAML` — `pip install pyyaml` (needed for the unified network layer only) - `curl` ## Step 2 — Get the repo @@ -223,11 +224,7 @@ multi-adapter stack only the hosts differ. 9200 is the container's own listener and must match `port` in `overrides.yaml`. Tools: -- `bin/unified-network-layer-up.sh`: start and smoke-test. -- `bin/demo-unified-network-layer.sh`: step-by-step live demo. -- `bin/bench-unified-network-layer.sh`: multi vs unified network layer, parallel curl, JSON results under `.bench/`. -- `bin/poc-flow.py --mode multi|unified-network-layer`: quick publish/discover/select check. -- Postman: `api-collection/OpenAgriNet.unified-network-layer.*.json`. +- `bin/unified-network-layer-up.sh`: start and smoke-test (publish, discover, select). # Part 2 — Run it on a VM @@ -241,7 +238,7 @@ Nothing is cloned yet, so fetch the script rather than running it from the repo: curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash ``` -Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt +Installs `git`, `make`, `python3-cryptography`, `python3-yaml` and Docker from Docker's own apt repo, then adds you to the `docker` group — **log out and back in** for that to take effect. Idempotent, and it deliberately does not clone, write `.env` or start anything. diff --git a/quick-start/bin/bootstrap-ubuntu.sh b/quick-start/bin/bootstrap-ubuntu.sh index 519c6bf0..37afcca4 100755 --- a/quick-start/bin/bootstrap-ubuntu.sh +++ b/quick-start/bin/bootstrap-ubuntu.sh @@ -43,8 +43,8 @@ sudo apt-get update -qq # python as externally-managed (PEP 668), so `pip install cryptography` refuses # without --break-system-packages. The apt build is the same library. sudo apt-get install -y -qq \ - ca-certificates curl gnupg git make python3 python3-cryptography -info "git, make, python3, python3-cryptography" + ca-certificates curl gnupg git make python3 python3-cryptography python3-yaml +info "git, make, python3, python3-cryptography, python3-yaml" # --------------------------------------------------------------- docker diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 25fe1639..11c3ae8b 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -656,8 +656,12 @@ def _render_unified_network_layer(identities, shared): try: import yaml # only this mode needs it; the multi-adapter stack does not except ImportError: + hint = "needs PyYAML (pip install pyyaml, or apt install python3-yaml)" + if os.environ.get("UNIFIED_NETWORK_LAYER") == "1": + sys.exit("setup: cannot render config/adapters/unified-network-layer/" + "unified-network-layer.yaml: " + hint) print(" config/adapters/unified-network-layer/unified-network-layer.yaml" - " SKIPPED: needs PyYAML (pip install pyyaml, or apt install python3-yaml)") + " SKIPPED: " + hint) return overrides = yaml.safe_load((UNIFIED_NETWORK_LAYER / "overrides.yaml").read_text()) tiers = {} diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 7ef6121e..613749e6 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -64,6 +64,13 @@ preflight() { || die "the python 'cryptography' package is missing -- pip install cryptography" } +# The unified network layer renders its config with PyYAML (multi mode does not). +preflight_unified_network_layer() { + preflight + python3 -c 'import yaml' >/dev/null 2>&1 \ + || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" +} + # ------------------------------------------------------------------- up # The full stack, in the order the compose file's own header documents. Five @@ -172,7 +179,9 @@ NEXT # The unified network layer: ONE adapter runs the consumer, network and # provider tiers as modules of one process (the unified single adapter), with # tier-to-tier hops over loopback HTTP inside the container. The multi-adapter -# containers are NOT started. Mocks are not started either, as in `up`. +# containers are NOT started. `make up-unified-network-layer` does not start the +# mocks either, as in `up`; bin/unified-network-layer-up.sh does start them, for +# its local smoke test. # # Its stack is its own complete compose file (docker-compose.unified-network-layer.yml), # so docker-compose.yml is not touched. Passing -f overrides COMPOSE_FILE, so @@ -184,10 +193,13 @@ unified_network_layer_compose() { } up_unified_network_layer() { - preflight + preflight_unified_network_layer step 1 3 "registry and discovery (unified network layer compose file)" unified_network_layer_compose up -d sunbird-registry-service discovery-service - up_setup 3 + UNIFIED_NETWORK_LAYER=1 up_setup 3 + # Never let compose create a directory where the bind-mounted config file belongs. + [ -f config/adapters/unified-network-layer/unified-network-layer.yaml ] \ + || die "config/adapters/unified-network-layer/unified-network-layer.yaml was not rendered (is it a directory? rm -r it, then re-run)" step 3 3 "network-layer-adapter (consumer + network + provider in one process)" # The multi-adapter containers share this project; one topology at a time. docker compose stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true diff --git a/quick-start/bin/unified-network-layer-up.sh b/quick-start/bin/unified-network-layer-up.sh index 4446129f..719eeba8 100755 --- a/quick-start/bin/unified-network-layer-up.sh +++ b/quick-start/bin/unified-network-layer-up.sh @@ -25,15 +25,22 @@ die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } [ -f .env ] || die ".env missing: cp .env.example .env and fill it in" [ -n "$IMAGE" ] || die "no image: set ADAPTER_IMAGE in .env" +command -v python3 >/dev/null 2>&1 || die "python3 is not installed -- bin/setup.py needs it" +python3 -c 'import cryptography' >/dev/null 2>&1 \ + || die "the python 'cryptography' package is missing -- pip install cryptography" +python3 -c 'import yaml' >/dev/null 2>&1 \ + || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" # --- 1. infrastructure ------------------------------------------------------ -step "1/4 registry, discovery, mocks" +step "1/4 registry, discovery, mocks (the mocks are for this script's local smoke test; make up-unified-network-layer does not start them)" docker compose "${COMPOSE[@]}" up -d sunbird-registry-service discovery-service mock-imd mock-agmarknet # --- 2. config -------------------------------------------------------------- step "2/4 keys, registry entries, rendered config (bin/setup.py)" -python3 bin/setup.py >/dev/null -echo " config/adapters/unified-network-layer/unified-network-layer.yaml rendered" +UNIFIED_NETWORK_LAYER=1 python3 bin/setup.py >/dev/null +CFG=config/adapters/unified-network-layer/unified-network-layer.yaml +[ -f "$CFG" ] || die "$CFG was not rendered (a directory there? rm -r it, then re-run)" +echo " $CFG rendered" # --- 3. adapter ------------------------------------------------------------- step "3/4 network-layer-adapter on $IMAGE (loopback tier hops)" @@ -50,10 +57,40 @@ echo " health: $(curl -s "http://127.0.0.1:$PORT/health")" # --- 4. smoke test ---------------------------------------------------------- step "4/4 publish, discover, select through the unified network layer" -python3 bin/poc-flow.py --mode unified-network-layer --n 3 +ROOT="$ROOT" PORT="$PORT" python3 - <<'PY' +import json, os, pathlib, sys, urllib.error, urllib.request, uuid +root = pathlib.Path(os.environ["ROOT"]); port = os.environ["PORT"] +env = {} +for line in (root / ".env").read_text().splitlines(): + if "=" in line and not line.lstrip().startswith("#"): + k, v = line.split("=", 1); env[k.strip()] = v.strip() +steps = (("publish", "provider/publish"), ("discover", "consumer/discover"), ("select", "consumer/select")) +rc = 0 +for step, path in steps: + f = next((root / "capability-examples" / "WeatherObservation" / step).glob("*.json")) + text = f.read_text() + for old, key in (("provider.oan.dev", "PROVIDER_SUBSCRIBER_ID"), + ("consumer.oan.dev", "CONSUMER_SUBSCRIBER_ID"), + ("network.oan.dev", "NETWORK_SUBSCRIBER_ID")): + text = text.replace(old, env[key]) + text = text.replace('"mausamgram"', json.dumps(env["MAUSAMGRAM_PARTICIPANT_ID"])) + body = json.loads(text) + body["context"]["transactionId"], body["context"]["messageId"] = str(uuid.uuid4()), str(uuid.uuid4()) + req = urllib.request.Request(f"http://127.0.0.1:{port}/{path}", json.dumps(body).encode(), + {"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=20) as r: + status, data = r.status, r.read() + except urllib.error.HTTPError as e: + status, data = e.code, e.read() + print(f" {step:9} /{path:18} HTTP {status}") + if status != 200: + rc = 1 + print(" body:", data[:300].decode(errors="replace")) +sys.exit(rc) +PY echo echo " tier hops seen in the adapter log:" docker logs --since 2m network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' printf '\n\033[1;32mUnified network layer running on %s. Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" -echo "Demo: bin/demo-unified-network-layer.sh Postman: api-collection/OpenAgriNet.unified-network-layer.collection.json" diff --git a/quick-start/docker-compose.unified-network-layer.yml b/quick-start/docker-compose.unified-network-layer.yml index 015c4f9f..0e45c9a9 100644 --- a/quick-start/docker-compose.unified-network-layer.yml +++ b/quick-start/docker-compose.unified-network-layer.yml @@ -40,7 +40,7 @@ # 3. python3 bin/setup.py keys, the three tier identities, and # config/adapters/unified-network-layer/unified-network-layer.yaml # 4. docker compose -f docker-compose.unified-network-layer.yml up -d now the adapter -# 5. docker compose --profile reverse-proxy --profile observability up -d +# 5. docker compose -f docker-compose.unified-network-layer.yml --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack # # Naming registry and discovery in step 2 is not tidiness. An adapter config @@ -79,11 +79,11 @@ # decoration and becomes the actual control. NPM's "Forward Hostname" is a # free-text field, so anyone with the admin password can point a public route # at `sunbird-registry-service` or `sunbird-registry-keycloak`. NPM sits only on -# oan-edge, where neither name resolves and neither address is routable -- the three adapters and hyperdx -# are the only containers it shares a network with. Those four straddle both +# oan-edge, where neither name resolves and neither address is routable -- network-layer-adapter and hyperdx +# are the only containers it shares a network with. Those three straddle both # networks; everything else is internal-only. # -# Neither is `internal: true`: the adapters and discovery fetch the Beckn spec +# Neither is `internal: true`: the adapter and discovery fetch the Beckn spec # from raw.githubusercontent.com at boot, the provider adapter calls an # upstream API that lives outside this VM, and NPM has to reach Let's Encrypt. # Cutting egress would break all three. @@ -137,9 +137,9 @@ x-adapter: &adapter image: ${ADAPTER_IMAGE} pull_policy: missing restart: unless-stopped - # The adapters are the only services on oan-edge, and therefore the only - # ones NPM can reach. They still need oan-internal to read the registry and - # to call discovery, so they sit on both. + # The adapter is the only service on oan-edge besides hyperdx, and therefore + # one of the only ones NPM can reach. It still needs oan-internal to read the + # registry and to call discovery, so it sits on both. networks: [oan-internal, oan-edge] logging: *logging @@ -546,10 +546,11 @@ services: condition: service_healthy # ========================================================================== - # adapters -- consumer, network and provider. Same image, three configs. + # adapter -- network-layer-adapter: the consumer, network and provider tiers as + # modules of one process, one image, one config. # - # Their configs are bind-mounted FILES that do not exist until bin/setup.py - # renders them from the .tmpl beside them, which is what the step ordering + # Its config is a bind-mounted FILE that does not exist until bin/setup.py + # renders it from the tier templates beside it, which is what the step ordering # above is about. # ========================================================================== @@ -602,8 +603,8 @@ services: # The unified network layer: the consumer, network and provider tiers as # modules of ONE adapter process (the unified single adapter), on ONE config # built by bin/setup.py from the three tier templates - # (config/adapters/unified-network-layer/unified-network-layer.yaml), with ONE - # routing file. Tier-to-tier hops go over loopback (http://localhost:9200), + # (config/adapters/unified-network-layer/unified-network-layer.yaml), with one + # routing file per module. Tier-to-tier hops go over loopback (http://localhost:9200), # inside the container, so there is no consumer-adapter, network-adapter or # provider-adapter here. # @@ -655,7 +656,7 @@ services: # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI # over it. # - # docker compose --profile observability up -d + # docker compose -f docker-compose.unified-network-layer.yml --profile observability up -d # # Sizing: ClickHouse alone wants 2-4 GB. With this profile on, the VM needs # 16 GB (t3.xlarge or m6i.xlarge); without it, 8 GB is workable. @@ -821,7 +822,7 @@ services: # ========================================================================== # edge -- Nginx Proxy Manager. # - # docker compose --profile reverse-proxy up -d + # docker compose -f docker-compose.unified-network-layer.yml --profile reverse-proxy up -d # # NPM rather than a hand-written nginx.conf because it owns the part that is # genuinely tedious to do by hand -- ACME. It requests, installs and renews @@ -844,7 +845,7 @@ services: # NPM's "Forward Hostname" is a free-text field. Anyone with the admin # password can type `sunbird-registry-service`, `sunbird-registry-keycloak` or `discovery-postgres` into it. On # this network none of those names resolve and none of those addresses are - # routable from here -- the three adapters, and hyperdx when the + # routable from here -- network-layer-adapter, and hyperdx when the # observability profile is up, are the only containers NPM shares a network # with. So the blast radius of a wrong click, or of the admin UI being # reached by someone who should not have it, is bounded to that set. From 7f73f0ee2a5613eddf73339e420922e98179e914 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 15:37:29 +0530 Subject: [PATCH 05/43] feat(adapter-service): add network-layer role (single network adapter) alongside the multi-adapter roles role: network-layer runs the consumer, network and provider tiers as modules of one adapter process, removing the cross-pod hops between tiers. The existing consumer/network/provider roles are unchanged. Each environment picks a mode with adapters.singleNetworkAdapter: true|false (README "Choosing a mode"). adapter-service: - _network-layer.tpl: generate adapter.yaml and routing at render time from config/-config.yaml and config/routing-.yaml (one source of truth for both modes); tier hops rewritten to loopback http://localhost:/; per-tier ___*__ placeholders - configmap: network-layer branch, private-key placeholder check on the parsed config, `config` override skips the generator - deployment: per-tier key Secrets (keys-, 0400), init substitution per tier, networkLayer.extraSubstitutions; optional terminationGracePeriodSeconds, topologySpreadConstraints, preStop sleep, GOMEMLIMIT, registry-cache Redis password - ingress: network-layer allows only /consumer... and /provider/ (never /network or /provider/publish), Prefix/Exact pathType only, no regex paths or nginx use-regex/rewrite-target - generated config honours logLevel, http.timeout.*, otel.* and otel.serviceName - hpa: autoscaling.behavior passthrough - values: networkLayer block (keys, httpClient pool, registryCache); probes.* replaced by startupProbe/livenessProbe/readinessProbe (breaking) - ci/network-layer-values.yaml, examples/network-layer.yaml, NOTES, README openbao-secrets: - reflect consumer/network/provider adapter keys and provider-upstream into the network-layer-adapter namespace --- charts/adapter-service/CHANGELOG.md | 37 ++++ charts/adapter-service/Chart.yaml | 10 +- charts/adapter-service/README.md | 68 +++++- .../ci/network-layer-values.yaml | 67 ++++++ .../examples/network-layer.yaml | 126 +++++++++++ charts/adapter-service/templates/NOTES.txt | 33 ++- charts/adapter-service/templates/_helpers.tpl | 11 +- .../templates/_network-layer.tpl | 205 ++++++++++++++++++ .../adapter-service/templates/configmap.yaml | 35 +++ .../adapter-service/templates/deployment.yaml | 87 ++++++++ charts/adapter-service/templates/hpa.yaml | 4 + charts/adapter-service/templates/ingress.yaml | 34 +++ charts/adapter-service/values.yaml | 138 ++++++++++-- charts/openbao-secrets/CHANGELOG.md | 8 + charts/openbao-secrets/values.yaml | 12 +- 15 files changed, 834 insertions(+), 41 deletions(-) create mode 100644 charts/adapter-service/ci/network-layer-values.yaml create mode 100644 charts/adapter-service/examples/network-layer.yaml create mode 100644 charts/adapter-service/templates/_network-layer.tpl diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index b8e36b07..2969ce91 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -5,6 +5,43 @@ All notable changes to this chart are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.3] - 2026-10-09 + +### Added +- `role: network-layer`: all three tiers in one adapter (four modules: + `/consumer/`, `/network/`, `/provider/`, `/provider/publish`), as in the + quick-start's network-layer compose stack. Its config and routing are generated + at render time from `config/*.yaml`, so there is no second copy of any module. + Tier hops are loopback HTTP inside the pod. Signs as three identities from + `networkLayer.keys.{consumer,network,provider}`; `routing.config` fails the + render for this role. +- `networkLayer.httpClient` (connection pool on every module; `null` turns it + off) and `networkLayer.registryCache` (optional Redis registry cache), + network-layer only. Ingress may expose only `/consumer...` and + `/provider/`, as literal paths with `pathType` Prefix or Exact and + no `use-regex` / `rewrite-target` annotation. The generated config takes + `logLevel`, `http.timeout.*` and `otel.*` from values. Example + `examples/network-layer.yaml`. +- `autoscaling.behavior`, `topologySpreadConstraints`, + `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off + by default. + +### Fixed +- Liveness and readiness probes now render. Values used `probes.liveness` / + `probes.readiness`, which `common.probes` never read, so no adapter pod had a + readiness probe. Upgrading rolls every adapter pod once. + +### Removed +- **Breaking:** the `probes.liveness` and `probes.readiness` values keys, renamed to + `livenessProbe` / `readinessProbe` (and a new `startupProbe`, off). The + timings are unchanged. An environment that overrides `probes.*` must rename + those keys. + +### Unchanged +- Roles `provider`, `network`, `consumer`: same config, routing and manifests + apart from the probes (the chart version label and the config checksum + annotation also change with the version). + ## [0.5.2] - 2026-09-29 ### Changed diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index e3c0eb0d..fabc308f 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -1,13 +1,14 @@ apiVersion: v2 name: adapter-service description: >- - One chart for the OAN Beckn adapters. provider, network and consumer are the same - image and config format, so `role` selects the handler role, the step list - and the routing target; install once per role. Stateless: an adapter's + One chart for the OAN Beckn adapters: provider, network and consumer (one + release per tier), or network-layer (all three tiers in one adapter). Same + image and config format; role selects the handler role, the step list and + the routing target. Stateless: an adapter's identity is a keypair held in a Secret, and everything else it reads from the registry. type: application -version: 0.5.2 +version: 0.5.3 # The adapter binary, not this chart. `/health` reports the build it is # actually running, which is the number to trust over this one. appVersion: "v1.9.0" @@ -22,6 +23,7 @@ keywords: - provider - network - consumer + - network-layer home: https://github.com/OpenAgriNet/helmcharts sources: - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index 7bd5cc81..25b3e4ae 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -1,8 +1,10 @@ # adapter-service -One chart for all three Beckn adapters. `provider`, `network` and `consumer` -are the same image and the same config format, so `role` selects the handler -role, the step list and where requests are routed. +One chart for the Beckn adapters, in two modes. Multi-adapter: three roles, +`provider`, `network` and `consumer`, one release per tier. Network layer: +`network-layer`, all three tiers in one adapter. All roles are the same image +and the same config format, so `role` selects the handler role, the step list +and where requests are routed. ``` consumer ──unsigned──▶ network ──▶ discovery @@ -12,11 +14,19 @@ role, the step list and where requests are routed. └──▶ mandi / agmarknet upstreams ``` +``` + network-layer (one adapter, one pod): + /consumer/ ──lo──▶ /network/ ──▶ discovery + └──────lo────▶ /provider/ ──▶ upstream APIs + /provider/publish ──lo──▶ /network/publish +``` + | Role | Handler | Steps | Routes to | |---|---|---|---| | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | | `network` | `bpp` | `validateSign → addRoute → sign` | discovery | | `consumer` | `bap` | `addRoute → sign` | network | +| `network-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | `consumer` is the only one that accepts **unsigned** requests: the consumer app is inside the trust boundary, so there is no network signature to check. @@ -58,7 +68,7 @@ render error rather than a default: | Value | Why there is no default | |---|---| | `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | -| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs | +| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `network-layer` uses `networkLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | | `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | | `role` | It decides the handler role, the step list and the routing target. A default would give one adapter another one's behaviour | | `routing.rules` | An adapter with no route accepts requests and has nowhere to send them | @@ -121,12 +131,56 @@ the **public** half of the keypair. Missing, or carrying a different key, and every peer rejects what this adapter signs while the pod stays perfectly healthy. +## Choosing a mode: multi-adapter or network layer + +| Mode | Releases (release = namespace) | Callers use | +|---|---|---| +| multi | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | +| network layer | `network-layer-adapter` (examples/network-layer.yaml) | `network-layer-adapter:9200/consumer/`, `network-layer-adapter:9200/provider/publish` | + +`role: network-layer` runs all three tiers in one adapter, as four +modules of one process. Its config and routing are +**generated** from `config/*.yaml` at render time, so a change to a tier file +reaches both modes. Tier hops are loopback HTTP inside the pod (any image). It +signs as the same three identities, from the same three Secrets, so the +registry is the same in both modes. An environment's `config:` override for a +multi role is NOT seen by network-layer; give the network-layer release its own +`config:` if needed (it replaces the generated config whole). + +Pick the mode per environment with ONE switch where releases are chosen. For a +helmfile (adapt to infra-automation's layout): + +```yaml +# environments: values carry adapters: {singleNetworkAdapter: false} +releases: +{{- range $tier := list "consumer" "network" "provider" }} + - name: {{ $tier }}-adapter + namespace: {{ $tier }}-adapter + chart: ../helmcharts/charts/adapter-service + installed: {{ not $.Values.adapters.singleNetworkAdapter }} + values: [values/{{ $tier }}-adapter.yaml] +{{- end }} + - name: network-layer-adapter + namespace: network-layer-adapter + chart: ../helmcharts/charts/adapter-service + installed: {{ .Values.adapters.singleNetworkAdapter }} + values: [values/network-layer-adapter.yaml] +``` + +With Argo CD, gate the four Applications on the same boolean (for example an +ApplicationSet list generator, or `{{ if }}` in an app-of-apps chart). + +Switching an environment: install `network-layer-adapter` alongside the multi +releases (both modes sign with the same identities), repoint the app and the +catalogue publisher to the paths above, then remove the three multi releases. +Reverse the steps to switch back. + ## Values See `values.yaml` — every field is commented with what it does and what breaks -without it. `examples/{provider,network,consumer}.yaml` are working dev deployments; -`ci/` holds the two files lint renders, one minimal and one with every switch -turned on. +without it. `examples/{provider,network,consumer}.yaml` are working dev deployments for the +multi-adapter mode, and `examples/network-layer.yaml` for the network layer; +`ci/` holds the files lint renders, including `ci/network-layer-values.yaml`. ## Related diff --git a/charts/adapter-service/ci/network-layer-values.yaml b/charts/adapter-service/ci/network-layer-values.yaml new file mode 100644 index 00000000..572c2932 --- /dev/null +++ b/charts/adapter-service/ci/network-layer-values.yaml @@ -0,0 +1,67 @@ +# charts/adapter-service/ci/network-layer-values.yaml +# role network-layer: the unified single adapter (consumer, network and +# provider tiers as modules of one process). Renders the generated config, the +# loopback routing and the three tier identities. +role: network-layer +fullnameOverride: network-layer-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + +registry: + url: http://registry:8081/api/v1 + +service: + port: 9200 + targetPort: 9200 + +networkLayer: + keys: + consumer: + existingSecret: consumer-adapter-keys + network: + existingSecret: network-adapter-keys + provider: + existingSecret: provider-adapter-keys + extraSubstitutions: + __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + +# Scale defaults for the unified single adapter: roughly the three tiers' +# requests in one pod, at least one replica per node. +resources: + requests: + cpu: 300m + memory: 384Mi + limits: + cpu: 1500m + memory: 1Gi +goMemLimit: 900MiB +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Pods + value: 1 + periodSeconds: 60 +podDisruptionBudget: + enabled: true + minAvailable: 1 +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway +terminationGracePeriodSeconds: 40 +preStopSleepSeconds: 10 diff --git a/charts/adapter-service/examples/network-layer.yaml b/charts/adapter-service/examples/network-layer.yaml new file mode 100644 index 00000000..497e150a --- /dev/null +++ b/charts/adapter-service/examples/network-layer.yaml @@ -0,0 +1,126 @@ +# =========================================================================== +# network-layer role -- the unified single adapter. +# +# The consumer, network and provider tiers as four modules of ONE adapter +# process, on one Service: +# +# /consumer/ the caller (bap); the app posts here, unsigned +# /network/ internal: verifies and forwards to discovery +# /provider/ answers from the upstream APIs (bpp) +# /provider/publish the catalogue system publishes here +# +# The alternative to installing the three multi-adapter releases +# (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the +# other in an environment, not both. +# +# helm install network-layer-adapter . -n network-layer-adapter --create-namespace \ +# -f examples/network-layer.yaml +# +# The config and routing are generated from the three tier files in config/; +# tier hops are loopback HTTP inside the pod, so any adapter image works. +# =========================================================================== +role: network-layer + +# The Service name and DNS name callers use: +# network-layer-adapter.network-layer-adapter.svc.cluster.local:9200 +fullnameOverride: network-layer-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "latest" + +registry: + url: http://registry:8081/api/v1 + +# The same port the quick-start publishes. It is also the loopback port the +# generated routing uses, so changing targetPort moves both. +service: + port: 9200 + targetPort: 9200 + +networkLayer: + # The three tier Secrets the multi-adapter releases use. openbao-secrets + # reflects them into this namespace. + keys: + consumer: + existingSecret: consumer-adapter-keys + network: + existingSecret: network-adapter-keys + provider: + existingSecret: provider-adapter-keys + # The provider tier's token endpoint, from the provider Secret. Without it + # the init container stops on the leftover placeholder. + extraSubstitutions: + __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + +# The provider tier reads its upstream credentials (MAUSAMGRAM_AUTH, +# AGMARKNET_*, VISTAAR_CLIENT_*) from the environment. +envFromSecrets: + - provider-upstream + +resources: + requests: + cpu: 300m + memory: 384Mi + limits: + cpu: 1500m + memory: 1Gi +goMemLimit: 900MiB + +autoscaling: + enabled: true + minReplicas: 3 # one per node with the spread constraint below + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Pods + value: 1 + periodSeconds: 60 + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + +terminationGracePeriodSeconds: 40 +preStopSleepSeconds: 10 # needs Kubernetes 1.30+; 0 on older clusters + +# Off here. When on, only these paths are accepted: /, /provider, +# /provider/publish and /network* fail the render. +ingress: + enabled: false + className: "" + hosts: + - host: network-layer.oan.example.com + paths: + - path: /consumer + pathType: Prefix + - path: /provider/select + pathType: Prefix + - path: /provider/init + pathType: Prefix + - path: /provider/confirm + pathType: Prefix + - path: /provider/status + pathType: Prefix + +waitFor: + enabled: false + +otel: + enabled: false + endpoint: "" diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index 9a0a7db3..feddc34d 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,11 +1,15 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} - +{{ if include "adapter-service.isNetworkLayer" . }} + role network-layer (all three tiers in one adapter: /consumer/ bap, /network/ bpp, /provider/ bpp, /provider/publish bap) + hops loopback HTTP between tiers (http://localhost:{{ .Values.service.targetPort }}/) +{{- else }} role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) steps {{ include "adapter-service.steps" . | fromYamlArray | join " -> " }} +{{- end }} image {{ include "adapter-service.image" . }} service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} registry {{ .Values.registry.url }} - routes from config/routing-{{ include "adapter-service.role" . }}.yaml + routes {{ if include "adapter-service.isNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. @@ -22,6 +26,21 @@ chart's appVersion. TWO THINGS THIS CHART CANNOT CHECK FOR YOU +{{ if include "adapter-service.isNetworkLayer" . -}} +1. The Participant rows. This adapter signs as three identities -- consumer, + network and provider -- from Secrets {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, + {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the + multi-adapter releases use. Each must carry its tier's public key. + +2. Key rotation. Changing any of those Secrets restarts nothing: + + kubectl -n {{ include "common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} + +Callers address the tiers by path on this one Service: + app (discover, select, ...) {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/consumer/ + catalogue publisher {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/provider/publish + /network is an internal tier: never route to it from outside. +{{- else -}} 1. The Participant row. This adapter signs as {{ .Values.keys.existingSecret.name }}'s subscriberId, and every peer verifies that signature against the public key the registry publishes for it. If the row is missing, or carries a different @@ -33,6 +52,7 @@ TWO THINGS THIS CHART CANNOT CHECK FOR YOU checksum. Changing that Secret does not restart anything: kubectl -n {{ include "common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} +{{- end }} {{- if not .Values.otel.enabled }} @@ -42,7 +62,14 @@ on and nothing listening the exporter logs a failure every interval. {{- if .Values.ingress.enabled }} Ingress is ON. -{{- if eq (include "adapter-service.role" .) "consumer" }} +{{- if include "adapter-service.isNetworkLayer" . }} +This is the network-layer role. /consumer accepts UNSIGNED requests: the consumer tier is +inside the trust boundary and has no validateSign step, so an Ingress on it exposes an +unauthenticated entry point to the network. Rate-limit this host. +Rule: only /consumer and /provider/ may be exposed (the render refuses other paths), +and never /network or /provider/publish: /network is an internal tier, and /provider/publish +has no validateSign and signs as the provider. +{{- else if eq (include "adapter-service.role" .) "consumer" }} This is the consumer role, which accepts UNSIGNED requests — it is inside the trust boundary and has no validateSign step. An Ingress therefore exposes an unauthenticated entry point to the network: rate-limit this host, and do not diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl index ccbf19ad..7691a581 100644 --- a/charts/adapter-service/templates/_helpers.tpl +++ b/charts/adapter-service/templates/_helpers.tpl @@ -5,7 +5,8 @@ # Purpose: chart-local helpers delegating to common, plus the role, # identity, upstream and config-rendering wiring an adapter needs. # -# One chart, three roles. provider, network and consumer are the same image and the +# One chart, four roles. provider, network and consumer (one release per tier) +# and network-layer (all three tiers in one adapter) are the same image and the # same config format; the role decides the handler role, the step list and # where requests are routed. Install once per role, and keep the RELEASE name # per-role -- the release name is what becomes the Service DNS name that the @@ -14,7 +15,7 @@ */}} {{/* -The adapter's role: provider | network | consumer. +The adapter's role: provider | network | consumer | network-layer. Required, with no default. A default would silently give one role's step list and handler role to a different adapter -- which renders, starts, reports Ready @@ -23,10 +24,10 @@ and then mis-handles every request, at the far end, in a peer's logs. {{- define "adapter-service.role" -}} {{- $role := .Values.role | default "" -}} {{- if not $role -}} -{{- fail (printf "%s: role is required -- one of provider, network, consumer. It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} +{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or network-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} {{- end -}} -{{- if not (has $role (list "provider" "network" "consumer")) -}} -{{- fail (printf "%s: role must be one of provider, network, consumer (got %q)." .Chart.Name $role) -}} +{{- if not (has $role (list "provider" "network" "consumer" "network-layer")) -}} +{{- fail (printf "%s: role must be one of provider, network, consumer, network-layer (got %q)." .Chart.Name $role) -}} {{- end -}} {{- $role -}} {{- end }} diff --git a/charts/adapter-service/templates/_network-layer.tpl b/charts/adapter-service/templates/_network-layer.tpl new file mode 100644 index 00000000..e8a4fd54 --- /dev/null +++ b/charts/adapter-service/templates/_network-layer.tpl @@ -0,0 +1,205 @@ +{{/* +# ============================================================================ +# role: network-layer -- the unified single adapter. +# +# The consumer, network and provider tiers as four modules of ONE adapter +# process. Nothing here is a second copy of a tier: the config and the routing +# are GENERATED at render time from config/-config.yaml and +# config/routing-.yaml, the same files the three multi-adapter roles +# mount. A change to a tier file reaches both modes on the next render. +# +# Mirrors quick-start/bin/setup.py _render_unified_network_layer and +# quick-start/config/adapters/unified-network-layer/overrides.yaml. +# ============================================================================ +*/}} + +{{/* +"true" when this release runs the unified single adapter, else empty. +*/}} +{{- define "adapter-service.isNetworkLayer" -}} +{{- if eq (include "adapter-service.role" .) "network-layer" -}}true{{- end -}} +{{- end }} + +{{/* +Which tier-file module becomes which module of the unified single adapter. +`index` is the module's position in that tier file's `modules:` list. +`routing` is the routing file the module's router reads; the provider module +has no router (it answers requests itself). Topology, not environment, so it +is a constant here rather than a value -- the same rule config/routing-*.yaml +follows. +*/}} +{{- define "adapter-service.networkLayer.moduleMap" -}} +- {from: consumer, index: 0, name: consumer, path: /consumer/, routing: routing-network-layer-consumer.yaml} +- {from: network, index: 0, name: network, path: /network/, routing: routing-network.yaml} +- {from: provider, index: 0, name: provider, path: /provider/} +- {from: provider, index: 1, name: provider-publish, path: /provider/publish, routing: routing-network-layer-publish.yaml} +{{- end }} + +{{/* +One tier file, parsed, with its __ADAPTER_*__ placeholders renamed to that +tier's (__CONSUMER_*__, __NETWORK_*__, __PROVIDER_*__) so the init container can +fill each module from its own tier's Secret. +Usage: include "adapter-service.networkLayer.tier" (dict "ctx" $ "tier" "provider") | fromYaml +*/}} +{{- define "adapter-service.networkLayer.tier" -}} +{{- $file := printf "config/%s-config.yaml" .tier -}} +{{- $raw := .ctx.Files.Get $file -}} +{{- if not $raw -}} +{{- fail (printf "%s: role network-layer is generated from %s, which is missing from the chart." .ctx.Chart.Name $file) -}} +{{- end -}} +{{- $doc := $raw | replace "__ADAPTER_" (printf "__%s_" (upper .tier)) | fromYaml -}} +{{- if or (not $doc) (hasKey $doc "Error") -}} +{{- fail (printf "%s: %s does not parse as YAML, so the network-layer config cannot be generated from it." .ctx.Chart.Name $file) -}} +{{- end -}} +{{- toYaml $doc -}} +{{- end }} + +{{/* +The unified single adapter's config, as YAML. + +Process-level settings (log, http, pluginManager, otelsetup) come from the +consumer tier file, as in quick-start; appName, the listener port, the http +timeouts (http.timeout), the log level (logLevel) and telemetry (otel.*) are +set here from values. Each module is the +tier file's module with only its name, path, routingConfig, +extendedSchema_enabled, httpClientConfig and (when registryCache is on) the +cache plugin and registry cacheTTL changed. +*/}} +{{- define "adapter-service.networkLayer.config" -}} +{{- $ctx := . -}} +{{- $nl := .Values.networkLayer -}} +{{- $rc := $nl.registryCache -}} +{{- if and $rc.enabled (not $rc.addr) -}} +{{- fail (printf "%s: networkLayer.registryCache.enabled needs networkLayer.registryCache.addr (host:port of Redis). Without it the registry logs that no cache plugin is configured and caches nothing." $ctx.Chart.Name) -}} +{{- end -}} +{{- $tiers := dict -}} +{{- range $t := list "consumer" "network" "provider" -}} +{{- $_ := set $tiers $t (include "adapter-service.networkLayer.tier" (dict "ctx" $ctx "tier" $t) | fromYaml) -}} +{{- end -}} +{{- $modules := list -}} +{{- range $s := include "adapter-service.networkLayer.moduleMap" . | fromYamlArray -}} +{{- $src := (index $tiers $s.from).modules -}} +{{- if le (len $src) (int $s.index) -}} +{{- fail (printf "%s: network-layer module %s expects config/%s-config.yaml to have a module at index %d; it has %d." $ctx.Chart.Name $s.name $s.from (int $s.index) (len $src)) -}} +{{- end -}} +{{- $m := deepCopy (index $src (int $s.index)) -}} +{{- $_ := set $m "name" $s.name -}} +{{- $_ := set $m "path" $s.path -}} +{{- $p := $m.handler.plugins -}} +{{- if hasKey $p "router" -}} +{{- if not $s.routing -}} +{{- fail (printf "%s: network-layer module %s has a router but the module map names no routing file for it." $ctx.Chart.Name $s.name) -}} +{{- end -}} +{{- $_ := set $p.router.config "routingConfig" (printf "%s/%s" (include "adapter-service.configDir" $ctx) $s.routing) -}} +{{- end -}} +{{- if and (hasKey $p "schemaValidator") $nl.extendedSchemaEnabled -}} +{{- $_ := set $p.schemaValidator.config "extendedSchema_enabled" ($nl.extendedSchemaEnabled | toString) -}} +{{- end -}} +{{- with $nl.httpClient -}} +{{- $_ := set $m.handler "httpClientConfig" (deepCopy .) -}} +{{- end -}} +{{- if $rc.enabled -}} +{{- $_ := set $p "cache" (dict "id" "cache" "config" (dict "addr" $rc.addr "use_tls" ($rc.useTLS | toString))) -}} +{{- if hasKey $p "registry" -}} +{{- $_ := set $p.registry.config "cacheTTL" $rc.ttl -}} +{{- end -}} +{{- end -}} +{{- $modules = append $modules $m -}} +{{- end -}} +{{- $doc := omit (deepCopy $tiers.consumer) "modules" -}} +{{- $_ := set $doc "appName" (include "adapter-service.appName" .) -}} +{{- $_ := set $doc.log "level" .Values.logLevel -}} +{{- $_ := set $doc.http "port" (int .Values.service.targetPort) -}} +{{- with .Values.http.timeout -}} +{{- $_ := set $doc.http "timeout" (dict "read" (int .read) "write" (int .write) "idle" (int .idle)) -}} +{{- end -}} +{{- with (dig "plugins" "otelsetup" "config" dict $doc) -}} +{{- $on := $ctx.Values.otel.enabled | toString -}} +{{- $_ := set . "serviceName" (include "adapter-service.otelServiceName" $ctx) -}} +{{- $_ := set . "environment" $ctx.Values.otel.environment -}} +{{- $_ := set . "enableMetrics" $on -}} +{{- $_ := set . "enableTracing" $on -}} +{{- $_ := set . "enableLogs" $on -}} +{{- if $ctx.Values.otel.enabled -}} +{{- $_ := set . "otlpEndpoint" (include "adapter-service.otlpEndpoint" $ctx) -}} +{{- end -}} +{{- end -}} +{{- $_ := set $doc "modules" $modules -}} +{{- /* toYaml drops the quotes the tier files put round each placeholder. Put + them back: the init container substitutes the raw Secret value, and an + unquoted value can change type (an all-digit id becomes a number). */ -}} +# GENERATED by adapter-service (templates/_network-layer.tpl) from +# config/{consumer,network,provider}-config.yaml. Edit those, not this. +{{ regexReplaceAll "(?m):[ ]+(__[A-Z][A-Z0-9_]*__)$" (toYaml $doc) ": \"${1}\"" -}} +{{- end }} + +{{/* +Fails unless every private-key VALUE in a network-layer config is one of the +tier placeholders. Parsed, not grepped: it reads each module's keyManager. +Usage: include "adapter-service.networkLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) +*/}} +{{- define "adapter-service.networkLayer.checkKeys" -}} +{{- range $m := .cfg.modules -}} +{{- $km := dig "handler" "plugins" "keyManager" "config" dict $m -}} +{{- range $field := list "signingPrivateKey" "encrPrivateKey" -}} +{{- $v := index $km $field | default "" | toString -}} +{{- if and $v (not (regexMatch "^__(CONSUMER|NETWORK|PROVIDER)_(SIGNING|ENCR)_PRIVATE__$" $v)) -}} +{{- fail (printf "%s: network-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +Routing for the unified single adapter, as YAML: {: }. + +routing-network.yaml is the multi-adapter file, unchanged (discover and publish +to discovery). The other two are generated from routing-consumer.yaml and +routing-provider.yaml by pointing every tier address at THIS adapter's own +listener over loopback HTTP: http://-adapter[....][:port] becomes +http://localhost:/, and any path after it is kept +(.../publish becomes /network/publish). A target that is not a tier adapter is +kept as it is, and a rule with no target is left alone (the same nil guard as +configmap.yaml). So the two modes are one routing source, and splitting the +modules across pods again changes only the hosts. +*/}} +{{- define "adapter-service.networkLayer.routing" -}} +{{- $ctx := . -}} +{{- $loopback := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} +{{- /* Whole URL, anchored both ends: only a tier host (optionally with a + namespace suffix and port) followed by nothing or a path is rewritten. */ -}} +{{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} +{{- $out := dict -}} +{{- range $pair := list (list "routing-consumer.yaml" "routing-network-layer-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-network-layer-publish.yaml") -}} +{{- $src := index $pair 0 -}} +{{- $doc := $ctx.Files.Get (printf "config/%s" $src) | fromYaml -}} +{{- if not $doc.routingRules -}} +{{- fail (printf "%s: config/%s parsed to no routingRules; the network-layer routing is generated from it." $ctx.Chart.Name $src) -}} +{{- end -}} +{{- range $i, $rule := $doc.routingRules -}} +{{- if $rule.target -}} +{{- $url := $rule.target.url | default "" -}} +{{- $url = regexReplaceAll $tierHost $url $loopback -}} +{{- if hasSuffix "/" $url -}} +{{- fail (printf "%s: network-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} +{{- end -}} +{{- $_ := set $rule.target "url" $url -}} +{{- end -}} +{{- end -}} +{{- $_ := set $out (index $pair 1) $doc -}} +{{- end -}} +{{- toYaml $out -}} +{{- end }} + +{{/* +Secret holding one tier's keypair. Fails naming the tier: a missing identity +would otherwise start, report Ready, and fail every signature that tier makes. +Usage: include "adapter-service.networkLayer.tierSecret" (dict "ctx" $ "tier" "network") +*/}} +{{- define "adapter-service.networkLayer.tierSecret" -}} +{{- $name := (index .ctx.Values.networkLayer.keys .tier).existingSecret | default "" -}} +{{- if not $name -}} +{{- fail (printf "%s: networkLayer.keys.%s.existingSecret is required -- the %s tier signs as its own identity, from its own Secret (the six keys under keys.existingSecret)." .ctx.Chart.Name .tier .tier) -}} +{{- end -}} +{{- $name -}} +{{- end }} diff --git a/charts/adapter-service/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml index f5c47701..50140264 100644 --- a/charts/adapter-service/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -9,6 +9,10 @@ into an emptyDir at start-up. Everything that differs between the provider, network and consumer adapters is a value: they are one image with one config format, and the role decides the identity, the handler role, the step list and where requests are routed. + +role network-layer is the exception to "one file per role": its adapter.yaml and +routing files are generated from the three tier files (_network-layer.tpl), and +it renders more than one routing file. */}} {{/* RAW CONFIG OVERRIDE. @@ -24,6 +28,23 @@ at pod start, which is why no rendered config ever holds a key. A raw config with a literal private key in it would put that key in git, in a ConfigMap, and in `helm get values` -- so the render refuses it. */}} +{{- $networkLayer := include "adapter-service.isNetworkLayer" . }} +{{- $nlCfg := "" }} +{{- if $networkLayer }} +{{- /* role network-layer: the config is GENERATED from the three tier files + (_network-layer.tpl), unless `config` replaces it whole. Either way every + private key must be a tier placeholder; checked on the parsed config. */}} +{{- if .Values.config }} +{{- $nlCfg = .Values.config }} +{{- else }} +{{- $nlCfg = include "adapter-service.networkLayer.config" . }} +{{- end }} +{{- $nlParsed := $nlCfg | fromYaml }} +{{- if or (not $nlParsed) (hasKey $nlParsed "Error") }} +{{- fail (printf "%s: the network-layer adapter config (`config` override or generated) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} +{{- end }} +{{- include "adapter-service.networkLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} +{{- else }} {{- $cfgFile := printf "config/%s-config.yaml" (include "adapter-service.role" .) }} {{- $cfg := .Values.config | default (.Files.Get $cfgFile) }} {{- if $cfg }} @@ -33,6 +54,7 @@ in `helm get values` -- so the render refuses it. {{- end }} {{- end }} {{- end }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -44,6 +66,18 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: + {{- if $networkLayer }} + adapter.yaml: | + {{- $nlCfg | nindent 4 }} + {{- if .Values.routing.config }} + {{- fail (printf "%s: routing.config is not used by the network-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} + {{- end }} + {{- range $file, $doc := include "adapter-service.networkLayer.routing" . | fromYaml }} + {{ $file }}: | + # GENERATED by adapter-service from config/routing-*.yaml. Do not edit; edit those files. + {{- toYaml $doc | nindent 4 }} + {{- end }} + {{- else }} adapter.yaml: | {{- /* The config comes from a FILE, not from this template. @@ -99,3 +133,4 @@ data: {{- end }} {{- end }} {{- $routes | nindent 4 }} + {{- end }} diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index e6c623c2..f0a8c2b5 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -44,6 +44,22 @@ spec: securityContext: {{- . | nindent 8 }} {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- range . }} + - {{- toYaml (omit . "labelSelector") | nindent 10 }} + labelSelector: + {{- if .labelSelector }} + {{- toYaml .labelSelector | nindent 12 }} + {{- else }} + matchLabels: + {{- include "adapter-service.selectorLabels" $ | nindent 14 }} + {{- end }} + {{- end }} + {{- end }} initContainers: {{- include "common.waitFor" (dict "ctx" . "tcp" list "http" list) | nindent 8 }} {{/* @@ -70,13 +86,23 @@ spec: - -c - | set -eu + {{- if include "adapter-service.isNetworkLayer" . }} + # network-layer: adapter.yaml plus one routing file per routed module. + cp /config-template/*.yaml /rendered/ + {{- else }} cp /config-template/adapter.yaml /rendered/adapter.yaml cp /config-template/{{ include "adapter-service.routingFile" . }} /rendered/{{ include "adapter-service.routingFile" . }} + {{- end }} subst() { # $1 placeholder, $2 file holding the value if [ ! -s "$2" ]; then + {{- if include "adapter-service.isNetworkLayer" . }} + # three tier Secrets share key names: name the tier too (e.g. network/keyId) + echo "render-config: ${2#/keys/} is missing or empty in its tier's keys Secret" >&2 + {{- else }} echo "render-config: $(basename "$2") is missing or empty in the keys Secret" >&2 + {{- end }} exit 1 fi # A value the registry has not assigned yet. keyId is the osid it @@ -85,7 +111,11 @@ spec: # -- the sentinel contains a colon -- or an adapter signing with an # identity nothing can verify. if head -c 9 "$2" | grep -q "<&2 + {{- else }} echo "render-config: $(basename "$2") is still unresolved." >&2 + {{- end }} echo " The registry assigns it when registry-seed writes this" >&2 echo " adapter's public key. Run that, then:" >&2 echo " ./scripts/resolve-key-ids.sh -e " >&2 @@ -93,6 +123,24 @@ spec: fi sed -i "s|$1|$(cat "$2")|g" /rendered/adapter.yaml } + {{- if include "adapter-service.isNetworkLayer" . }} + + {{- /* Three identities: each tier's module signs as that tier, from + that tier's Secret mounted at /keys/. */}} + {{- $k := .Values.keys.existingSecret }} + {{- range $tier := list "consumer" "network" "provider" }} + {{- $T := upper $tier }} + subst __{{ $T }}_SUBSCRIBER_ID__ /keys/{{ $tier }}/{{ $k.subscriberIdKey }} + subst __{{ $T }}_KEY_ID__ /keys/{{ $tier }}/{{ $k.keyIdKey }} + subst __{{ $T }}_SIGNING_PRIVATE__ /keys/{{ $tier }}/{{ $k.signingPrivateKey }} + subst __{{ $T }}_SIGNING_PUBLIC__ /keys/{{ $tier }}/{{ $k.signingPublicKey }} + subst __{{ $T }}_ENCR_PRIVATE__ /keys/{{ $tier }}/{{ $k.encrPrivateKey }} + subst __{{ $T }}_ENCR_PUBLIC__ /keys/{{ $tier }}/{{ $k.encrPublicKey }} + {{- end }} + {{- range $ph, $path := .Values.networkLayer.extraSubstitutions }} + subst {{ $ph }} /keys/{{ $path }} + {{- end }} + {{- else }} subst __ADAPTER_SUBSCRIBER_ID__ /keys/{{ .Values.keys.existingSecret.subscriberIdKey }} subst __ADAPTER_KEY_ID__ /keys/{{ .Values.keys.existingSecret.keyIdKey }} @@ -107,6 +155,7 @@ spec: {{- range $ph, $key := .Values.keys.existingSecret.extraSubstitutions }} subst {{ $ph }} /keys/{{ $key }} {{- end }} + {{- end }} # A placeholder that survived means a key was substituted with # nothing. Catch it here rather than at the far end, in a peer's @@ -121,9 +170,17 @@ spec: - name: config-template mountPath: /config-template readOnly: true + {{- if include "adapter-service.isNetworkLayer" . }} + {{- range $tier := list "consumer" "network" "provider" }} + - name: keys-{{ $tier }} + mountPath: /keys/{{ $tier }} + readOnly: true + {{- end }} + {{- else }} - name: keys mountPath: /keys readOnly: true + {{- end }} - name: config mountPath: /rendered resources: @@ -135,6 +192,12 @@ spec: - name: {{ .Chart.Name }} image: {{ include "adapter-service.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.preStopSleepSeconds }} + lifecycle: + preStop: + sleep: + seconds: {{ .Values.preStopSleepSeconds }} + {{- end }} {{- with (include "common.securityContext" .) }} securityContext: {{- . | nindent 12 }} @@ -150,6 +213,21 @@ spec: */}} - name: CONFIG_FILE value: {{ include "adapter-service.configDir" . }}/adapter.yaml + {{- with .Values.goMemLimit }} + - name: GOMEMLIMIT + value: {{ . | quote }} + {{- end }} + {{- if include "adapter-service.isNetworkLayer" . }} + {{- with .Values.networkLayer.registryCache }} + {{- if and .enabled .passwordSecret.name }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .passwordSecret.name }} + key: {{ .passwordSecret.key }} + {{- end }} + {{- end }} + {{- end }} {{- with .Values.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -176,10 +254,19 @@ spec: - name: config-template configMap: name: {{ include "adapter-service.fullname" . }}-config + {{- if include "adapter-service.isNetworkLayer" . }} + {{- range $tier := list "consumer" "network" "provider" }} + - name: keys-{{ $tier }} + secret: + secretName: {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" $ "tier" $tier) }} + defaultMode: 0400 + {{- end }} + {{- else }} - name: keys secret: secretName: {{ include "adapter-service.keysSecretName" . }} defaultMode: 0400 + {{- end }} {{/* The rendered config. emptyDir rather than anything durable: it is derived from a ConfigMap and a Secret and is rebuilt on every start, diff --git a/charts/adapter-service/templates/hpa.yaml b/charts/adapter-service/templates/hpa.yaml index 6f696997..0720c2a3 100644 --- a/charts/adapter-service/templates/hpa.yaml +++ b/charts/adapter-service/templates/hpa.yaml @@ -29,4 +29,8 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index 529e1f6a..68dd7ad8 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,4 +1,38 @@ {{- if .Values.ingress.enabled }} +{{- if include "adapter-service.isNetworkLayer" . }} +{{- /* role network-layer: only /consumer... and individual /provider/ + paths may be public. "/" or "/provider" with Prefix would also expose + /network (an internal tier) and /provider/publish, which has no + validateSign and signs as the provider. + The path check holds only for literal prefix matching, so pathType must be + Prefix or Exact (ImplementationSpecific is a plain string prefix on + ingress-nginx: /provider/p would match /provider/publish), the path must + be literal, and regex or rewrite annotations are refused. */}} +{{- $ann := merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict) }} +{{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString) "true" }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} +{{- end }} +{{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} +{{- end }} +{{- range .Values.ingress.hosts }} +{{- range .paths }} +{{- $p := .path | default "" }} +{{- $pt := .pathType | default "Prefix" }} +{{- if not (has $pt (list "Prefix" "Exact")) }} +{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} +{{- end }} +{{- if or (not (regexMatch "^/[A-Za-z0-9/_.-]*$" $p)) (contains ".." $p) (contains "//" $p) }} +{{- fail (printf "%s: ingress path %q is not allowed for the network-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} +{{- end }} +{{- $consumer := or (eq $p "/consumer") (hasPrefix "/consumer/" $p) }} +{{- $provider := and (hasPrefix "/provider/" $p) (ne $p "/provider/") (not (hasPrefix "/provider/publish" $p)) }} +{{- if not (or $consumer $provider) }} +{{- fail (printf "%s: ingress path %q is not allowed for the network-layer role. Allowed: /consumer... and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} {{/* Off by default, and worth a thought before turning on. diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index d00b624a..baf9f19e 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -1,9 +1,10 @@ # =========================================================================== # adapter-service # -# One chart for all three Beckn adapters. provider, network and consumer are the -# same image and the same config format; `role` below decides the handler -# role, the step list and where requests are routed. +# One chart for the Beckn adapters: provider, network and consumer (one release +# per tier), or network-layer (all three tiers in one adapter). Same image and +# config format; `role` below decides the handler role, the step list and +# where requests are routed. # # INSTALL ONCE PER ROLE, and keep the RELEASE name per-role: # @@ -11,6 +12,10 @@ # helm install network-adapter . -f examples/network.yaml # helm install consumer-adapter . -f examples/consumer.yaml # +# OR, all three tiers in one adapter (one release instead of the three above): +# +# helm install network-layer-adapter . -f examples/network-layer.yaml +# # The release name is what the Service is called, which is the DNS name the # other adapters address. Rename a release and you must update whatever routes # to it -- consumer routes to the network adapter, provider is routed to by network. @@ -25,7 +30,7 @@ # =========================================================================== # --------------------------------------------------------------------------- -# Role: provider | network | consumer +# Role: provider | network | consumer | network-layer # # Required, with no default. A default would hand one role's handler role and # step list to a different adapter, which renders, starts, reports Ready and @@ -265,6 +270,11 @@ otel: # the compose stack so traces from either deployment line up. serviceName: "" +# Read ONLY by role network-layer, whose adapter.yaml is generated. The +# provider, network and consumer files hardcode `log.level: debug`, so this does +# not change them. The same holds for `http.timeout` below and for `otel` +# above: role network-layer writes them into its config, the other roles keep +# their tier file's values. logLevel: info http: @@ -327,21 +337,26 @@ resources: # Both hit /health, which the adapter serves from its own mux and which does # not touch the registry or discovery -- so a readiness failure here means # this process is unwell, not that something downstream is. -probes: - liveness: - enabled: true +startupProbe: + enabled: false +livenessProbe: + enabled: true + httpGet: path: /health - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 3 - failureThreshold: 3 - readiness: - enabled: true + port: http + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 +readinessProbe: + enabled: true + httpGet: path: /health - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 autoscaling: enabled: false @@ -349,6 +364,8 @@ autoscaling: maxReplicas: 6 targetCPUUtilizationPercentage: 75 targetMemoryUtilizationPercentage: "" + # Passed through to the HPA as spec.behavior. {} = Kubernetes defaults. + behavior: {} podDisruptionBudget: enabled: false @@ -376,6 +393,20 @@ securityContext: drop: - ALL +# Spread replicas so one node loss is not an outage. labelSelector defaults to +# this release's selector labels when omitted. +topologySpreadConstraints: [] + +# Graceful shutdown. preStop sleeps so the endpoint leaves the Service before +# the process stops accepting. preStop.sleep needs Kubernetes 1.30+; keep 0 on +# older clusters. Empty grace period = the Kubernetes default (30s). +terminationGracePeriodSeconds: "" +preStopSleepSeconds: 0 + +# Soft memory limit for the Go runtime (e.g. 900MiB under a 1Gi limit), so GC +# works harder before the container is OOM-killed. Empty = unset. +goMemLimit: "" + nodeSelector: {} tolerations: [] affinity: {} @@ -389,3 +420,76 @@ extraVolumes: [] extraVolumeMounts: [] extraEnv: [] envFromSecrets: [] + +# --------------------------------------------------------------------------- +# role: network-layer -- the unified single adapter. +# +# The consumer, network and provider tiers as four modules of ONE process +# (/consumer/, /network/, /provider/, /provider/publish). Its config and routing +# are generated at render time from the three tier files in config/, so there +# is no second copy to keep in step. Tier-to-tier hops are loopback HTTP to +# this pod's own listener (http://localhost:/): any +# adapter image works. +# +# Ignored by the provider, network and consumer roles. +# +# NOT USED by this role: `routing.config` (setting it fails the render, because +# the routing is generated) and `handler.*` (role, steps and path come from +# the tier files). `config` replaces the generated adapter.yaml WHOLE: the +# private-key guard still runs on it, but then extendedSchemaEnabled, +# httpClient and registryCache do not apply, it must point its routers at +# /app/config/routing-network-layer-consumer.yaml, routing-network.yaml and +# routing-network-layer-publish.yaml, and `logLevel`, `http.timeout`, `otel` +# and the port are yours to set. +# --------------------------------------------------------------------------- +networkLayer: + # Forces extendedSchema_enabled on every module that validates schemas, as + # the quick-start does. "" keeps each tier file's own setting. + extendedSchemaEnabled: "true" + + # Three identities, one per tier: the same Secrets (six keys each, named as + # under keys.existingSecret) and the same registry rows the multi-adapter + # releases use, so either mode runs against the same registry. They must + # exist in THIS release's namespace (openbao-secrets reflects them there). + keys: + consumer: + existingSecret: "" + network: + existingSecret: "" + provider: + existingSecret: "" + + # Extra placeholders, as "/". + # __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + extraSubstitutions: {} + + # Outbound HTTP pool for every module: discovery, the registry, upstream APIs + # and the loopback hops to this pod's own listener. Go's default keeps 2 idle + # connections per host, so under load most calls open a new connection. + # responseHeaderTimeout bounds the wait for an answer's headers, which on the + # consumer -> provider hop includes the upstream API call: keep it above the + # slowest upstream. + # Read ONLY by role network-layer; the multi tier files keep their own settings. + # To leave the adapter's own defaults (no httpClientConfig at all) set it to + # null: `--set networkLayer.httpClient=null`, or `httpClient: null` in an + # environment values file. `{}` does NOT work: Helm merges maps, so {} keeps + # the four defaults below. + httpClient: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + + # Registry lookups cached in Redis. Off: every request asks the registry. + # ttl is how long a suspended participant keeps verifying and a withdrawn + # capability keeps being called -- keep it short. Redis is not deployed by + # this chart. Read ONLY by role network-layer. + registryCache: + enabled: false + addr: "" # host:port, e.g. redis-master.redis.svc.cluster.local:6379 + useTLS: false + ttl: 60s + # Optional. Read by the cache plugin from REDIS_PASSWORD. + passwordSecret: + name: "" + key: password diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index 9e3e163e..d25114c2 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to the `openbao-secrets` chart are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.0] - 2026-10-09 + +### Added +- `network-layer-adapter` in the namespaces of `consumer-adapter-keys`, + `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for + `adapter-service` role `network-layer` (all three tiers in one adapter, three + identities). Inert where that namespace does not exist. + ## [0.1.0] - 2026-09-27 ### Added diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index 75fc2c67..6964dcf7 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -57,14 +57,16 @@ secrets: # -- adapter identities. keyId inside these is filled in by # resolve-key-ids.sh AFTER the registry has been seeded, so these entries # change once per environment after the first bring-up. + # network-layer-adapter (role network-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. - name: consumer-adapter-keys - namespaces: [consumer-adapter] + namespaces: [consumer-adapter, network-layer-adapter] - name: network-adapter-keys - namespaces: [network-adapter] + namespaces: [network-adapter, network-layer-adapter] - name: provider-adapter-keys - namespaces: [provider-adapter] + namespaces: [provider-adapter, network-layer-adapter] # The provider adapter's upstream credentials, named as environment variables - # by its config rather than substituted into it. + # by its config rather than substituted into it. network-layer-adapter runs + # the provider tier too, so it reads them as well. - name: provider-upstream - namespaces: [provider-adapter] + namespaces: [provider-adapter, network-layer-adapter] From 204446c8be739db4e1c4d4ccafe1ee041b9e2cc0 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 16:12:11 +0530 Subject: [PATCH 06/43] feat(adapter-service): enable OpenTelemetry by default and update related configurations --- charts/adapter-service/CHANGELOG.md | 3 +++ charts/adapter-service/examples/network-layer.yaml | 4 ++-- charts/adapter-service/templates/NOTES.txt | 2 +- charts/adapter-service/templates/_network-layer.tpl | 5 +++-- charts/adapter-service/values.yaml | 7 +++++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 2969ce91..a6348f11 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -22,6 +22,9 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h no `use-regex` / `rewrite-target` annotation. The generated config takes `logLevel`, `http.timeout.*` and `otel.*` from values. Example `examples/network-layer.yaml`. +- `otel.enabled` now defaults to `true`. Read by role network-layer; with + `otel.endpoint` empty it keeps the collector named in the tier config + files. Set `otel.enabled: false` where no collector runs. - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. diff --git a/charts/adapter-service/examples/network-layer.yaml b/charts/adapter-service/examples/network-layer.yaml index 497e150a..28cd1cf9 100644 --- a/charts/adapter-service/examples/network-layer.yaml +++ b/charts/adapter-service/examples/network-layer.yaml @@ -122,5 +122,5 @@ waitFor: enabled: false otel: - enabled: false - endpoint: "" + enabled: true + endpoint: "" # empty: the collector named in config/*-config.yaml diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index feddc34d..c062433b 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -10,7 +10,7 @@ service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} registry {{ .Values.registry.url }} routes {{ if include "adapter-service.isNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} - telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} + telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint | default "collector from config/*-config.yaml" }}{{ else }}off{{ end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. That name comes from the RELEASE name — whatever routes here must use it, so diff --git a/charts/adapter-service/templates/_network-layer.tpl b/charts/adapter-service/templates/_network-layer.tpl index e8a4fd54..0d9d3cdd 100644 --- a/charts/adapter-service/templates/_network-layer.tpl +++ b/charts/adapter-service/templates/_network-layer.tpl @@ -114,14 +114,15 @@ cache plugin and registry cacheTTL changed. {{- $_ := set $doc.http "timeout" (dict "read" (int .read) "write" (int .write) "idle" (int .idle)) -}} {{- end -}} {{- with (dig "plugins" "otelsetup" "config" dict $doc) -}} +{{- $cfg := . -}} {{- $on := $ctx.Values.otel.enabled | toString -}} {{- $_ := set . "serviceName" (include "adapter-service.otelServiceName" $ctx) -}} {{- $_ := set . "environment" $ctx.Values.otel.environment -}} {{- $_ := set . "enableMetrics" $on -}} {{- $_ := set . "enableTracing" $on -}} {{- $_ := set . "enableLogs" $on -}} -{{- if $ctx.Values.otel.enabled -}} -{{- $_ := set . "otlpEndpoint" (include "adapter-service.otlpEndpoint" $ctx) -}} +{{- with $ctx.Values.otel.endpoint -}} +{{- $_ := set $cfg "otlpEndpoint" . -}} {{- end -}} {{- end -}} {{- $_ := set $doc "modules" $modules -}} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index baf9f19e..4c039115 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -262,8 +262,11 @@ becknSpec: # and does not go stale. # --------------------------------------------------------------------------- otel: - enabled: false - # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" + # On by default. Set false where no collector runs: an exporter with nothing + # listening logs a failure every interval. + enabled: true + # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317". + # Left empty, the collector named in the tier config files is used. endpoint: "" environment: dev # Left empty, defaults to oan--adapter, matching OTEL_SERVICE_NAME in From f2d4a31651c3e5471f7916ee138dc90ad26534c5 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 16:49:50 +0530 Subject: [PATCH 07/43] feat(adapter-service): implement single network layer adapter with unified configuration and routing --- charts/adapter-service/CHANGELOG.md | 18 +- charts/adapter-service/README.md | 16 +- .../ci/network-layer-values.yaml | 9 +- .../config/single-network-layer-config.yaml | 365 ++++++++++++++++++ .../examples/network-layer.yaml | 16 +- charts/adapter-service/templates/NOTES.txt | 2 +- .../templates/_network-layer.tpl | 131 +------ .../adapter-service/templates/configmap.yaml | 24 +- .../adapter-service/templates/deployment.yaml | 11 - charts/adapter-service/values.yaml | 73 +--- 10 files changed, 431 insertions(+), 234 deletions(-) create mode 100644 charts/adapter-service/config/single-network-layer-config.yaml diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index a6348f11..c04843cb 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -10,21 +10,17 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ### Added - `role: network-layer`: all three tiers in one adapter (four modules: `/consumer/`, `/network/`, `/provider/`, `/provider/publish`), as in the - quick-start's network-layer compose stack. Its config and routing are generated - at render time from `config/*.yaml`, so there is no second copy of any module. - Tier hops are loopback HTTP inside the pod. Signs as three identities from + quick-start's network-layer compose stack. Its config is + `config/single-network-layer-config.yaml` (extended schema, an outbound + connection pool and log level info on every module); its routing is generated + at render time from `config/routing-*.yaml`. Tier hops are loopback HTTP + inside the pod. Signs as three identities from `networkLayer.keys.{consumer,network,provider}`; `routing.config` fails the render for this role. -- `networkLayer.httpClient` (connection pool on every module; `null` turns it - off) and `networkLayer.registryCache` (optional Redis registry cache), - network-layer only. Ingress may expose only `/consumer...` and +- network-layer ingress may expose only `/consumer...` and `/provider/`, as literal paths with `pathType` Prefix or Exact and - no `use-regex` / `rewrite-target` annotation. The generated config takes - `logLevel`, `http.timeout.*` and `otel.*` from values. Example + no `use-regex` / `rewrite-target` annotation. Example `examples/network-layer.yaml`. -- `otel.enabled` now defaults to `true`. Read by role network-layer; with - `otel.endpoint` empty it keeps the collector named in the tier config - files. Set `otel.enabled: false` where no collector runs. - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index 25b3e4ae..e73d77c0 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -139,13 +139,15 @@ healthy. | network layer | `network-layer-adapter` (examples/network-layer.yaml) | `network-layer-adapter:9200/consumer/`, `network-layer-adapter:9200/provider/publish` | `role: network-layer` runs all three tiers in one adapter, as four -modules of one process. Its config and routing are -**generated** from `config/*.yaml` at render time, so a change to a tier file -reaches both modes. Tier hops are loopback HTTP inside the pod (any image). It -signs as the same three identities, from the same three Secrets, so the -registry is the same in both modes. An environment's `config:` override for a -multi role is NOT seen by network-layer; give the network-layer release its own -`config:` if needed (it replaces the generated config whole). +modules of one process. Its config is `config/single-network-layer-config.yaml` +(each module is its tier's module from `config/-config.yaml`, so a change +to a tier file that should apply to both modes goes in both files). Its routing +is **generated** at render time from `config/routing-*.yaml`, so routes stay in +one place. Tier hops are loopback HTTP inside the pod (any image). It signs as +the same three identities, from the same three Secrets, so the registry is the +same in both modes. An environment's `config:` override for a multi role is NOT +seen by network-layer; give the network-layer release its own `config:` if +needed (it replaces `single-network-layer-config.yaml` whole). Pick the mode per environment with ONE switch where releases are chosen. For a helmfile (adapt to infra-automation's layout): diff --git a/charts/adapter-service/ci/network-layer-values.yaml b/charts/adapter-service/ci/network-layer-values.yaml index 572c2932..e4a4e00c 100644 --- a/charts/adapter-service/ci/network-layer-values.yaml +++ b/charts/adapter-service/ci/network-layer-values.yaml @@ -1,7 +1,8 @@ # charts/adapter-service/ci/network-layer-values.yaml -# role network-layer: the unified single adapter (consumer, network and -# provider tiers as modules of one process). Renders the generated config, the -# loopback routing and the three tier identities. +# role network-layer: the single network layer adapter (consumer, network and +# provider tiers as modules of one process). Renders +# config/single-network-layer-config.yaml, the loopback routing and the three +# tier identities. role: network-layer fullnameOverride: network-layer-adapter @@ -28,7 +29,7 @@ networkLayer: extraSubstitutions: __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl -# Scale defaults for the unified single adapter: roughly the three tiers' +# Scale defaults for the single network layer adapter: roughly the three tiers' # requests in one pod, at least one replica per node. resources: requests: diff --git a/charts/adapter-service/config/single-network-layer-config.yaml b/charts/adapter-service/config/single-network-layer-config.yaml new file mode 100644 index 00000000..2ee9fb9a --- /dev/null +++ b/charts/adapter-service/config/single-network-layer-config.yaml @@ -0,0 +1,365 @@ +# OAN single network layer adapter (role network-layer). The consumer, network +# and provider tiers as four modules of ONE adapter process: +# +# /consumer/ the caller (bap); the app posts here, unsigned +# /network/ verifies and forwards to discovery +# /provider/ answers from the upstream APIs (bpp) +# /provider/publish the catalogue system publishes here +# +# Each module is its tier's module from consumer-config.yaml, network-config.yaml +# and provider-config.yaml. A change to one of those files that should also +# apply here must be made in this file too. +# +# Differences from the three tier files: +# - one listener (9200) and one appName; tier hops are loopback HTTP to this +# same listener (see the routing-network-layer-*.yaml files, which the chart +# generates from routing-consumer.yaml and routing-provider.yaml); +# - extended schema validation on in every module; +# - an outbound connection pool (httpClientConfig) on every module; +# - log level info. +# +# This file is mounted verbatim. Each module signs as its OWN tier: the +# __CONSUMER_*__, __NETWORK_*__ and __PROVIDER_*__ placeholders are substituted +# by the init container from that tier's Secret +# (networkLayer.keys..existingSecret) at pod start, so no key is in this +# file, in the ConfigMap, or in git. +appName: "network-layer-adapter" + +log: + level: info + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + # Also the port the generated loopback routing uses: keep it equal to the + # chart's service.targetPort. + port: 9200 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +plugins: + # Ships metrics, traces and logs over OTLP/gRPC. On because clickstack is + # deployed alongside. Turn all three off for a cluster without a collector: + # the exporter logs a connection failure every few seconds when nothing + # answers. + otelsetup: + id: otelsetup + config: + serviceName: "oan-network-layer-adapter" + environment: "dev" + otlpEndpoint: "clickstack-otel-collector.observability.svc.cluster.local:4317" + enableMetrics: "true" + enableTracing: "true" + enableLogs: "true" + +modules: + # ------------------------------------------------------------------ consumer + # From consumer-config.yaml. Accepts unsigned requests from the app, signs as + # the consumer, and routes discover to /network and transactions to /provider. + - name: consumer + path: / + handler: + type: std + role: bap + subscriberId: "__CONSUMER_SUBSCRIBER_ID__" + # Outbound pool. Go's default keeps 2 idle connections per host, so under + # load most calls would open a new connection. responseHeaderTimeout + # bounds the wait for an answer's headers, which on the consumer -> + # provider hop includes the upstream API call. + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry.registry.svc.cluster.local:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + cacheTTL: 0 + keyManager: + id: simplekeymanager + config: + subscriberId: "__CONSUMER_SUBSCRIBER_ID__" + keyId: "__CONSUMER_KEY_ID__" + signingPrivateKey: "__CONSUMER_SIGNING_PRIVATE__" + signingPublicKey: "__CONSUMER_SIGNING_PUBLIC__" + encrPrivateKey: "__CONSUMER_ENCR_PRIVATE__" + encrPublicKey: "__CONSUMER_ENCR_PUBLIC__" + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "true" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + signValidator: + id: signvalidator + signer: + id: signer + router: + id: router + config: + routingConfig: /app/config/routing-network-layer-consumer.yaml + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + steps: + - validateSchema + - addRoute + - sign + + # ------------------------------------------------------------------- network + # From network-config.yaml. Verifies the caller's signature and forwards + # discover and publish to discovery. Internal: never expose it on an Ingress. + - name: network + path: /network/ + handler: + type: std + role: bpp + subscriberId: "__NETWORK_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry.registry.svc.cluster.local:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + cacheTTL: 0 + keyManager: + id: simplekeymanager + config: + subscriberId: "__NETWORK_SUBSCRIBER_ID__" + keyId: "__NETWORK_KEY_ID__" + signingPrivateKey: "__NETWORK_SIGNING_PRIVATE__" + signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" + encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" + encrPublicKey: "__NETWORK_ENCR_PUBLIC__" + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "true" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + signValidator: + id: signvalidator + signer: + id: signer + router: + id: router + config: + # The multi-adapter network routing, unchanged: discover and + # publish to discovery. + routingConfig: /app/config/routing-network.yaml + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + steps: + - validateSign + - addRoute + + # ------------------------------------------------------------------ provider + # From provider-config.yaml, module 1. Verifies the caller and answers from + # the upstream APIs itself, so it has no router. Upstream credentials come + # from the environment (envFromSecrets: provider-upstream). + - name: provider + path: /provider/ + handler: + type: std + role: bpp + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry.registry.svc.cluster.local:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + cacheTTL: 0 + keyManager: + id: simplekeymanager + config: + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + keyId: "__PROVIDER_KEY_ID__" + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "true" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + signValidator: + id: signvalidator + signer: + id: signer + mapper: + id: jsonmapper + config: + fetchTimeout: 10s + maxMappingBytes: 262144 + cacheTTL: 1m + negativeTTL: 1m + maxCacheEntries: 200 + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + providerSteps: + - id: WeatherObservation + config: + bindingKeys: "mausamgram|openagrinet:WeatherObservation" + mausamgram: + authScheme: header + headerName: Authorization + headerValueEnv: MAUSAMGRAM_AUTH + + - id: MandiPrice + config: + bindingKeys: "agmarknet|openagrinet:MandiPrice" + agmarknet: + authScheme: none + + - id: KnowledgeAdvisory + config: + bindingKeys: "bharat-vistaar|openagrinet:KnowledgeAdvisory" + bharat-vistaar: + authScheme: oauth2 + # Substituted from the provider Secret (vistaarTokenUrl) via + # networkLayer.extraSubstitutions. + tokenUrl: "__VISTAAR_TOKEN_URL__" + clientIdEnv: VISTAAR_CLIENT_ID + clientSecretEnv: VISTAAR_CLIENT_SECRET + maxResponseBytes: 1048576 + + - id: AgricultureFacility + config: + bindingKeys: "pocra|openagrinet:AgricultureFacility" + pocra: + authScheme: none + searchConcurrency: 1 + steps: + - validateSign + - validateSchema + - WeatherObservation + - MandiPrice + - KnowledgeAdvisory + - AgricultureFacility + + # ---------------------------------------------------------- provider-publish + # From provider-config.yaml, module 2. The catalogue system publishes here; + # signed as the provider and routed to /network/publish. No validateSign: + # never expose it on an Ingress. + - name: provider-publish + path: /network/publish + handler: + type: std + role: bap + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry.registry.svc.cluster.local:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + cacheTTL: 0 + keyManager: + id: simplekeymanager + config: + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + keyId: "__PROVIDER_KEY_ID__" + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "true" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + signer: + id: signer + router: + id: router + config: + routingConfig: /app/config/routing-network-layer-publish.yaml + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + steps: + - validateSchema + - addRoute + - sign diff --git a/charts/adapter-service/examples/network-layer.yaml b/charts/adapter-service/examples/network-layer.yaml index 28cd1cf9..d3779ab2 100644 --- a/charts/adapter-service/examples/network-layer.yaml +++ b/charts/adapter-service/examples/network-layer.yaml @@ -1,5 +1,5 @@ # =========================================================================== -# network-layer role -- the unified single adapter. +# network-layer role -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: @@ -16,8 +16,9 @@ # helm install network-layer-adapter . -n network-layer-adapter --create-namespace \ # -f examples/network-layer.yaml # -# The config and routing are generated from the three tier files in config/; -# tier hops are loopback HTTP inside the pod, so any adapter image works. +# The config is config/single-network-layer-config.yaml; the routing is +# generated from config/routing-*.yaml. Tier hops are loopback HTTP inside the +# pod, so any adapter image works. # =========================================================================== role: network-layer @@ -33,8 +34,9 @@ image: registry: url: http://registry:8081/api/v1 -# The same port the quick-start publishes. It is also the loopback port the -# generated routing uses, so changing targetPort moves both. +# The same port the quick-start publishes. targetPort is also the loopback port +# the generated routing uses, and must equal http.port in +# config/single-network-layer-config.yaml (the render checks). service: port: 9200 targetPort: 9200 @@ -122,5 +124,5 @@ waitFor: enabled: false otel: - enabled: true - endpoint: "" # empty: the collector named in config/*-config.yaml + enabled: false + endpoint: "" diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index c062433b..feddc34d 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -10,7 +10,7 @@ service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} registry {{ .Values.registry.url }} routes {{ if include "adapter-service.isNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} - telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint | default "collector from config/*-config.yaml" }}{{ else }}off{{ end }} + telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. That name comes from the RELEASE name — whatever routes here must use it, so diff --git a/charts/adapter-service/templates/_network-layer.tpl b/charts/adapter-service/templates/_network-layer.tpl index 0d9d3cdd..ed71fb2d 100644 --- a/charts/adapter-service/templates/_network-layer.tpl +++ b/charts/adapter-service/templates/_network-layer.tpl @@ -1,139 +1,22 @@ {{/* # ============================================================================ -# role: network-layer -- the unified single adapter. +# role: network-layer -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE adapter -# process. Nothing here is a second copy of a tier: the config and the routing -# are GENERATED at render time from config/-config.yaml and -# config/routing-.yaml, the same files the three multi-adapter roles -# mount. A change to a tier file reaches both modes on the next render. -# -# Mirrors quick-start/bin/setup.py _render_unified_network_layer and -# quick-start/config/adapters/unified-network-layer/overrides.yaml. +# process. The adapter config is config/single-network-layer-config.yaml +# (configmap.yaml mounts it like any role's file). The routing is GENERATED at +# render time from config/routing-.yaml, the same files the three +# multi-adapter roles mount, with tier hosts pointed at this pod's own listener. # ============================================================================ */}} {{/* -"true" when this release runs the unified single adapter, else empty. +"true" when this release runs the single network layer adapter, else empty. */}} {{- define "adapter-service.isNetworkLayer" -}} {{- if eq (include "adapter-service.role" .) "network-layer" -}}true{{- end -}} {{- end }} -{{/* -Which tier-file module becomes which module of the unified single adapter. -`index` is the module's position in that tier file's `modules:` list. -`routing` is the routing file the module's router reads; the provider module -has no router (it answers requests itself). Topology, not environment, so it -is a constant here rather than a value -- the same rule config/routing-*.yaml -follows. -*/}} -{{- define "adapter-service.networkLayer.moduleMap" -}} -- {from: consumer, index: 0, name: consumer, path: /consumer/, routing: routing-network-layer-consumer.yaml} -- {from: network, index: 0, name: network, path: /network/, routing: routing-network.yaml} -- {from: provider, index: 0, name: provider, path: /provider/} -- {from: provider, index: 1, name: provider-publish, path: /provider/publish, routing: routing-network-layer-publish.yaml} -{{- end }} - -{{/* -One tier file, parsed, with its __ADAPTER_*__ placeholders renamed to that -tier's (__CONSUMER_*__, __NETWORK_*__, __PROVIDER_*__) so the init container can -fill each module from its own tier's Secret. -Usage: include "adapter-service.networkLayer.tier" (dict "ctx" $ "tier" "provider") | fromYaml -*/}} -{{- define "adapter-service.networkLayer.tier" -}} -{{- $file := printf "config/%s-config.yaml" .tier -}} -{{- $raw := .ctx.Files.Get $file -}} -{{- if not $raw -}} -{{- fail (printf "%s: role network-layer is generated from %s, which is missing from the chart." .ctx.Chart.Name $file) -}} -{{- end -}} -{{- $doc := $raw | replace "__ADAPTER_" (printf "__%s_" (upper .tier)) | fromYaml -}} -{{- if or (not $doc) (hasKey $doc "Error") -}} -{{- fail (printf "%s: %s does not parse as YAML, so the network-layer config cannot be generated from it." .ctx.Chart.Name $file) -}} -{{- end -}} -{{- toYaml $doc -}} -{{- end }} - -{{/* -The unified single adapter's config, as YAML. - -Process-level settings (log, http, pluginManager, otelsetup) come from the -consumer tier file, as in quick-start; appName, the listener port, the http -timeouts (http.timeout), the log level (logLevel) and telemetry (otel.*) are -set here from values. Each module is the -tier file's module with only its name, path, routingConfig, -extendedSchema_enabled, httpClientConfig and (when registryCache is on) the -cache plugin and registry cacheTTL changed. -*/}} -{{- define "adapter-service.networkLayer.config" -}} -{{- $ctx := . -}} -{{- $nl := .Values.networkLayer -}} -{{- $rc := $nl.registryCache -}} -{{- if and $rc.enabled (not $rc.addr) -}} -{{- fail (printf "%s: networkLayer.registryCache.enabled needs networkLayer.registryCache.addr (host:port of Redis). Without it the registry logs that no cache plugin is configured and caches nothing." $ctx.Chart.Name) -}} -{{- end -}} -{{- $tiers := dict -}} -{{- range $t := list "consumer" "network" "provider" -}} -{{- $_ := set $tiers $t (include "adapter-service.networkLayer.tier" (dict "ctx" $ctx "tier" $t) | fromYaml) -}} -{{- end -}} -{{- $modules := list -}} -{{- range $s := include "adapter-service.networkLayer.moduleMap" . | fromYamlArray -}} -{{- $src := (index $tiers $s.from).modules -}} -{{- if le (len $src) (int $s.index) -}} -{{- fail (printf "%s: network-layer module %s expects config/%s-config.yaml to have a module at index %d; it has %d." $ctx.Chart.Name $s.name $s.from (int $s.index) (len $src)) -}} -{{- end -}} -{{- $m := deepCopy (index $src (int $s.index)) -}} -{{- $_ := set $m "name" $s.name -}} -{{- $_ := set $m "path" $s.path -}} -{{- $p := $m.handler.plugins -}} -{{- if hasKey $p "router" -}} -{{- if not $s.routing -}} -{{- fail (printf "%s: network-layer module %s has a router but the module map names no routing file for it." $ctx.Chart.Name $s.name) -}} -{{- end -}} -{{- $_ := set $p.router.config "routingConfig" (printf "%s/%s" (include "adapter-service.configDir" $ctx) $s.routing) -}} -{{- end -}} -{{- if and (hasKey $p "schemaValidator") $nl.extendedSchemaEnabled -}} -{{- $_ := set $p.schemaValidator.config "extendedSchema_enabled" ($nl.extendedSchemaEnabled | toString) -}} -{{- end -}} -{{- with $nl.httpClient -}} -{{- $_ := set $m.handler "httpClientConfig" (deepCopy .) -}} -{{- end -}} -{{- if $rc.enabled -}} -{{- $_ := set $p "cache" (dict "id" "cache" "config" (dict "addr" $rc.addr "use_tls" ($rc.useTLS | toString))) -}} -{{- if hasKey $p "registry" -}} -{{- $_ := set $p.registry.config "cacheTTL" $rc.ttl -}} -{{- end -}} -{{- end -}} -{{- $modules = append $modules $m -}} -{{- end -}} -{{- $doc := omit (deepCopy $tiers.consumer) "modules" -}} -{{- $_ := set $doc "appName" (include "adapter-service.appName" .) -}} -{{- $_ := set $doc.log "level" .Values.logLevel -}} -{{- $_ := set $doc.http "port" (int .Values.service.targetPort) -}} -{{- with .Values.http.timeout -}} -{{- $_ := set $doc.http "timeout" (dict "read" (int .read) "write" (int .write) "idle" (int .idle)) -}} -{{- end -}} -{{- with (dig "plugins" "otelsetup" "config" dict $doc) -}} -{{- $cfg := . -}} -{{- $on := $ctx.Values.otel.enabled | toString -}} -{{- $_ := set . "serviceName" (include "adapter-service.otelServiceName" $ctx) -}} -{{- $_ := set . "environment" $ctx.Values.otel.environment -}} -{{- $_ := set . "enableMetrics" $on -}} -{{- $_ := set . "enableTracing" $on -}} -{{- $_ := set . "enableLogs" $on -}} -{{- with $ctx.Values.otel.endpoint -}} -{{- $_ := set $cfg "otlpEndpoint" . -}} -{{- end -}} -{{- end -}} -{{- $_ := set $doc "modules" $modules -}} -{{- /* toYaml drops the quotes the tier files put round each placeholder. Put - them back: the init container substitutes the raw Secret value, and an - unquoted value can change type (an all-digit id becomes a number). */ -}} -# GENERATED by adapter-service (templates/_network-layer.tpl) from -# config/{consumer,network,provider}-config.yaml. Edit those, not this. -{{ regexReplaceAll "(?m):[ ]+(__[A-Z][A-Z0-9_]*__)$" (toYaml $doc) ": \"${1}\"" -}} -{{- end }} - {{/* Fails unless every private-key VALUE in a network-layer config is one of the tier placeholders. Parsed, not grepped: it reads each module's keyManager. @@ -152,7 +35,7 @@ Usage: include "adapter-service.networkLayer.checkKeys" (dict "ctx" $ "cfg" $par {{- end }} {{/* -Routing for the unified single adapter, as YAML: {: }. +Routing for the single network layer adapter, as YAML: {: }. routing-network.yaml is the multi-adapter file, unchanged (discover and publish to discovery). The other two are generated from routing-consumer.yaml and diff --git a/charts/adapter-service/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml index 50140264..98085c7e 100644 --- a/charts/adapter-service/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -10,9 +10,9 @@ Everything that differs between the provider, network and consumer adapters is a value: they are one image with one config format, and the role decides the identity, the handler role, the step list and where requests are routed. -role network-layer is the exception to "one file per role": its adapter.yaml and -routing files are generated from the three tier files (_network-layer.tpl), and -it renders more than one routing file. +role network-layer reads config/single-network-layer-config.yaml; its routing +files are generated from the tier routing files (_network-layer.tpl), and it +renders more than one routing file. */}} {{/* RAW CONFIG OVERRIDE. @@ -31,19 +31,21 @@ in `helm get values` -- so the render refuses it. {{- $networkLayer := include "adapter-service.isNetworkLayer" . }} {{- $nlCfg := "" }} {{- if $networkLayer }} -{{- /* role network-layer: the config is GENERATED from the three tier files - (_network-layer.tpl), unless `config` replaces it whole. Either way every - private key must be a tier placeholder; checked on the parsed config. */}} -{{- if .Values.config }} -{{- $nlCfg = .Values.config }} -{{- else }} -{{- $nlCfg = include "adapter-service.networkLayer.config" . }} +{{- /* role network-layer: the config is config/single-network-layer-config.yaml, + unless `config` replaces it whole. Either way every private key must be a + tier placeholder; checked on the parsed config. */}} +{{- $nlCfg = .Values.config | default (.Files.Get "config/single-network-layer-config.yaml") }} +{{- if not $nlCfg }} +{{- fail (printf "%s: role network-layer reads config/single-network-layer-config.yaml, which is missing from the chart." .Chart.Name) }} {{- end }} {{- $nlParsed := $nlCfg | fromYaml }} {{- if or (not $nlParsed) (hasKey $nlParsed "Error") }} -{{- fail (printf "%s: the network-layer adapter config (`config` override or generated) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} +{{- fail (printf "%s: the network-layer adapter config (`config` override or config/single-network-layer-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} {{- end }} {{- include "adapter-service.networkLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} +{{- if ne (toString (dig "http" "port" "" $nlParsed)) (toString .Values.service.targetPort) }} +{{- fail (printf "%s: the network-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} +{{- end }} {{- else }} {{- $cfgFile := printf "config/%s-config.yaml" (include "adapter-service.role" .) }} {{- $cfg := .Values.config | default (.Files.Get $cfgFile) }} diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index f0a8c2b5..295a35a3 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -217,17 +217,6 @@ spec: - name: GOMEMLIMIT value: {{ . | quote }} {{- end }} - {{- if include "adapter-service.isNetworkLayer" . }} - {{- with .Values.networkLayer.registryCache }} - {{- if and .enabled .passwordSecret.name }} - - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .passwordSecret.name }} - key: {{ .passwordSecret.key }} - {{- end }} - {{- end }} - {{- end }} {{- with .Values.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 4c039115..0b469dea 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -262,22 +262,14 @@ becknSpec: # and does not go stale. # --------------------------------------------------------------------------- otel: - # On by default. Set false where no collector runs: an exporter with nothing - # listening logs a failure every interval. - enabled: true - # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317". - # Left empty, the collector named in the tier config files is used. + enabled: false + # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" endpoint: "" environment: dev # Left empty, defaults to oan--adapter, matching OTEL_SERVICE_NAME in # the compose stack so traces from either deployment line up. serviceName: "" -# Read ONLY by role network-layer, whose adapter.yaml is generated. The -# provider, network and consumer files hardcode `log.level: debug`, so this does -# not change them. The same holds for `http.timeout` below and for `otel` -# above: role network-layer writes them into its config, the other roles keep -# their tier file's values. logLevel: info http: @@ -425,31 +417,27 @@ extraEnv: [] envFromSecrets: [] # --------------------------------------------------------------------------- -# role: network-layer -- the unified single adapter. +# role: network-layer -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE process -# (/consumer/, /network/, /provider/, /provider/publish). Its config and routing -# are generated at render time from the three tier files in config/, so there -# is no second copy to keep in step. Tier-to-tier hops are loopback HTTP to -# this pod's own listener (http://localhost:/): any -# adapter image works. +# (/consumer/, /network/, /provider/, /provider/publish). Its adapter config is +# config/single-network-layer-config.yaml, mounted verbatim like any role's +# file: log level, timeouts, telemetry, extended schema and the outbound +# connection pool are set there. The routing is generated at render time from +# the tier routing files, with tier-to-tier hops pointed at this pod's own +# listener (http://localhost:/): any adapter image +# works. # # Ignored by the provider, network and consumer roles. # # NOT USED by this role: `routing.config` (setting it fails the render, because -# the routing is generated) and `handler.*` (role, steps and path come from -# the tier files). `config` replaces the generated adapter.yaml WHOLE: the -# private-key guard still runs on it, but then extendedSchemaEnabled, -# httpClient and registryCache do not apply, it must point its routers at -# /app/config/routing-network-layer-consumer.yaml, routing-network.yaml and -# routing-network-layer-publish.yaml, and `logLevel`, `http.timeout`, `otel` -# and the port are yours to set. +# the routing is generated) and `handler.*` (role, steps and path are in the +# config file). `config` replaces config/single-network-layer-config.yaml +# WHOLE: the private-key guard still runs on it, and it must point its routers +# at /app/config/routing-network-layer-consumer.yaml, routing-network.yaml and +# routing-network-layer-publish.yaml. # --------------------------------------------------------------------------- networkLayer: - # Forces extendedSchema_enabled on every module that validates schemas, as - # the quick-start does. "" keeps each tier file's own setting. - extendedSchemaEnabled: "true" - # Three identities, one per tier: the same Secrets (six keys each, named as # under keys.existingSecret) and the same registry rows the multi-adapter # releases use, so either mode runs against the same registry. They must @@ -465,34 +453,3 @@ networkLayer: # Extra placeholders, as "/". # __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl extraSubstitutions: {} - - # Outbound HTTP pool for every module: discovery, the registry, upstream APIs - # and the loopback hops to this pod's own listener. Go's default keeps 2 idle - # connections per host, so under load most calls open a new connection. - # responseHeaderTimeout bounds the wait for an answer's headers, which on the - # consumer -> provider hop includes the upstream API call: keep it above the - # slowest upstream. - # Read ONLY by role network-layer; the multi tier files keep their own settings. - # To leave the adapter's own defaults (no httpClientConfig at all) set it to - # null: `--set networkLayer.httpClient=null`, or `httpClient: null` in an - # environment values file. `{}` does NOT work: Helm merges maps, so {} keeps - # the four defaults below. - httpClient: - maxIdleConns: 512 - maxIdleConnsPerHost: 128 - idleConnTimeout: 90s - responseHeaderTimeout: 30s - - # Registry lookups cached in Redis. Off: every request asks the registry. - # ttl is how long a suspended participant keeps verifying and a withdrawn - # capability keeps being called -- keep it short. Redis is not deployed by - # this chart. Read ONLY by role network-layer. - registryCache: - enabled: false - addr: "" # host:port, e.g. redis-master.redis.svc.cluster.local:6379 - useTLS: false - ttl: 60s - # Optional. Read by the cache plugin from REDIS_PASSWORD. - passwordSecret: - name: "" - key: password From a977c4fbcc9b4d2fa1cf651c97fda22f2a4edc43 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 16:55:22 +0530 Subject: [PATCH 08/43] feat(adapter-service): add network-layer role with configuration and routing templates --- ...network-layer-values.yaml => single-network-layer-values.yaml} | 0 .../examples/{network-layer.yaml => single-network-layer.yaml} | 0 .../templates/{_network-layer.tpl => _single-network-layer.tpl} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename charts/adapter-service/ci/{network-layer-values.yaml => single-network-layer-values.yaml} (100%) rename charts/adapter-service/examples/{network-layer.yaml => single-network-layer.yaml} (100%) rename charts/adapter-service/templates/{_network-layer.tpl => _single-network-layer.tpl} (100%) diff --git a/charts/adapter-service/ci/network-layer-values.yaml b/charts/adapter-service/ci/single-network-layer-values.yaml similarity index 100% rename from charts/adapter-service/ci/network-layer-values.yaml rename to charts/adapter-service/ci/single-network-layer-values.yaml diff --git a/charts/adapter-service/examples/network-layer.yaml b/charts/adapter-service/examples/single-network-layer.yaml similarity index 100% rename from charts/adapter-service/examples/network-layer.yaml rename to charts/adapter-service/examples/single-network-layer.yaml diff --git a/charts/adapter-service/templates/_network-layer.tpl b/charts/adapter-service/templates/_single-network-layer.tpl similarity index 100% rename from charts/adapter-service/templates/_network-layer.tpl rename to charts/adapter-service/templates/_single-network-layer.tpl From 16117291d107783524ad193ce7cf08ed89b742d0 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 16:58:33 +0530 Subject: [PATCH 09/43] Refactor adapter service to use single-network-layer role - Renamed role from `network-layer` to `single-network-layer` across configuration files and templates. - Updated routing and configuration references to reflect the new single-network-layer structure. - Adjusted Helm chart values and examples to align with the single-network-layer role. - Modified ingress rules to restrict access to appropriate paths for the single-network-layer. - Updated deployment scripts and configuration management to support the new role structure. - Ensured backward compatibility by updating references in the documentation and examples. --- charts/adapter-service/CHANGELOG.md | 12 +++---- charts/adapter-service/Chart.yaml | 4 +-- charts/adapter-service/README.md | 34 +++++++++--------- .../ci/single-network-layer-values.yaml | 10 +++--- .../config/single-network-layer-config.yaml | 20 +++++------ .../examples/single-network-layer.yaml | 35 ++++++++++++------- charts/adapter-service/templates/NOTES.txt | 20 +++++------ charts/adapter-service/templates/_helpers.tpl | 10 +++--- .../templates/_single-network-layer.tpl | 30 ++++++++-------- .../adapter-service/templates/configmap.yaml | 24 ++++++------- .../adapter-service/templates/deployment.yaml | 18 +++++----- charts/adapter-service/templates/ingress.yaml | 23 ++++++------ charts/adapter-service/values.yaml | 16 ++++----- charts/openbao-secrets/CHANGELOG.md | 4 +-- charts/openbao-secrets/values.yaml | 12 +++---- quick-start/README.md | 2 +- quick-start/bin/unified-network-layer-up.sh | 2 +- .../unified-network-layer/overrides.yaml | 5 ++- .../docker-compose.unified-network-layer.yml | 2 +- 19 files changed, 148 insertions(+), 135 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index c04843cb..a876cb80 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -8,19 +8,19 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ## [0.5.3] - 2026-10-09 ### Added -- `role: network-layer`: all three tiers in one adapter (four modules: - `/consumer/`, `/network/`, `/provider/`, `/provider/publish`), as in the - quick-start's network-layer compose stack. Its config is +- `role: single-network-layer`: all three tiers in one adapter (four modules: + `/` (consumer), `/network/`, `/provider/`, `/provider/publish`), as in the + quick-start's single-network-layer compose stack. Its config is `config/single-network-layer-config.yaml` (extended schema, an outbound connection pool and log level info on every module); its routing is generated at render time from `config/routing-*.yaml`. Tier hops are loopback HTTP inside the pod. Signs as three identities from - `networkLayer.keys.{consumer,network,provider}`; `routing.config` fails the + `singleNetworkLayer.keys.{consumer,network,provider}`; `routing.config` fails the render for this role. -- network-layer ingress may expose only `/consumer...` and +- single-network-layer ingress may expose only single-segment `/` and `/provider/`, as literal paths with `pathType` Prefix or Exact and no `use-regex` / `rewrite-target` annotation. Example - `examples/network-layer.yaml`. + `examples/single-network-layer.yaml`. - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index fabc308f..f6390ffc 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: adapter-service description: >- One chart for the OAN Beckn adapters: provider, network and consumer (one - release per tier), or network-layer (all three tiers in one adapter). Same + release per tier), or single-network-layer (all three tiers in one adapter). Same image and config format; role selects the handler role, the step list and the routing target. Stateless: an adapter's identity is a keypair held in a Secret, and everything else it reads from @@ -23,7 +23,7 @@ keywords: - provider - network - consumer - - network-layer + - single-network-layer home: https://github.com/OpenAgriNet/helmcharts sources: - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index e73d77c0..d845c8c9 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -2,7 +2,7 @@ One chart for the Beckn adapters, in two modes. Multi-adapter: three roles, `provider`, `network` and `consumer`, one release per tier. Network layer: -`network-layer`, all three tiers in one adapter. All roles are the same image +`single-network-layer`, all three tiers in one adapter. All roles are the same image and the same config format, so `role` selects the handler role, the step list and where requests are routed. @@ -15,8 +15,8 @@ and where requests are routed. ``` ``` - network-layer (one adapter, one pod): - /consumer/ ──lo──▶ /network/ ──▶ discovery + single-network-layer (one adapter, one pod): + / ──lo──▶ /network/ ──▶ discovery └──────lo────▶ /provider/ ──▶ upstream APIs /provider/publish ──lo──▶ /network/publish ``` @@ -26,7 +26,7 @@ and where requests are routed. | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | | `network` | `bpp` | `validateSign → addRoute → sign` | discovery | | `consumer` | `bap` | `addRoute → sign` | network | -| `network-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | +| `single-network-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | `consumer` is the only one that accepts **unsigned** requests: the consumer app is inside the trust boundary, so there is no network signature to check. @@ -68,7 +68,7 @@ render error rather than a default: | Value | Why there is no default | |---|---| | `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | -| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `network-layer` uses `networkLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | +| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `single-network-layer` uses `singleNetworkLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | | `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | | `role` | It decides the handler role, the step list and the routing target. A default would give one adapter another one's behaviour | | `routing.rules` | An adapter with no route accepts requests and has nowhere to send them | @@ -136,9 +136,9 @@ healthy. | Mode | Releases (release = namespace) | Callers use | |---|---|---| | multi | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | -| network layer | `network-layer-adapter` (examples/network-layer.yaml) | `network-layer-adapter:9200/consumer/`, `network-layer-adapter:9200/provider/publish` | +| network layer | `single-network-layer-adapter` (examples/single-network-layer.yaml) | `single-network-layer-adapter:9200/`, `single-network-layer-adapter:9200/provider/publish` | -`role: network-layer` runs all three tiers in one adapter, as four +`role: single-network-layer` runs all three tiers in one adapter, as four modules of one process. Its config is `config/single-network-layer-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change to a tier file that should apply to both modes goes in both files). Its routing @@ -146,33 +146,33 @@ is **generated** at render time from `config/routing-*.yaml`, so routes stay in one place. Tier hops are loopback HTTP inside the pod (any image). It signs as the same three identities, from the same three Secrets, so the registry is the same in both modes. An environment's `config:` override for a multi role is NOT -seen by network-layer; give the network-layer release its own `config:` if +seen by network-layer; give the single-network-layer release its own `config:` if needed (it replaces `single-network-layer-config.yaml` whole). Pick the mode per environment with ONE switch where releases are chosen. For a helmfile (adapt to infra-automation's layout): ```yaml -# environments: values carry adapters: {singleNetworkAdapter: false} +# environments: values carry adapters: {singleNetworkLayerAdapter: false} releases: {{- range $tier := list "consumer" "network" "provider" }} - name: {{ $tier }}-adapter namespace: {{ $tier }}-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ not $.Values.adapters.singleNetworkAdapter }} + installed: {{ not $.Values.adapters.singleNetworkLayerAdapter }} values: [values/{{ $tier }}-adapter.yaml] {{- end }} - - name: network-layer-adapter - namespace: network-layer-adapter + - name: single-network-layer-adapter + namespace: single-network-layer-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ .Values.adapters.singleNetworkAdapter }} - values: [values/network-layer-adapter.yaml] + installed: {{ .Values.adapters.singleNetworkLayerAdapter }} + values: [values/single-network-layer-adapter.yaml] ``` With Argo CD, gate the four Applications on the same boolean (for example an ApplicationSet list generator, or `{{ if }}` in an app-of-apps chart). -Switching an environment: install `network-layer-adapter` alongside the multi +Switching an environment: install `single-network-layer-adapter` alongside the multi releases (both modes sign with the same identities), repoint the app and the catalogue publisher to the paths above, then remove the three multi releases. Reverse the steps to switch back. @@ -181,8 +181,8 @@ Reverse the steps to switch back. See `values.yaml` — every field is commented with what it does and what breaks without it. `examples/{provider,network,consumer}.yaml` are working dev deployments for the -multi-adapter mode, and `examples/network-layer.yaml` for the network layer; -`ci/` holds the files lint renders, including `ci/network-layer-values.yaml`. +multi-adapter mode, and `examples/single-network-layer.yaml` for the network layer; +`ci/` holds the files lint renders, including `ci/single-network-layer-values.yaml`. ## Related diff --git a/charts/adapter-service/ci/single-network-layer-values.yaml b/charts/adapter-service/ci/single-network-layer-values.yaml index e4a4e00c..829892cb 100644 --- a/charts/adapter-service/ci/single-network-layer-values.yaml +++ b/charts/adapter-service/ci/single-network-layer-values.yaml @@ -1,10 +1,10 @@ -# charts/adapter-service/ci/network-layer-values.yaml -# role network-layer: the single network layer adapter (consumer, network and +# charts/adapter-service/ci/single-network-layer-values.yaml +# role single-network-layer: the single network layer adapter (consumer, network and # provider tiers as modules of one process). Renders # config/single-network-layer-config.yaml, the loopback routing and the three # tier identities. -role: network-layer -fullnameOverride: network-layer-adapter +role: single-network-layer +fullnameOverride: single-network-layer-adapter image: registry: ghcr.io @@ -18,7 +18,7 @@ service: port: 9200 targetPort: 9200 -networkLayer: +singleNetworkLayer: keys: consumer: existingSecret: consumer-adapter-keys diff --git a/charts/adapter-service/config/single-network-layer-config.yaml b/charts/adapter-service/config/single-network-layer-config.yaml index 2ee9fb9a..baf27416 100644 --- a/charts/adapter-service/config/single-network-layer-config.yaml +++ b/charts/adapter-service/config/single-network-layer-config.yaml @@ -1,7 +1,7 @@ -# OAN single network layer adapter (role network-layer). The consumer, network +# OAN single network layer adapter (role single-network-layer). The consumer, network # and provider tiers as four modules of ONE adapter process: # -# /consumer/ the caller (bap); the app posts here, unsigned +# / the consumer (bap); the app posts here, unsigned # /network/ verifies and forwards to discovery # /provider/ answers from the upstream APIs (bpp) # /provider/publish the catalogue system publishes here @@ -12,7 +12,7 @@ # # Differences from the three tier files: # - one listener (9200) and one appName; tier hops are loopback HTTP to this -# same listener (see the routing-network-layer-*.yaml files, which the chart +# same listener (see the routing-single-network-layer-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; @@ -21,9 +21,9 @@ # This file is mounted verbatim. Each module signs as its OWN tier: the # __CONSUMER_*__, __NETWORK_*__ and __PROVIDER_*__ placeholders are substituted # by the init container from that tier's Secret -# (networkLayer.keys..existingSecret) at pod start, so no key is in this +# (singleNetworkLayer.keys..existingSecret) at pod start, so no key is in this # file, in the ConfigMap, or in git. -appName: "network-layer-adapter" +appName: "single-network-layer-adapter" log: level: info @@ -51,7 +51,7 @@ plugins: otelsetup: id: otelsetup config: - serviceName: "oan-network-layer-adapter" + serviceName: "oan-single-network-layer-adapter" environment: "dev" otlpEndpoint: "clickstack-otel-collector.observability.svc.cluster.local:4317" enableMetrics: "true" @@ -117,7 +117,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-network-layer-consumer.yaml + routingConfig: /app/config/routing-single-network-layer-consumer.yaml middleware: - id: reqpreprocessor config: @@ -280,7 +280,7 @@ modules: bharat-vistaar: authScheme: oauth2 # Substituted from the provider Secret (vistaarTokenUrl) via - # networkLayer.extraSubstitutions. + # singleNetworkLayer.extraSubstitutions. tokenUrl: "__VISTAAR_TOKEN_URL__" clientIdEnv: VISTAAR_CLIENT_ID clientSecretEnv: VISTAAR_CLIENT_SECRET @@ -305,7 +305,7 @@ modules: # signed as the provider and routed to /network/publish. No validateSign: # never expose it on an Ingress. - name: provider-publish - path: /network/publish + path: /provider/publish handler: type: std role: bap @@ -353,7 +353,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-network-layer-publish.yaml + routingConfig: /app/config/routing-single-network-layer-publish.yaml middleware: - id: reqpreprocessor config: diff --git a/charts/adapter-service/examples/single-network-layer.yaml b/charts/adapter-service/examples/single-network-layer.yaml index d3779ab2..c8807a15 100644 --- a/charts/adapter-service/examples/single-network-layer.yaml +++ b/charts/adapter-service/examples/single-network-layer.yaml @@ -1,10 +1,10 @@ # =========================================================================== -# network-layer role -- the single network layer adapter. +# single-network-layer role -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: # -# /consumer/ the caller (bap); the app posts here, unsigned +# / the consumer (bap); the app posts here, unsigned # /network/ internal: verifies and forwards to discovery # /provider/ answers from the upstream APIs (bpp) # /provider/publish the catalogue system publishes here @@ -13,18 +13,18 @@ # (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the # other in an environment, not both. # -# helm install network-layer-adapter . -n network-layer-adapter --create-namespace \ -# -f examples/network-layer.yaml +# helm install single-network-layer-adapter . -n single-network-layer-adapter --create-namespace \ +# -f examples/single-network-layer.yaml # # The config is config/single-network-layer-config.yaml; the routing is # generated from config/routing-*.yaml. Tier hops are loopback HTTP inside the # pod, so any adapter image works. # =========================================================================== -role: network-layer +role: single-network-layer # The Service name and DNS name callers use: -# network-layer-adapter.network-layer-adapter.svc.cluster.local:9200 -fullnameOverride: network-layer-adapter +# single-network-layer-adapter.single-network-layer-adapter.svc.cluster.local:9200 +fullnameOverride: single-network-layer-adapter image: registry: ghcr.io @@ -41,7 +41,7 @@ service: port: 9200 targetPort: 9200 -networkLayer: +singleNetworkLayer: # The three tier Secrets the multi-adapter releases use. openbao-secrets # reflects them into this namespace. keys: @@ -101,16 +101,25 @@ topologySpreadConstraints: terminationGracePeriodSeconds: 40 preStopSleepSeconds: 10 # needs Kubernetes 1.30+; 0 on older clusters -# Off here. When on, only these paths are accepted: /, /provider, -# /provider/publish and /network* fail the render. +# Off here. When on, only single-segment / paths (the consumer) and +# /provider/ are accepted: /, /network*, /provider and +# /provider/publish fail the render. ingress: enabled: false className: "" hosts: - - host: network-layer.oan.example.com + - host: single-network-layer.oan.example.com paths: - - path: /consumer - pathType: Prefix + - path: /discover + pathType: Exact + - path: /select + pathType: Exact + - path: /init + pathType: Exact + - path: /confirm + pathType: Exact + - path: /status + pathType: Exact - path: /provider/select pathType: Prefix - path: /provider/init diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index feddc34d..423ef25a 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,6 +1,6 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} -{{ if include "adapter-service.isNetworkLayer" . }} - role network-layer (all three tiers in one adapter: /consumer/ bap, /network/ bpp, /provider/ bpp, /provider/publish bap) +{{ if include "adapter-service.isSingleNetworkLayer" . }} + role single-network-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) hops loopback HTTP between tiers (http://localhost:{{ .Values.service.targetPort }}/) {{- else }} role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) @@ -9,7 +9,7 @@ image {{ include "adapter-service.image" . }} service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} registry {{ .Values.registry.url }} - routes {{ if include "adapter-service.isNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} + routes {{ if include "adapter-service.isSingleNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. @@ -26,10 +26,10 @@ chart's appVersion. TWO THINGS THIS CHART CANNOT CHECK FOR YOU -{{ if include "adapter-service.isNetworkLayer" . -}} +{{ if include "adapter-service.isSingleNetworkLayer" . -}} 1. The Participant rows. This adapter signs as three identities -- consumer, - network and provider -- from Secrets {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, - {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the + network and provider -- from Secrets {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, + {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the multi-adapter releases use. Each must carry its tier's public key. 2. Key rotation. Changing any of those Secrets restarts nothing: @@ -37,7 +37,7 @@ TWO THINGS THIS CHART CANNOT CHECK FOR YOU kubectl -n {{ include "common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} Callers address the tiers by path on this one Service: - app (discover, select, ...) {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/consumer/ + app (discover, select, ...) {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/ catalogue publisher {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/provider/publish /network is an internal tier: never route to it from outside. {{- else -}} @@ -62,11 +62,11 @@ on and nothing listening the exporter logs a failure every interval. {{- if .Values.ingress.enabled }} Ingress is ON. -{{- if include "adapter-service.isNetworkLayer" . }} -This is the network-layer role. /consumer accepts UNSIGNED requests: the consumer tier is +{{- if include "adapter-service.isSingleNetworkLayer" . }} +This is the single-network-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is inside the trust boundary and has no validateSign step, so an Ingress on it exposes an unauthenticated entry point to the network. Rate-limit this host. -Rule: only /consumer and /provider/ may be exposed (the render refuses other paths), +Rule: only single-segment / paths and /provider/ may be exposed (the render refuses other paths), and never /network or /provider/publish: /network is an internal tier, and /provider/publish has no validateSign and signs as the provider. {{- else if eq (include "adapter-service.role" .) "consumer" }} diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl index 7691a581..fc62e6db 100644 --- a/charts/adapter-service/templates/_helpers.tpl +++ b/charts/adapter-service/templates/_helpers.tpl @@ -6,7 +6,7 @@ # identity, upstream and config-rendering wiring an adapter needs. # # One chart, four roles. provider, network and consumer (one release per tier) -# and network-layer (all three tiers in one adapter) are the same image and the +# and single-network-layer (all three tiers in one adapter) are the same image and the # same config format; the role decides the handler role, the step list and # where requests are routed. Install once per role, and keep the RELEASE name # per-role -- the release name is what becomes the Service DNS name that the @@ -15,7 +15,7 @@ */}} {{/* -The adapter's role: provider | network | consumer | network-layer. +The adapter's role: provider | network | consumer | single-network-layer. Required, with no default. A default would silently give one role's step list and handler role to a different adapter -- which renders, starts, reports Ready @@ -24,10 +24,10 @@ and then mis-handles every request, at the far end, in a peer's logs. {{- define "adapter-service.role" -}} {{- $role := .Values.role | default "" -}} {{- if not $role -}} -{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or network-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} +{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or single-network-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} {{- end -}} -{{- if not (has $role (list "provider" "network" "consumer" "network-layer")) -}} -{{- fail (printf "%s: role must be one of provider, network, consumer, network-layer (got %q)." .Chart.Name $role) -}} +{{- if not (has $role (list "provider" "network" "consumer" "single-network-layer")) -}} +{{- fail (printf "%s: role must be one of provider, network, consumer, single-network-layer (got %q)." .Chart.Name $role) -}} {{- end -}} {{- $role -}} {{- end }} diff --git a/charts/adapter-service/templates/_single-network-layer.tpl b/charts/adapter-service/templates/_single-network-layer.tpl index ed71fb2d..478902f5 100644 --- a/charts/adapter-service/templates/_single-network-layer.tpl +++ b/charts/adapter-service/templates/_single-network-layer.tpl @@ -1,6 +1,6 @@ {{/* # ============================================================================ -# role: network-layer -- the single network layer adapter. +# role: single-network-layer -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE adapter # process. The adapter config is config/single-network-layer-config.yaml @@ -13,22 +13,22 @@ {{/* "true" when this release runs the single network layer adapter, else empty. */}} -{{- define "adapter-service.isNetworkLayer" -}} -{{- if eq (include "adapter-service.role" .) "network-layer" -}}true{{- end -}} +{{- define "adapter-service.isSingleNetworkLayer" -}} +{{- if eq (include "adapter-service.role" .) "single-network-layer" -}}true{{- end -}} {{- end }} {{/* -Fails unless every private-key VALUE in a network-layer config is one of the +Fails unless every private-key VALUE in a single-network-layer config is one of the tier placeholders. Parsed, not grepped: it reads each module's keyManager. -Usage: include "adapter-service.networkLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) +Usage: include "adapter-service.singleNetworkLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) */}} -{{- define "adapter-service.networkLayer.checkKeys" -}} +{{- define "adapter-service.singleNetworkLayer.checkKeys" -}} {{- range $m := .cfg.modules -}} {{- $km := dig "handler" "plugins" "keyManager" "config" dict $m -}} {{- range $field := list "signingPrivateKey" "encrPrivateKey" -}} {{- $v := index $km $field | default "" | toString -}} {{- if and $v (not (regexMatch "^__(CONSUMER|NETWORK|PROVIDER)_(SIGNING|ENCR)_PRIVATE__$" $v)) -}} -{{- fail (printf "%s: network-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} +{{- fail (printf "%s: single-network-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} {{- end -}} {{- end -}} {{- end -}} @@ -47,25 +47,25 @@ kept as it is, and a rule with no target is left alone (the same nil guard as configmap.yaml). So the two modes are one routing source, and splitting the modules across pods again changes only the hosts. */}} -{{- define "adapter-service.networkLayer.routing" -}} +{{- define "adapter-service.singleNetworkLayer.routing" -}} {{- $ctx := . -}} {{- $loopback := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} {{- /* Whole URL, anchored both ends: only a tier host (optionally with a namespace suffix and port) followed by nothing or a path is rewritten. */ -}} {{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} {{- $out := dict -}} -{{- range $pair := list (list "routing-consumer.yaml" "routing-network-layer-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-network-layer-publish.yaml") -}} +{{- range $pair := list (list "routing-consumer.yaml" "routing-single-network-layer-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-single-network-layer-publish.yaml") -}} {{- $src := index $pair 0 -}} {{- $doc := $ctx.Files.Get (printf "config/%s" $src) | fromYaml -}} {{- if not $doc.routingRules -}} -{{- fail (printf "%s: config/%s parsed to no routingRules; the network-layer routing is generated from it." $ctx.Chart.Name $src) -}} +{{- fail (printf "%s: config/%s parsed to no routingRules; the single-network-layer routing is generated from it." $ctx.Chart.Name $src) -}} {{- end -}} {{- range $i, $rule := $doc.routingRules -}} {{- if $rule.target -}} {{- $url := $rule.target.url | default "" -}} {{- $url = regexReplaceAll $tierHost $url $loopback -}} {{- if hasSuffix "/" $url -}} -{{- fail (printf "%s: network-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} +{{- fail (printf "%s: single-network-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} {{- end -}} {{- $_ := set $rule.target "url" $url -}} {{- end -}} @@ -78,12 +78,12 @@ modules across pods again changes only the hosts. {{/* Secret holding one tier's keypair. Fails naming the tier: a missing identity would otherwise start, report Ready, and fail every signature that tier makes. -Usage: include "adapter-service.networkLayer.tierSecret" (dict "ctx" $ "tier" "network") +Usage: include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" $ "tier" "network") */}} -{{- define "adapter-service.networkLayer.tierSecret" -}} -{{- $name := (index .ctx.Values.networkLayer.keys .tier).existingSecret | default "" -}} +{{- define "adapter-service.singleNetworkLayer.tierSecret" -}} +{{- $name := (index .ctx.Values.singleNetworkLayer.keys .tier).existingSecret | default "" -}} {{- if not $name -}} -{{- fail (printf "%s: networkLayer.keys.%s.existingSecret is required -- the %s tier signs as its own identity, from its own Secret (the six keys under keys.existingSecret)." .ctx.Chart.Name .tier .tier) -}} +{{- fail (printf "%s: singleNetworkLayer.keys.%s.existingSecret is required -- the %s tier signs as its own identity, from its own Secret (the six keys under keys.existingSecret)." .ctx.Chart.Name .tier .tier) -}} {{- end -}} {{- $name -}} {{- end }} diff --git a/charts/adapter-service/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml index 98085c7e..1ad09b01 100644 --- a/charts/adapter-service/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -10,8 +10,8 @@ Everything that differs between the provider, network and consumer adapters is a value: they are one image with one config format, and the role decides the identity, the handler role, the step list and where requests are routed. -role network-layer reads config/single-network-layer-config.yaml; its routing -files are generated from the tier routing files (_network-layer.tpl), and it +role single-network-layer reads config/single-network-layer-config.yaml; its routing +files are generated from the tier routing files (_single-network-layer.tpl), and it renders more than one routing file. */}} {{/* @@ -28,23 +28,23 @@ at pod start, which is why no rendered config ever holds a key. A raw config with a literal private key in it would put that key in git, in a ConfigMap, and in `helm get values` -- so the render refuses it. */}} -{{- $networkLayer := include "adapter-service.isNetworkLayer" . }} +{{- $singleNetworkLayer := include "adapter-service.isSingleNetworkLayer" . }} {{- $nlCfg := "" }} -{{- if $networkLayer }} -{{- /* role network-layer: the config is config/single-network-layer-config.yaml, +{{- if $singleNetworkLayer }} +{{- /* role single-network-layer: the config is config/single-network-layer-config.yaml, unless `config` replaces it whole. Either way every private key must be a tier placeholder; checked on the parsed config. */}} {{- $nlCfg = .Values.config | default (.Files.Get "config/single-network-layer-config.yaml") }} {{- if not $nlCfg }} -{{- fail (printf "%s: role network-layer reads config/single-network-layer-config.yaml, which is missing from the chart." .Chart.Name) }} +{{- fail (printf "%s: role single-network-layer reads config/single-network-layer-config.yaml, which is missing from the chart." .Chart.Name) }} {{- end }} {{- $nlParsed := $nlCfg | fromYaml }} {{- if or (not $nlParsed) (hasKey $nlParsed "Error") }} -{{- fail (printf "%s: the network-layer adapter config (`config` override or config/single-network-layer-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} +{{- fail (printf "%s: the single-network-layer adapter config (`config` override or config/single-network-layer-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} {{- end }} -{{- include "adapter-service.networkLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} +{{- include "adapter-service.singleNetworkLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} {{- if ne (toString (dig "http" "port" "" $nlParsed)) (toString .Values.service.targetPort) }} -{{- fail (printf "%s: the network-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} +{{- fail (printf "%s: the single-network-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} {{- end }} {{- else }} {{- $cfgFile := printf "config/%s-config.yaml" (include "adapter-service.role" .) }} @@ -68,13 +68,13 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - {{- if $networkLayer }} + {{- if $singleNetworkLayer }} adapter.yaml: | {{- $nlCfg | nindent 4 }} {{- if .Values.routing.config }} - {{- fail (printf "%s: routing.config is not used by the network-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} + {{- fail (printf "%s: routing.config is not used by the single-network-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} {{- end }} - {{- range $file, $doc := include "adapter-service.networkLayer.routing" . | fromYaml }} + {{- range $file, $doc := include "adapter-service.singleNetworkLayer.routing" . | fromYaml }} {{ $file }}: | # GENERATED by adapter-service from config/routing-*.yaml. Do not edit; edit those files. {{- toYaml $doc | nindent 4 }} diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index 295a35a3..1e5fa7b1 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -86,8 +86,8 @@ spec: - -c - | set -eu - {{- if include "adapter-service.isNetworkLayer" . }} - # network-layer: adapter.yaml plus one routing file per routed module. + {{- if include "adapter-service.isSingleNetworkLayer" . }} + # single-network-layer: adapter.yaml plus one routing file per routed module. cp /config-template/*.yaml /rendered/ {{- else }} cp /config-template/adapter.yaml /rendered/adapter.yaml @@ -97,7 +97,7 @@ spec: subst() { # $1 placeholder, $2 file holding the value if [ ! -s "$2" ]; then - {{- if include "adapter-service.isNetworkLayer" . }} + {{- if include "adapter-service.isSingleNetworkLayer" . }} # three tier Secrets share key names: name the tier too (e.g. network/keyId) echo "render-config: ${2#/keys/} is missing or empty in its tier's keys Secret" >&2 {{- else }} @@ -111,7 +111,7 @@ spec: # -- the sentinel contains a colon -- or an adapter signing with an # identity nothing can verify. if head -c 9 "$2" | grep -q "<&2 {{- else }} echo "render-config: $(basename "$2") is still unresolved." >&2 @@ -123,7 +123,7 @@ spec: fi sed -i "s|$1|$(cat "$2")|g" /rendered/adapter.yaml } - {{- if include "adapter-service.isNetworkLayer" . }} + {{- if include "adapter-service.isSingleNetworkLayer" . }} {{- /* Three identities: each tier's module signs as that tier, from that tier's Secret mounted at /keys/. */}} @@ -137,7 +137,7 @@ spec: subst __{{ $T }}_ENCR_PRIVATE__ /keys/{{ $tier }}/{{ $k.encrPrivateKey }} subst __{{ $T }}_ENCR_PUBLIC__ /keys/{{ $tier }}/{{ $k.encrPublicKey }} {{- end }} - {{- range $ph, $path := .Values.networkLayer.extraSubstitutions }} + {{- range $ph, $path := .Values.singleNetworkLayer.extraSubstitutions }} subst {{ $ph }} /keys/{{ $path }} {{- end }} {{- else }} @@ -170,7 +170,7 @@ spec: - name: config-template mountPath: /config-template readOnly: true - {{- if include "adapter-service.isNetworkLayer" . }} + {{- if include "adapter-service.isSingleNetworkLayer" . }} {{- range $tier := list "consumer" "network" "provider" }} - name: keys-{{ $tier }} mountPath: /keys/{{ $tier }} @@ -243,11 +243,11 @@ spec: - name: config-template configMap: name: {{ include "adapter-service.fullname" . }}-config - {{- if include "adapter-service.isNetworkLayer" . }} + {{- if include "adapter-service.isSingleNetworkLayer" . }} {{- range $tier := list "consumer" "network" "provider" }} - name: keys-{{ $tier }} secret: - secretName: {{ include "adapter-service.networkLayer.tierSecret" (dict "ctx" $ "tier" $tier) }} + secretName: {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" $ "tier" $tier) }} defaultMode: 0400 {{- end }} {{- else }} diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index 68dd7ad8..ac9cb28e 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,34 +1,35 @@ {{- if .Values.ingress.enabled }} -{{- if include "adapter-service.isNetworkLayer" . }} -{{- /* role network-layer: only /consumer... and individual /provider/ - paths may be public. "/" or "/provider" with Prefix would also expose - /network (an internal tier) and /provider/publish, which has no - validateSign and signs as the provider. +{{- if include "adapter-service.isSingleNetworkLayer" . }} +{{- /* role single-network-layer: only single-segment / paths (the + consumer module is mounted at /) and individual /provider/ paths + may be public. "/" or "/provider" with Prefix would also expose /network + (an internal tier) and /provider/publish, which has no validateSign and + signs as the provider. The path check holds only for literal prefix matching, so pathType must be Prefix or Exact (ImplementationSpecific is a plain string prefix on ingress-nginx: /provider/p would match /provider/publish), the path must be literal, and regex or rewrite annotations are refused. */}} {{- $ann := merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict) }} {{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString) "true" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the single-network-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} {{- end }} {{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the single-network-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} {{- end }} {{- range .Values.ingress.hosts }} {{- range .paths }} {{- $p := .path | default "" }} {{- $pt := .pathType | default "Prefix" }} {{- if not (has $pt (list "Prefix" "Exact")) }} -{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} +{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the single-network-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} {{- end }} {{- if or (not (regexMatch "^/[A-Za-z0-9/_.-]*$" $p)) (contains ".." $p) (contains "//" $p) }} -{{- fail (printf "%s: ingress path %q is not allowed for the network-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} +{{- fail (printf "%s: ingress path %q is not allowed for the single-network-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} {{- end }} -{{- $consumer := or (eq $p "/consumer") (hasPrefix "/consumer/" $p) }} +{{- $consumer := and (regexMatch "^/[A-Za-z0-9_.-]+$" $p) (not (has $p (list "/network" "/provider"))) }} {{- $provider := and (hasPrefix "/provider/" $p) (ne $p "/provider/") (not (hasPrefix "/provider/publish" $p)) }} {{- if not (or $consumer $provider) }} -{{- fail (printf "%s: ingress path %q is not allowed for the network-layer role. Allowed: /consumer... and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} +{{- fail (printf "%s: ingress path %q is not allowed for the single-network-layer role. Allowed: single-segment / (the consumer, e.g. /discover) and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} {{- end }} {{- end }} {{- end }} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 0b469dea..7192650a 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -2,7 +2,7 @@ # adapter-service # # One chart for the Beckn adapters: provider, network and consumer (one release -# per tier), or network-layer (all three tiers in one adapter). Same image and +# per tier), or single-network-layer (all three tiers in one adapter). Same image and # config format; `role` below decides the handler role, the step list and # where requests are routed. # @@ -14,7 +14,7 @@ # # OR, all three tiers in one adapter (one release instead of the three above): # -# helm install network-layer-adapter . -f examples/network-layer.yaml +# helm install single-network-layer-adapter . -f examples/single-network-layer.yaml # # The release name is what the Service is called, which is the DNS name the # other adapters address. Rename a release and you must update whatever routes @@ -30,7 +30,7 @@ # =========================================================================== # --------------------------------------------------------------------------- -# Role: provider | network | consumer | network-layer +# Role: provider | network | consumer | single-network-layer # # Required, with no default. A default would hand one role's handler role and # step list to a different adapter, which renders, starts, reports Ready and @@ -417,10 +417,10 @@ extraEnv: [] envFromSecrets: [] # --------------------------------------------------------------------------- -# role: network-layer -- the single network layer adapter. +# role: single-network-layer -- the single network layer adapter. # # The consumer, network and provider tiers as four modules of ONE process -# (/consumer/, /network/, /provider/, /provider/publish). Its adapter config is +# (/ for the consumer, /network/, /provider/, /provider/publish). Its adapter config is # config/single-network-layer-config.yaml, mounted verbatim like any role's # file: log level, timeouts, telemetry, extended schema and the outbound # connection pool are set there. The routing is generated at render time from @@ -434,10 +434,10 @@ envFromSecrets: [] # the routing is generated) and `handler.*` (role, steps and path are in the # config file). `config` replaces config/single-network-layer-config.yaml # WHOLE: the private-key guard still runs on it, and it must point its routers -# at /app/config/routing-network-layer-consumer.yaml, routing-network.yaml and -# routing-network-layer-publish.yaml. +# at /app/config/routing-single-network-layer-consumer.yaml, routing-network.yaml and +# routing-single-network-layer-publish.yaml. # --------------------------------------------------------------------------- -networkLayer: +singleNetworkLayer: # Three identities, one per tier: the same Secrets (six keys each, named as # under keys.existingSecret) and the same registry rows the multi-adapter # releases use, so either mode runs against the same registry. They must diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index d25114c2..49291130 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.0] - 2026-10-09 ### Added -- `network-layer-adapter` in the namespaces of `consumer-adapter-keys`, +- `single-network-layer-adapter` in the namespaces of `consumer-adapter-keys`, `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for - `adapter-service` role `network-layer` (all three tiers in one adapter, three + `adapter-service` role `single-network-layer` (all three tiers in one adapter, three identities). Inert where that namespace does not exist. ## [0.1.0] - 2026-09-27 diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index 6964dcf7..3bdd9b13 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -57,16 +57,16 @@ secrets: # -- adapter identities. keyId inside these is filled in by # resolve-key-ids.sh AFTER the registry has been seeded, so these entries # change once per environment after the first bring-up. - # network-layer-adapter (role network-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. + # single-network-layer-adapter (role single-network-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. - name: consumer-adapter-keys - namespaces: [consumer-adapter, network-layer-adapter] + namespaces: [consumer-adapter, single-network-layer-adapter] - name: network-adapter-keys - namespaces: [network-adapter, network-layer-adapter] + namespaces: [network-adapter, single-network-layer-adapter] - name: provider-adapter-keys - namespaces: [provider-adapter, network-layer-adapter] + namespaces: [provider-adapter, single-network-layer-adapter] # The provider adapter's upstream credentials, named as environment variables - # by its config rather than substituted into it. network-layer-adapter runs + # by its config rather than substituted into it. single-network-layer-adapter runs # the provider tier too, so it reads them as well. - name: provider-upstream - namespaces: [provider-adapter, network-layer-adapter] + namespaces: [provider-adapter, single-network-layer-adapter] diff --git a/quick-start/README.md b/quick-start/README.md index 6d276142..7f504f0f 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -214,7 +214,7 @@ Each module keeps its own tier identity. | Module | Path | From | Routing file | Routes | |---|---|---|---|---| -| `consumer` | `/consumer/` | `consumer.yaml.tmpl` | `unified-network-layer/single-network-layer-consumer-routing.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | +| `consumer` | `/` | `consumer.yaml.tmpl` | `unified-network-layer/single-network-layer-consumer-routing.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | | `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (multi file, reused) | discover, publish to discovery | | `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | | `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `unified-network-layer/single-network-layer-publish-routing.yaml` | to `localhost:9200/network/publish` | diff --git a/quick-start/bin/unified-network-layer-up.sh b/quick-start/bin/unified-network-layer-up.sh index 719eeba8..c53148ec 100755 --- a/quick-start/bin/unified-network-layer-up.sh +++ b/quick-start/bin/unified-network-layer-up.sh @@ -64,7 +64,7 @@ env = {} for line in (root / ".env").read_text().splitlines(): if "=" in line and not line.lstrip().startswith("#"): k, v = line.split("=", 1); env[k.strip()] = v.strip() -steps = (("publish", "provider/publish"), ("discover", "consumer/discover"), ("select", "consumer/select")) +steps = (("publish", "provider/publish"), ("discover", "discover"), ("select", "select")) rc = 0 for step, path in steps: f = next((root / "capability-examples" / "WeatherObservation" / step).glob("*.json")) diff --git a/quick-start/config/adapters/unified-network-layer/overrides.yaml b/quick-start/config/adapters/unified-network-layer/overrides.yaml index 02392862..3f01ddcc 100644 --- a/quick-start/config/adapters/unified-network-layer/overrides.yaml +++ b/quick-start/config/adapters/unified-network-layer/overrides.yaml @@ -31,7 +31,10 @@ modules: - from: consumer index: 0 name: consumer - path: /consumer/ + # Mounted at /, so the app calls /discover, /select, ... as it does on the + # multi-adapter consumer-adapter. The more specific /network/, /provider/ + # and /provider/publish paths still go to their own modules. + path: / routingConfig: /app/config/single-network-layer-consumer-routing.yaml - from: network index: 0 diff --git a/quick-start/docker-compose.unified-network-layer.yml b/quick-start/docker-compose.unified-network-layer.yml index 0e45c9a9..52a4679c 100644 --- a/quick-start/docker-compose.unified-network-layer.yml +++ b/quick-start/docker-compose.unified-network-layer.yml @@ -610,7 +610,7 @@ services: # # :9200 inside and on 127.0.0.1 (ADAPTER_IMAGE, same as the multi-adapter stack). # The multi-adapter provider-adapter uses 9200 too: run one stack at a time. - # /consumer/* the caller's entry (bap), takes unsigned requests + # / the consumer, the caller's entry (bap), takes unsigned requests # /network/* hands publish on to discovery # /provider/* calls the upstream, answers synchronously # /provider/publish sends the provider's catalogue to /network/publish From b42f82d12facb018327f19560844914634f6a725 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Fri, 9 Oct 2026 17:26:01 +0530 Subject: [PATCH 10/43] Refactor code structure for improved readability and maintainability --- quick-start/.gitignore | 8 +- quick-start/Makefile | 6 +- quick-start/README.md | 34 +- quick-start/bin/bootstrap-ubuntu.sh | 4 +- quick-start/bin/setup.py | 74 +--- ...layer-up.sh => single-network-layer-up.sh} | 34 +- quick-start/bin/stack.sh | 64 ++-- ...outing-single-network-layer-consumer.yaml} | 6 +- ...routing-single-network-layer-publish.yaml} | 4 +- .../adapters/single-network-layer.yaml.tmpl | 356 ++++++++++++++++++ .../unified-network-layer/overrides.yaml | 54 --- ...> docker-compose.single-network-layer.yml} | 53 ++- 12 files changed, 475 insertions(+), 222 deletions(-) rename quick-start/bin/{unified-network-layer-up.sh => single-network-layer-up.sh} (74%) rename quick-start/config/adapters/{unified-network-layer/single-network-layer-consumer-routing.yaml => routing-single-network-layer-consumer.yaml} (81%) rename quick-start/config/adapters/{unified-network-layer/single-network-layer-publish-routing.yaml => routing-single-network-layer-publish.yaml} (83%) create mode 100644 quick-start/config/adapters/single-network-layer.yaml.tmpl delete mode 100644 quick-start/config/adapters/unified-network-layer/overrides.yaml rename quick-start/{docker-compose.unified-network-layer.yml => docker-compose.single-network-layer.yml} (95%) diff --git a/quick-start/.gitignore b/quick-start/.gitignore index 70bdeef9..cbb83f8e 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -16,9 +16,9 @@ config/adapters/experience.yaml config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml -config/adapters/unified-network-layer/unified-network-layer.yaml -bin/demo-unified-network-layer.sh -bin/bench-unified-network-layer.sh +config/adapters/single-network-layer.yaml +bin/demo-single-network-layer.sh +bin/bench-single-network-layer.sh poc-flow.py bin/poc # Apple-silicon-only override that forces amd64 emulation for images with no @@ -42,5 +42,5 @@ config/discovery/instance.yaml # nothing here to ignore -- and nothing here to review either. Back those # volumes up; they are the only copy. -# bin/bench-unified-network-layer.sh scratch: generated bodies and results +# bin/bench-single-network-layer.sh scratch: generated bodies and results .bench/ \ No newline at end of file diff --git a/quick-start/Makefile b/quick-start/Makefile index e68de4e0..304d7e7e 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core up-unified-network-layer down-unified-network-layer down destroy setup reverse-proxy observability pull restart restart-edge ps logs +.PHONY: help up up-core up-single-network-layer down-single-network-layer down destroy setup reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -25,8 +25,8 @@ help: up: ; @$(STACK) up up-core: ; @$(STACK) up-core down: ; @$(STACK) down -up-unified-network-layer: ; @$(STACK) up-unified-network-layer -down-unified-network-layer: ; @$(STACK) down-unified-network-layer +up-single-network-layer: ; @$(STACK) up-single-network-layer +down-single-network-layer: ; @$(STACK) down-single-network-layer destroy: ; @$(STACK) destroy # --------------------------------------------- the optional tiers, on their own diff --git a/quick-start/README.md b/quick-start/README.md index 7f504f0f..6891726c 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -55,7 +55,6 @@ below follow startup order, so they work top to bottom. - `git` - Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` - `python3` with `cryptography` — `pip install cryptography` -- `PyYAML` — `pip install pyyaml` (needed for the unified network layer only) - `curl` ## Step 2 — Get the repo @@ -190,41 +189,42 @@ collection. → Appendix N. --- -## Unified network layer (optional) +## Single network layer (optional) The same network can run two ways. Both use the same registry, keys and data: -| | Multi-adapter (default) | Unified network layer | +| | Multi-adapter (default) | Single network layer | |---|---|---| -| What runs | `consumer-adapter`, `network-adapter`, `provider-adapter` | one `network-layer-adapter` (the unified single adapter) | -| Compose file | `docker-compose.yml` | `docker-compose.unified-network-layer.yml` | -| Start / stop | `make up` / `make down` | `make up-unified-network-layer` / `make down-unified-network-layer` | +| What runs | `consumer-adapter`, `network-adapter`, `provider-adapter` | one `single-network-layer-adapter` (the single network layer adapter) | +| Compose file | `docker-compose.yml` | `docker-compose.single-network-layer.yml` | +| Start / stop | `make up` / `make down` | `make up-single-network-layer` / `make down-single-network-layer` | | Tier-to-tier hops | HTTP between containers | HTTP over loopback (`http://localhost:9200`) inside the one container | | Port | 9202 / 9201 / 9200 | `127.0.0.1:9200` (so only one mode runs at a time) | Use one or the other, not both: they share the project name and data volumes. Both use `ADAPTER_IMAGE` from `.env`; nothing else needs setting. -**Config.** There is no hand-written config for this mode. `bin/setup.py` builds -`config/adapters/unified-network-layer/unified-network-layer.yaml` from the three -tier templates plus `unified-network-layer/overrides.yaml`. That file holds only -the module names and paths, each module's routing file, extended schema, -appName and port. A change to a tier template reaches both modes on the next `setup.py`. -Each module keeps its own tier identity. +**Config.** `config/adapters/single-network-layer.yaml.tmpl`, beside the three +tier templates. `bin/setup.py` fills it into `config/adapters/single-network-layer.yaml` +like the others. Each module is its tier's module from the tier template (a +change to a tier template that should apply here goes in this file too), plus +extended schema validation, an outbound connection pool and log level info. +Each module keeps its own tier identity. The same layout as the Helm chart's +`config/single-network-layer-config.yaml`. | Module | Path | From | Routing file | Routes | |---|---|---|---|---| -| `consumer` | `/` | `consumer.yaml.tmpl` | `unified-network-layer/single-network-layer-consumer-routing.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | +| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-single-network-layer-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | | `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (multi file, reused) | discover, publish to discovery | | `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | -| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `unified-network-layer/single-network-layer-publish-routing.yaml` | to `localhost:9200/network/publish` | +| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-single-network-layer-publish.yaml` | to `localhost:9200/network/publish` | The hops between modules are loopback HTTP calls to the same container. In the multi-adapter stack only the hosts differ. 9200 is the container's own listener -and must match `port` in `overrides.yaml`. +and must match `http.port` in `single-network-layer.yaml.tmpl`. Tools: -- `bin/unified-network-layer-up.sh`: start and smoke-test (publish, discover, select). +- `bin/single-network-layer-up.sh`: start and smoke-test (publish, discover, select). # Part 2 — Run it on a VM @@ -238,7 +238,7 @@ Nothing is cloned yet, so fetch the script rather than running it from the repo: curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash ``` -Installs `git`, `make`, `python3-cryptography`, `python3-yaml` and Docker from Docker's own apt +Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt repo, then adds you to the `docker` group — **log out and back in** for that to take effect. Idempotent, and it deliberately does not clone, write `.env` or start anything. diff --git a/quick-start/bin/bootstrap-ubuntu.sh b/quick-start/bin/bootstrap-ubuntu.sh index 37afcca4..519c6bf0 100755 --- a/quick-start/bin/bootstrap-ubuntu.sh +++ b/quick-start/bin/bootstrap-ubuntu.sh @@ -43,8 +43,8 @@ sudo apt-get update -qq # python as externally-managed (PEP 668), so `pip install cryptography` refuses # without --break-system-packages. The apt build is the same library. sudo apt-get install -y -qq \ - ca-certificates curl gnupg git make python3 python3-cryptography python3-yaml -info "git, make, python3, python3-cryptography, python3-yaml" + ca-certificates curl gnupg git make python3 python3-cryptography +info "git, make, python3, python3-cryptography" # --------------------------------------------------------------- docker diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 11c3ae8b..dad0fdbc 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -637,68 +637,20 @@ def _render_one(name, template, substitutions): _write(f"{name}.yaml", _fill(name, template, substitutions)) -# ----------------------------------------------------- unified network layer +# ------------------------------------------------------ single network layer # -# The unified network layer runs the consumer, network and provider tiers as -# modules of ONE adapter (the unified single adapter). Its config is not a -# template of its own: it is BUILT from the three tier templates above, plus -# the few differences in config/adapters/unified-network-layer/overrides.yaml -# (module names and paths, each module's routing file, extended schema, -# appName, port). A change to a tier template therefore reaches both stacks on the next -# run, and there is no second copy of any module to drift. -# -# Each module keeps its own tier identity, so both stacks use the same keys and -# registry rows. -UNIFIED_NETWORK_LAYER = ADAPTERS / "unified-network-layer" - - -def _render_unified_network_layer(identities, shared): - try: - import yaml # only this mode needs it; the multi-adapter stack does not - except ImportError: - hint = "needs PyYAML (pip install pyyaml, or apt install python3-yaml)" - if os.environ.get("UNIFIED_NETWORK_LAYER") == "1": - sys.exit("setup: cannot render config/adapters/unified-network-layer/" - "unified-network-layer.yaml: " + hint) - print(" config/adapters/unified-network-layer/unified-network-layer.yaml" - " SKIPPED: " + hint) - return - overrides = yaml.safe_load((UNIFIED_NETWORK_LAYER / "overrides.yaml").read_text()) - tiers = {} +# The single network layer runs the consumer, network and provider tiers as +# modules of ONE adapter (docker-compose.single-network-layer.yml). Its +# template, single-network-layer.yaml.tmpl, carries all three tiers' +# placeholders: each module keeps its own tier identity, so both stacks use the +# same keys and registry rows. +def _render_single_network_layer(identities, shared): + substitutions = shared for role in ROLES: - text = _fill(role, (ADAPTERS / f"{role}.yaml.tmpl").read_text(), - _role_substitutions(role, identities[role]) + shared) - tiers[role] = yaml.safe_load(text) - - modules = [] - for spec in overrides["modules"]: - module = tiers[spec["from"]]["modules"][spec["index"]] - module["name"], module["path"] = spec["name"], spec["path"] - plugins = module["handler"].get("plugins", {}) - if "router" in plugins: - if "routingConfig" not in spec: - sys.exit(f"setup: overrides.yaml module {spec['name']} has a router but no routingConfig") - plugins["router"]["config"]["routingConfig"] = spec["routingConfig"] - if "schemaValidator" in plugins: - plugins["schemaValidator"]["config"]["extendedSchema_enabled"] = \ - overrides["extendedSchemaEnabled"] - modules.append(module) - - # Process-level settings come from the consumer template; only appName, - # the port and the telemetry service name change. - doc = {k: v for k, v in tiers["consumer"].items() if k != "modules"} - doc["appName"] = overrides["appName"] - doc["http"]["port"] = overrides["port"] - otel = doc.get("plugins", {}).get("otelsetup") - if otel: - otel["config"]["serviceName"] = overrides["otelServiceName"] - doc["modules"] = modules - - header = ("# GENERATED by bin/setup.py from consumer/network/provider.yaml.tmpl\n" - "# and unified-network-layer/overrides.yaml. Do not edit: edit those and\n" - "# re-run setup. Holds private keys (mode 600, gitignored).\n") - _write("unified-network-layer/unified-network-layer.yaml", - header + yaml.safe_dump(doc, sort_keys=False, width=1000)) + substitutions = _role_substitutions(role, identities[role]) + substitutions + _render_one("single-network-layer", + (ADAPTERS / "single-network-layer.yaml.tmpl").read_text(), + substitutions) def render(identities): @@ -708,7 +660,7 @@ def render(identities): template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() _render_one(role, template, _role_substitutions(role, identities[role]) + shared) - _render_unified_network_layer(identities, shared) + _render_single_network_layer(identities, shared) if __name__ == "__main__": diff --git a/quick-start/bin/unified-network-layer-up.sh b/quick-start/bin/single-network-layer-up.sh similarity index 74% rename from quick-start/bin/unified-network-layer-up.sh rename to quick-start/bin/single-network-layer-up.sh index c53148ec..b34b6aeb 100755 --- a/quick-start/bin/unified-network-layer-up.sh +++ b/quick-start/bin/single-network-layer-up.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash -# Start the unified network layer -- ONE adapter for the consumer, network and -# provider tiers (the unified single adapter), tier hops over loopback HTTP -- +# Start the single network layer -- ONE adapter for the consumer, network and +# provider tiers (the single network layer adapter), tier hops over loopback HTTP -- # then prove it works. # -# bin/unified-network-layer-up.sh +# bin/single-network-layer-up.sh # # Image: ADAPTER_IMAGE from .env (the same as the multi-adapter stack). Port 9200. -# Uses docker-compose.unified-network-layer.yml (and docker-compose.arm64.yml -# when present). Leaves the unified network layer RUNNING. -# Stop it: make down-unified-network-layer +# Uses docker-compose.single-network-layer.yml (and docker-compose.arm64.yml +# when present). Leaves the single network layer RUNNING. +# Stop it: make down-single-network-layer set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" -COMPOSE=(-f docker-compose.unified-network-layer.yml) +COMPOSE=(-f docker-compose.single-network-layer.yml) [ -f docker-compose.arm64.yml ] && COMPOSE+=(-f docker-compose.arm64.yml) envv() { grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-; } PORT=9200 @@ -32,31 +32,31 @@ python3 -c 'import yaml' >/dev/null 2>&1 \ || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" # --- 1. infrastructure ------------------------------------------------------ -step "1/4 registry, discovery, mocks (the mocks are for this script's local smoke test; make up-unified-network-layer does not start them)" +step "1/4 registry, discovery, mocks (the mocks are for this script's local smoke test; make up-single-network-layer does not start them)" docker compose "${COMPOSE[@]}" up -d sunbird-registry-service discovery-service mock-imd mock-agmarknet # --- 2. config -------------------------------------------------------------- step "2/4 keys, registry entries, rendered config (bin/setup.py)" -UNIFIED_NETWORK_LAYER=1 python3 bin/setup.py >/dev/null -CFG=config/adapters/unified-network-layer/unified-network-layer.yaml +python3 bin/setup.py >/dev/null +CFG=config/adapters/single-network-layer.yaml [ -f "$CFG" ] || die "$CFG was not rendered (a directory there? rm -r it, then re-run)" echo " $CFG rendered" # --- 3. adapter ------------------------------------------------------------- -step "3/4 network-layer-adapter on $IMAGE (loopback tier hops)" +step "3/4 single-network-layer-adapter on $IMAGE (loopback tier hops)" # The multi-adapter containers share this project; only one topology at a time. docker compose -f docker-compose.yml stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true -docker compose "${COMPOSE[@]}" up -d --force-recreate network-layer-adapter +docker compose "${COMPOSE[@]}" up -d --force-recreate single-network-layer-adapter for i in $(seq 1 60); do curl -sf "http://127.0.0.1:$PORT/health" >/dev/null && break - [ "$i" = 60 ] && { docker logs --tail 30 network-layer-adapter; die "adapter not healthy after 60s"; } + [ "$i" = 60 ] && { docker logs --tail 30 single-network-layer-adapter; die "adapter not healthy after 60s"; } sleep 1 done -echo " image: $(docker inspect network-layer-adapter --format '{{.Config.Image}}')" +echo " image: $(docker inspect single-network-layer-adapter --format '{{.Config.Image}}')" echo " health: $(curl -s "http://127.0.0.1:$PORT/health")" # --- 4. smoke test ---------------------------------------------------------- -step "4/4 publish, discover, select through the unified network layer" +step "4/4 publish, discover, select through the single network layer" ROOT="$ROOT" PORT="$PORT" python3 - <<'PY' import json, os, pathlib, sys, urllib.error, urllib.request, uuid root = pathlib.Path(os.environ["ROOT"]); port = os.environ["PORT"] @@ -91,6 +91,6 @@ sys.exit(rc) PY echo echo " tier hops seen in the adapter log:" -docker logs --since 2m network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' +docker logs --since 2m single-network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' -printf '\n\033[1;32mUnified network layer running on %s. Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" +printf '\n\033[1;32mSingle network layer running on %s. Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 613749e6..b7eeae57 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -64,8 +64,8 @@ preflight() { || die "the python 'cryptography' package is missing -- pip install cryptography" } -# The unified network layer renders its config with PyYAML (multi mode does not). -preflight_unified_network_layer() { +# The single network layer renders its config with PyYAML (multi mode does not). +preflight_single_network_layer() { preflight python3 -c 'import yaml' >/dev/null 2>&1 \ || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" @@ -150,8 +150,8 @@ up_adapters() { # new participant id and base URL in .env plus a setup.py re-run, because # the registry cannot repoint an existing row. step 3 "$1" "adapters (provider, network, consumer)" - # The unified network layer adapter also publishes 9200: stop it first. - docker compose -f docker-compose.unified-network-layer.yml stop network-layer-adapter >/dev/null 2>&1 || true + # The single network layer adapter also publishes 9200: stop it first. + docker compose -f docker-compose.single-network-layer.yml stop single-network-layer-adapter >/dev/null 2>&1 || true docker compose up -d provider-adapter network-adapter consumer-adapter } @@ -174,44 +174,44 @@ done_banner() { NEXT } -# ----------------------------------------------- unified network layer +# ----------------------------------------------- single network layer -# The unified network layer: ONE adapter runs the consumer, network and -# provider tiers as modules of one process (the unified single adapter), with +# The single network layer: ONE adapter runs the consumer, network and +# provider tiers as modules of one process (the single network layer adapter), with # tier-to-tier hops over loopback HTTP inside the container. The multi-adapter -# containers are NOT started. `make up-unified-network-layer` does not start the -# mocks either, as in `up`; bin/unified-network-layer-up.sh does start them, for +# containers are NOT started. `make up-single-network-layer` does not start the +# mocks either, as in `up`; bin/single-network-layer-up.sh does start them, for # its local smoke test. # -# Its stack is its own complete compose file (docker-compose.unified-network-layer.yml), +# Its stack is its own complete compose file (docker-compose.single-network-layer.yml), # so docker-compose.yml is not touched. Passing -f overrides COMPOSE_FILE, so # the Apple-silicon override is added here when it exists. -unified_network_layer_compose() { - local files=(-f docker-compose.unified-network-layer.yml) +single_network_layer_compose() { + local files=(-f docker-compose.single-network-layer.yml) [ -f docker-compose.arm64.yml ] && files+=(-f docker-compose.arm64.yml) docker compose "${files[@]}" "$@" } -up_unified_network_layer() { - preflight_unified_network_layer - step 1 3 "registry and discovery (unified network layer compose file)" - unified_network_layer_compose up -d sunbird-registry-service discovery-service - UNIFIED_NETWORK_LAYER=1 up_setup 3 +up_single_network_layer() { + preflight_single_network_layer + step 1 3 "registry and discovery (single network layer compose file)" + single_network_layer_compose up -d sunbird-registry-service discovery-service + up_setup 3 # Never let compose create a directory where the bind-mounted config file belongs. - [ -f config/adapters/unified-network-layer/unified-network-layer.yaml ] \ - || die "config/adapters/unified-network-layer/unified-network-layer.yaml was not rendered (is it a directory? rm -r it, then re-run)" - step 3 3 "network-layer-adapter (consumer + network + provider in one process)" + [ -f config/adapters/single-network-layer.yaml ] \ + || die "config/adapters/single-network-layer.yaml was not rendered (is it a directory? rm -r it, then re-run)" + step 3 3 "single-network-layer-adapter (consumer + network + provider in one process)" # The multi-adapter containers share this project; one topology at a time. docker compose stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true - unified_network_layer_compose up -d --force-recreate network-layer-adapter - printf '\n\033[1;32m==> unified network layer is up\033[0m on 127.0.0.1:9200\n' + single_network_layer_compose up -d --force-recreate single-network-layer-adapter + printf '\n\033[1;32m==> single network layer is up\033[0m on 127.0.0.1:9200\n' } -# Stops only the unified network layer adapter; registry, discovery and data stay. -down_unified_network_layer() { - step 1 1 "stopping network-layer-adapter only" - unified_network_layer_compose stop network-layer-adapter - unified_network_layer_compose rm -f network-layer-adapter +# Stops only the single network layer adapter; registry, discovery and data stay. +down_single_network_layer() { + step 1 1 "stopping single-network-layer-adapter only" + single_network_layer_compose stop single-network-layer-adapter + single_network_layer_compose rm -f single-network-layer-adapter } # ----------------------------------------------------------------- down @@ -376,9 +376,9 @@ bin/stack.sh up the whole stack: registry+discovery -> setup.py -> adapters -> gateway (public, 80/443) -> hyperdx (wants 16 GB) up-core steps 1-3 only. Nothing public, no ClickHouse. - up-unified-network-layer docker-compose.unified-network-layer.yml: registry+discovery - -> setup.py -> network-layer-adapter - down-unified-network-layer stop and remove network-layer-adapter only + up-single-network-layer docker-compose.single-network-layer.yml: registry+discovery + -> setup.py -> single-network-layer-adapter + down-single-network-layer stop and remove single-network-layer-adapter only down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only @@ -396,8 +396,8 @@ USAGE case "${1:-}" in up) up ;; up-core) up_core ;; - up-unified-network-layer) up_unified_network_layer ;; - down-unified-network-layer) down_unified_network_layer ;; + up-single-network-layer) up_single_network_layer ;; + down-single-network-layer) down_single_network_layer ;; down) down ;; destroy) destroy ;; setup) setup ;; diff --git a/quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml b/quick-start/config/adapters/routing-single-network-layer-consumer.yaml similarity index 81% rename from quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml rename to quick-start/config/adapters/routing-single-network-layer-consumer.yaml index 662f73dd..73ef9336 100644 --- a/quick-start/config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml +++ b/quick-start/config/adapters/routing-single-network-layer-consumer.yaml @@ -1,5 +1,5 @@ -# What the consumer layer does with each action, in the unified network layer -# (the unified single adapter: consumer, network and provider as modules of one +# What the consumer layer does with each action, in the single network layer +# (the single network layer adapter: consumer, network and provider as modules of one # adapter). # # Both targets are THIS SAME adapter on another path. In the multi-adapter @@ -10,7 +10,7 @@ # # targetType "url" appends the action to the path, so the base here is the # module mount point and /discover or /select is added to it. localhost:9200 is -# the container's own listener and must match `port` in overrides.yaml. +# the container's own listener and must match `http.port` in single-network-layer.yaml.tmpl. routingRules: # Discovery is answered from this network's own discovery service, which the # network module fronts. diff --git a/quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml b/quick-start/config/adapters/routing-single-network-layer-publish.yaml similarity index 83% rename from quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml rename to quick-start/config/adapters/routing-single-network-layer-publish.yaml index fd566ec0..8a04cb11 100644 --- a/quick-start/config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml +++ b/quick-start/config/adapters/routing-single-network-layer-publish.yaml @@ -1,5 +1,5 @@ -# This network's catalogue system publishing inward, in the unified network -# layer (the unified single adapter). +# This network's catalogue system publishing inward, in the single network +# layer (the single network layer adapter). # # It goes to the NETWORK module rather than straight to discovery, so a publish # takes the same path a publish from anywhere else takes -- one place that diff --git a/quick-start/config/adapters/single-network-layer.yaml.tmpl b/quick-start/config/adapters/single-network-layer.yaml.tmpl new file mode 100644 index 00000000..86d60f2d --- /dev/null +++ b/quick-start/config/adapters/single-network-layer.yaml.tmpl @@ -0,0 +1,356 @@ +# Single network layer adapter: the consumer, network and provider tiers as +# four modules of ONE adapter process (docker-compose.single-network-layer.yml). +# +# / the consumer (bap); the app posts here, unsigned +# /network/ verifies and forwards to discovery +# /provider/ answers from the upstream APIs (bpp) +# /provider/publish the catalogue publishes here +# +# Each module is its tier's module from consumer.yaml.tmpl, network.yaml.tmpl +# and provider.yaml.tmpl. A change to one of those templates that should also +# apply here must be made in this file too. The same layout as the Helm chart's +# charts/adapter-service/config/single-network-layer-config.yaml. +# +# Differences from the three tier templates: +# - one listener (9200) and one appName; tier hops are loopback HTTP to this +# same listener (routing-single-network-layer-*.yaml beside this file); +# - extended schema validation on in every module; +# - an outbound connection pool (httpClientConfig) on every module; +# - log level info. +# +# bin/setup.py fills the placeholders -- each module's own tier identity +# (the CONSUMER_, NETWORK_ and PROVIDER_ ones) and the shared ones -- and +# writes config/adapters/single-network-layer.yaml (mode 600, gitignored: it +# holds private keys). +appName: single-network-layer-adapter + +log: + level: info + destinations: + - type: stdout + contextKeys: + - transaction_id + - message_id + - subscriber_id + - module_id + +http: + port: 9200 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +plugins: + otelsetup: + id: otelsetup + config: + serviceName: single-network-layer-adapter + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + +modules: + # ------------------------------------------------------------------ consumer + # From consumer.yaml.tmpl. Mounted at /, so the app calls /discover, /select, + # ... as on the multi-adapter consumer-adapter. Accepts unsigned requests, + # signs as the consumer, routes discover to /network and transactions to + # /provider (routing-single-network-layer-consumer.yaml). + - name: consumer + path: / + handler: + type: std + role: bap + subscriberId: "__CONSUMER_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://sunbird-registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + keyManager: + id: simplekeymanager + config: + subscriberId: "__CONSUMER_SUBSCRIBER_ID__" + keyId: "__CONSUMER_KEY_ID__" + signingPrivateKey: "__CONSUMER_SIGNING_PRIVATE__" + signingPublicKey: "__CONSUMER_SIGNING_PUBLIC__" + encrPrivateKey: "__CONSUMER_ENCR_PRIVATE__" + encrPublicKey: "__CONSUMER_ENCR_PUBLIC__" + signer: + id: signer + signValidator: + id: signvalidator + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + schemaValidator: + id: schemav2validator + config: + type: url + location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml + cacheTTL: '3600' + extendedSchema_enabled: 'true' + extendedSchema_cacheTTL: '86400' + extendedSchema_maxCacheSize: '100' + extendedSchema_downloadTimeout: '30' + extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io + router: + id: router + config: + routingConfig: /app/config/routing-single-network-layer-consumer.yaml + steps: + - validateSchema + - addRoute + + # ------------------------------------------------------------------- network + # From network.yaml.tmpl. Verifies the caller and forwards discover and + # publish to discovery (routing-network.yaml, the multi-adapter file). + - name: network + path: /network/ + handler: + type: std + role: bpp + subscriberId: "__NETWORK_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://sunbird-registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + keyManager: + id: simplekeymanager + config: + subscriberId: "__NETWORK_SUBSCRIBER_ID__" + keyId: "__NETWORK_KEY_ID__" + signingPrivateKey: "__NETWORK_SIGNING_PRIVATE__" + signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" + encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" + encrPublicKey: "__NETWORK_ENCR_PUBLIC__" + signer: + id: signer + signValidator: + id: signvalidator + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + schemaValidator: + id: schemav2validator + config: + type: url + location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml + cacheTTL: '3600' + extendedSchema_enabled: 'true' + extendedSchema_cacheTTL: '86400' + extendedSchema_maxCacheSize: '100' + extendedSchema_downloadTimeout: '30' + extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io + router: + id: router + config: + routingConfig: /app/config/routing-network.yaml + steps: + - addRoute + + # ------------------------------------------------------------------ provider + # From provider.yaml.tmpl, module 1. Verifies the caller and answers from the + # upstream APIs itself: no router. + - name: provider + path: /provider/ + handler: + type: std + role: bpp + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://sunbird-registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + keyManager: + id: simplekeymanager + config: + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + keyId: "__PROVIDER_KEY_ID__" + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + signValidator: + id: signvalidator + signer: + id: signer + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + schemaValidator: + id: schemav2validator + config: + type: url + location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml + cacheTTL: '3600' + extendedSchema_enabled: 'true' + extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io + extendedSchema_cacheTTL: '86400' + extendedSchema_maxCacheSize: '100' + extendedSchema_downloadTimeout: '30' + mapper: + id: jsonmapper + config: + fetchTimeout: 10s + cacheTTL: 1m + negativeTTL: 1m + maxMappingBytes: 262144 + maxCacheEntries: 200 + providerSteps: + - id: WeatherObservation + config: + bindingKeys: "__MAUSAMGRAM_BINDING_KEY__" + __MAUSAMGRAM_PARTICIPANT_ID__: + authScheme: header + headerName: Authorization + headerValueEnv: MAUSAMGRAM_AUTH + - id: MandiPrice + config: + bindingKeys: "__AGMARKNET_BINDING_KEY__" + __AGMARKNET_PARTICIPANT_ID__: + authScheme: tokenQuery + tokenUrl: "__AGMARKNET_TOKEN_URL__" + tokenUserField: access_name + tokenUserEnv: AGMARKNET_ACCESS_NAME + tokenSecretField: password + tokenSecretEnv: AGMARKNET_PASSWORD + tokenResponseField: token + queryName: token + tokenTtl: 10m + - id: KnowledgeAdvisory + config: + bindingKeys: "__VISTAAR_BINDING_KEY__" + __VISTAAR_PARTICIPANT_ID__: + authScheme: oauth2 + tokenUrl: "__VISTAAR_TOKEN_URL__" + clientIdEnv: VISTAAR_CLIENT_ID + clientSecretEnv: VISTAAR_CLIENT_SECRET + maxResponseBytes: 1048576 + - id: AgricultureFacility + config: + bindingKeys: "__POCRA_BINDING_KEY__" + __POCRA_PARTICIPANT_ID__: + authScheme: none + searchConcurrency: 1 + steps: + - validateSchema + - WeatherObservation + - MandiPrice + - KnowledgeAdvisory + - AgricultureFacility + + # ---------------------------------------------------------- provider-publish + # From provider.yaml.tmpl, module 2. The catalogue publishes here; signed as + # the provider and routed to /network/publish + # (routing-single-network-layer-publish.yaml). + - name: provider-publish + path: /provider/publish + handler: + type: std + role: bap + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + httpClientConfig: + maxIdleConns: 512 + maxIdleConnsPerHost: 128 + idleConnTimeout: 90s + responseHeaderTimeout: 30s + plugins: + registry: + id: sunbirdRegistry + config: + url: http://sunbird-registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + keyManager: + id: simplekeymanager + config: + subscriberId: "__PROVIDER_SUBSCRIBER_ID__" + keyId: "__PROVIDER_KEY_ID__" + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + signer: + id: signer + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + schemaValidator: + id: schemav2validator + config: + type: url + location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml + cacheTTL: '3600' + extendedSchema_enabled: 'true' + extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io + extendedSchema_cacheTTL: '86400' + extendedSchema_maxCacheSize: '100' + extendedSchema_downloadTimeout: '30' + router: + id: router + config: + routingConfig: /app/config/routing-single-network-layer-publish.yaml + steps: + - validateSchema + - addRoute diff --git a/quick-start/config/adapters/unified-network-layer/overrides.yaml b/quick-start/config/adapters/unified-network-layer/overrides.yaml deleted file mode 100644 index 3f01ddcc..00000000 --- a/quick-start/config/adapters/unified-network-layer/overrides.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Unified network layer: the consumer, network and provider tiers as modules of -# ONE adapter process (the unified single adapter). -# -# This file holds only what differs from the three tier templates beside it -# (consumer.yaml.tmpl, network.yaml.tmpl, provider.yaml.tmpl). bin/setup.py -# builds unified-network-layer.yaml from those templates plus this file, so a -# change to a tier template reaches both the multi-adapter stack and this one -# on the next run -- there is no second copy of any module to keep in step. -# -# Every module keeps its own tier identity (__CONSUMER_*__, __NETWORK_*__, -# __PROVIDER_*__): the same keys and registry rows the multi-adapter stack -# uses, so either stack can run against the same registry. - -appName: network-layer-adapter -otelServiceName: network-layer-adapter -# Inside the container; published on 127.0.0.1:9200 (docker-compose.unified-network-layer.yml). -port: 9200 - - -# Extended schema validation on for every module that validates schemas. -extendedSchemaEnabled: "true" - -# Which tier-template module becomes which module here. `index` is the -# module's position in that template's `modules:` list. `routingConfig` is the -# path (inside the container) of that module's routing file; compose mounts: -# consumer unified-network-layer/single-network-layer-consumer-routing.yaml -# network routing-network.yaml (the multi-adapter file, unchanged) -# provider-publish unified-network-layer/single-network-layer-publish-routing.yaml -# The provider module has no router: it answers requests itself. -modules: - - from: consumer - index: 0 - name: consumer - # Mounted at /, so the app calls /discover, /select, ... as it does on the - # multi-adapter consumer-adapter. The more specific /network/, /provider/ - # and /provider/publish paths still go to their own modules. - path: / - routingConfig: /app/config/single-network-layer-consumer-routing.yaml - - from: network - index: 0 - name: network - path: /network/ - routingConfig: /app/config/routing-network.yaml - - from: provider - index: 0 - name: provider - path: /provider/ - # The provider's catalogue publishing. An exact path: stripping it leaves the - # empty endpoint, which the routing file keys on with excludeAction. - - from: provider - index: 1 - name: provider-publish - path: /provider/publish - routingConfig: /app/config/single-network-layer-publish-routing.yaml diff --git a/quick-start/docker-compose.unified-network-layer.yml b/quick-start/docker-compose.single-network-layer.yml similarity index 95% rename from quick-start/docker-compose.unified-network-layer.yml rename to quick-start/docker-compose.single-network-layer.yml index 52a4679c..361bfe36 100644 --- a/quick-start/docker-compose.unified-network-layer.yml +++ b/quick-start/docker-compose.single-network-layer.yml @@ -1,15 +1,15 @@ -# The whole OAN stack in one file, UNIFIED NETWORK LAYER variant: registry, +# The whole OAN stack in one file, SINGLE NETWORK LAYER variant: registry, # discovery, ONE adapter running the consumer, network and provider tiers as -# modules of one process (the unified single adapter), the telemetry stack and +# modules of one process (the single network layer adapter), the telemetry stack and # the public edge. # # This is docker-compose.yml with the three adapter services replaced by one -# `network-layer-adapter`. Everything else is the same, so use ONE of +# `single-network-layer-adapter`. Everything else is the same, so use ONE of # the two files, not both: they share the project name, container names and # data volumes. # -# make up-unified-network-layer (or: bin/unified-network-layer-up.sh) -# docker compose -f docker-compose.unified-network-layer.yml up -d +# make up-single-network-layer (or: bin/single-network-layer-up.sh) +# docker compose -f docker-compose.single-network-layer.yml up -d # # Tier-to-tier hops go over loopback HTTP to the container's own listener # (http://localhost:9200): they never leave the container. One routing file per @@ -24,7 +24,7 @@ # registry sunbird-registry-postgres, sunbird-registry-keycloak, # sunbird-registry-service # discovery discovery-postgres, discovery-service -# adapter network-layer-adapter (consumer + network + provider) +# adapter single-network-layer-adapter (consumer + network + provider) # observability hyperdx profile: observability # edge nginx-proxy-manager profile: reverse-proxy # @@ -35,12 +35,12 @@ # # 1. cp .env.example .env change every credential -- they are # shipped defaults -# 2. docker compose -f docker-compose.unified-network-layer.yml up -d sunbird-registry-service discovery-service +# 2. docker compose -f docker-compose.single-network-layer.yml up -d sunbird-registry-service discovery-service # everything EXCEPT the adapter # 3. python3 bin/setup.py keys, the three tier identities, and -# config/adapters/unified-network-layer/unified-network-layer.yaml -# 4. docker compose -f docker-compose.unified-network-layer.yml up -d now the adapter -# 5. docker compose -f docker-compose.unified-network-layer.yml --profile reverse-proxy --profile observability up -d +# config/adapters/single-network-layer.yaml +# 4. docker compose -f docker-compose.single-network-layer.yml up -d now the adapter +# 5. docker compose -f docker-compose.single-network-layer.yml --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack # # Naming registry and discovery in step 2 is not tidiness. An adapter config @@ -79,7 +79,7 @@ # decoration and becomes the actual control. NPM's "Forward Hostname" is a # free-text field, so anyone with the admin password can point a public route # at `sunbird-registry-service` or `sunbird-registry-keycloak`. NPM sits only on -# oan-edge, where neither name resolves and neither address is routable -- network-layer-adapter and hyperdx +# oan-edge, where neither name resolves and neither address is routable -- single-network-layer-adapter and hyperdx # are the only containers it shares a network with. Those three straddle both # networks; everything else is internal-only. # @@ -546,7 +546,7 @@ services: condition: service_healthy # ========================================================================== - # adapter -- network-layer-adapter: the consumer, network and provider tiers as + # adapter -- single-network-layer-adapter: the consumer, network and provider tiers as # modules of one process, one image, one config. # # Its config is a bind-mounted FILE that does not exist until bin/setup.py @@ -600,11 +600,10 @@ services: ports: - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" - # The unified network layer: the consumer, network and provider tiers as - # modules of ONE adapter process (the unified single adapter), on ONE config - # built by bin/setup.py from the three tier templates - # (config/adapters/unified-network-layer/unified-network-layer.yaml), with one - # routing file per module. Tier-to-tier hops go over loopback (http://localhost:9200), + # The single network layer: the consumer, network and provider tiers as + # modules of ONE adapter process (the single network layer adapter), on ONE + # config rendered by bin/setup.py from config/adapters/single-network-layer.yaml.tmpl + # (config/adapters/single-network-layer.yaml), with one routing file per module. Tier-to-tier hops go over loopback (http://localhost:9200), # inside the container, so there is no consumer-adapter, network-adapter or # provider-adapter here. # @@ -617,9 +616,9 @@ services: # # Each module signs as its own tier identity (consumer, network, provider): # the same keys and registry rows as the multi-adapter stack. - network-layer-adapter: + single-network-layer-adapter: <<: *adapter - container_name: network-layer-adapter + container_name: single-network-layer-adapter depends_on: sunbird-registry-service: condition: service_healthy @@ -627,7 +626,7 @@ services: condition: service_started environment: <<: *adapter-env - OTEL_SERVICE_NAME: network-layer-adapter + OTEL_SERVICE_NAME: single-network-layer-adapter # The weather upstream's credential, as ONE ready-made header value -- # "Basic ", prefix included. Named here and read at call time, so # the value is in neither a config file nor the registry. @@ -643,12 +642,12 @@ services: VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} volumes: - - ./config/adapters/unified-network-layer/unified-network-layer.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/single-network-layer.yaml:/app/config/adapter.yaml:ro # One routing file per module, as in the multi-adapter stack. The network # module reuses the multi-adapter routing-network.yaml unchanged. - - ./config/adapters/unified-network-layer/single-network-layer-consumer-routing.yaml:/app/config/single-network-layer-consumer-routing.yaml:ro + - ./config/adapters/routing-single-network-layer-consumer.yaml:/app/config/routing-single-network-layer-consumer.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - - ./config/adapters/unified-network-layer/single-network-layer-publish-routing.yaml:/app/config/single-network-layer-publish-routing.yaml:ro + - ./config/adapters/routing-single-network-layer-publish.yaml:/app/config/routing-single-network-layer-publish.yaml:ro ports: - "127.0.0.1:9200:9200" @@ -656,7 +655,7 @@ services: # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI # over it. # - # docker compose -f docker-compose.unified-network-layer.yml --profile observability up -d + # docker compose -f docker-compose.single-network-layer.yml --profile observability up -d # # Sizing: ClickHouse alone wants 2-4 GB. With this profile on, the VM needs # 16 GB (t3.xlarge or m6i.xlarge); without it, 8 GB is workable. @@ -822,7 +821,7 @@ services: # ========================================================================== # edge -- Nginx Proxy Manager. # - # docker compose -f docker-compose.unified-network-layer.yml --profile reverse-proxy up -d + # docker compose -f docker-compose.single-network-layer.yml --profile reverse-proxy up -d # # NPM rather than a hand-written nginx.conf because it owns the part that is # genuinely tedious to do by hand -- ACME. It requests, installs and renews @@ -845,7 +844,7 @@ services: # NPM's "Forward Hostname" is a free-text field. Anyone with the admin # password can type `sunbird-registry-service`, `sunbird-registry-keycloak` or `discovery-postgres` into it. On # this network none of those names resolve and none of those addresses are - # routable from here -- network-layer-adapter, and hyperdx when the + # routable from here -- single-network-layer-adapter, and hyperdx when the # observability profile is up, are the only containers NPM shares a network # with. So the blast radius of a wrong click, or of the admin UI being # reached by someone who should not have it, is bounded to that set. @@ -917,7 +916,7 @@ services: # whenever NPM does. That means the restart above is not optional after # `make observability` -- NPM was started before the name existed, so the # HyperDX host 502s until it re-resolves. - network-layer-adapter: + single-network-layer-adapter: condition: service_started healthcheck: From 9b51eb9967106a4a22528481474751dc81d6e67c Mon Sep 17 00:00:00 2001 From: Kelvin Date: Sat, 10 Oct 2026 15:36:41 +0530 Subject: [PATCH 11/43] feat(adapter-service): update single network layer configuration and documentation for unsigned requests --- charts/adapter-service/templates/ingress.yaml | 6 ++++-- quick-start/bin/single-network-layer-up.sh | 2 -- quick-start/bin/stack.sh | 9 +-------- .../adapters/single-network-layer.yaml.tmpl | 18 ++++++++++++------ .../docker-compose.single-network-layer.yml | 6 ++++-- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index ac9cb28e..2bfed931 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -37,8 +37,10 @@ {{/* Off by default, and worth a thought before turning on. -This adapter takes SIGNED requests -- validateSign is its first step -- so -publishing it is not the same risk as publishing an adapter that does not. But +The provider and network roles take SIGNED requests -- validateSign is their +first step -- so publishing them is not the same risk as publishing an adapter +that does not. The consumer role, and the single-network-layer consumer module +at /, take UNSIGNED requests (NOTES says so when ingress is on). Either way it is still a Beckn peer endpoint, and the thing that makes it safe is that every caller must hold a key the registry publishes. That is a property of the registry's contents, not of this Ingress. diff --git a/quick-start/bin/single-network-layer-up.sh b/quick-start/bin/single-network-layer-up.sh index b34b6aeb..ccd93d0e 100755 --- a/quick-start/bin/single-network-layer-up.sh +++ b/quick-start/bin/single-network-layer-up.sh @@ -28,8 +28,6 @@ die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } command -v python3 >/dev/null 2>&1 || die "python3 is not installed -- bin/setup.py needs it" python3 -c 'import cryptography' >/dev/null 2>&1 \ || die "the python 'cryptography' package is missing -- pip install cryptography" -python3 -c 'import yaml' >/dev/null 2>&1 \ - || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" # --- 1. infrastructure ------------------------------------------------------ step "1/4 registry, discovery, mocks (the mocks are for this script's local smoke test; make up-single-network-layer does not start them)" diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index b7eeae57..13fef204 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -64,13 +64,6 @@ preflight() { || die "the python 'cryptography' package is missing -- pip install cryptography" } -# The single network layer renders its config with PyYAML (multi mode does not). -preflight_single_network_layer() { - preflight - python3 -c 'import yaml' >/dev/null 2>&1 \ - || die "the python 'yaml' package (PyYAML) is missing -- pip install pyyaml, or apt install python3-yaml" -} - # ------------------------------------------------------------------- up # The full stack, in the order the compose file's own header documents. Five @@ -193,7 +186,7 @@ single_network_layer_compose() { } up_single_network_layer() { - preflight_single_network_layer + preflight step 1 3 "registry and discovery (single network layer compose file)" single_network_layer_compose up -d sunbird-registry-service discovery-service up_setup 3 diff --git a/quick-start/config/adapters/single-network-layer.yaml.tmpl b/quick-start/config/adapters/single-network-layer.yaml.tmpl index 86d60f2d..670dfacf 100644 --- a/quick-start/config/adapters/single-network-layer.yaml.tmpl +++ b/quick-start/config/adapters/single-network-layer.yaml.tmpl @@ -18,6 +18,12 @@ # - an outbound connection pool (httpClientConfig) on every module; # - log level info. # +# SIGNING IS OFF, as in the three quick-start tier templates: no module runs +# sign or validateSign, so /network/ and /provider/ accept unsigned requests +# from anyone who can reach :9200. Keep it on 127.0.0.1 (as the compose file +# does) and never put /network/ or /provider/ behind the reverse proxy. The +# Helm chart's single-network-layer-config.yaml has signing on. +# # bin/setup.py fills the placeholders -- each module's own tier identity # (the CONSUMER_, NETWORK_ and PROVIDER_ ones) and the shared ones -- and # writes config/adapters/single-network-layer.yaml (mode 600, gitignored: it @@ -58,8 +64,8 @@ plugins: modules: # ------------------------------------------------------------------ consumer # From consumer.yaml.tmpl. Mounted at /, so the app calls /discover, /select, - # ... as on the multi-adapter consumer-adapter. Accepts unsigned requests, - # signs as the consumer, routes discover to /network and transactions to + # ... as on the multi-adapter consumer-adapter. Accepts unsigned requests + # under the consumer identity (signing is off here), routes discover to /network and transactions to # /provider (routing-single-network-layer-consumer.yaml). - name: consumer path: / @@ -123,7 +129,7 @@ modules: - addRoute # ------------------------------------------------------------------- network - # From network.yaml.tmpl. Verifies the caller and forwards discover and + # From network.yaml.tmpl. Forwards discover and # publish to discovery (routing-network.yaml, the multi-adapter file). - name: network path: /network/ @@ -186,7 +192,7 @@ modules: - addRoute # ------------------------------------------------------------------ provider - # From provider.yaml.tmpl, module 1. Verifies the caller and answers from the + # From provider.yaml.tmpl, module 1. Answers from the # upstream APIs itself: no router. - name: provider path: /provider/ @@ -293,8 +299,8 @@ modules: - AgricultureFacility # ---------------------------------------------------------- provider-publish - # From provider.yaml.tmpl, module 2. The catalogue publishes here; signed as - # the provider and routed to /network/publish + # From provider.yaml.tmpl, module 2. The catalogue publishes here, under the + # provider identity (signing is off here); routed to /network/publish # (routing-single-network-layer-publish.yaml). - name: provider-publish path: /provider/publish diff --git a/quick-start/docker-compose.single-network-layer.yml b/quick-start/docker-compose.single-network-layer.yml index 361bfe36..cdadc5a0 100644 --- a/quick-start/docker-compose.single-network-layer.yml +++ b/quick-start/docker-compose.single-network-layer.yml @@ -614,8 +614,10 @@ services: # /provider/* calls the upstream, answers synchronously # /provider/publish sends the provider's catalogue to /network/publish # - # Each module signs as its own tier identity (consumer, network, provider): - # the same keys and registry rows as the multi-adapter stack. + # Each module carries its own tier identity (consumer, network, provider): + # the same keys and registry rows as the multi-adapter stack. Signing is off, + # as in the multi-adapter quick-start templates, so /network/ and /provider/ + # accept unsigned requests: keep :9200 on 127.0.0.1. single-network-layer-adapter: <<: *adapter container_name: single-network-layer-adapter From 05a016a7117217e38afc024a00e6f2d0444538fe Mon Sep 17 00:00:00 2001 From: Kelvin Date: Sat, 10 Oct 2026 16:44:39 +0530 Subject: [PATCH 12/43] feat(adapter-service): update values.yaml with default VISTAAR_TOKEN_URL substitution and increment chart version to 0.1.1 --- charts/adapter-service/values.yaml | 6 ++++-- charts/openbao-secrets/CHANGELOG.md | 2 +- charts/openbao-secrets/Chart.yaml | 2 +- quick-start/README.md | 2 +- .../config/adapters/single-network-layer.yaml.tmpl | 8 ++++---- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 7192650a..ea9df2d4 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -451,5 +451,7 @@ singleNetworkLayer: existingSecret: "" # Extra placeholders, as "/". - # __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl - extraSubstitutions: {} + # config/single-network-layer-config.yaml uses __VISTAAR_TOKEN_URL__, so it is + # set by default: the provider tier's Secret must carry vistaarTokenUrl. + extraSubstitutions: + __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index 49291130..63209e41 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to the `openbao-secrets` chart are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0] - 2026-10-09 +## [0.1.1] - 2026-10-09 ### Added - `single-network-layer-adapter` in the namespaces of `consumer-adapter-keys`, diff --git a/charts/openbao-secrets/Chart.yaml b/charts/openbao-secrets/Chart.yaml index 7a430ef1..b6971ecd 100644 --- a/charts/openbao-secrets/Chart.yaml +++ b/charts/openbao-secrets/Chart.yaml @@ -6,7 +6,7 @@ description: >- This is what turns an OpenBao entry into something a chart can reference by name. type: application -version: 0.1.0 +version: 0.1.1 appVersion: "0.10.7" maintainers: - name: OpenAgriNet Engineering Team diff --git a/quick-start/README.md b/quick-start/README.md index 6891726c..cbf67599 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -208,7 +208,7 @@ Both use `ADAPTER_IMAGE` from `.env`; nothing else needs setting. tier templates. `bin/setup.py` fills it into `config/adapters/single-network-layer.yaml` like the others. Each module is its tier's module from the tier template (a change to a tier template that should apply here goes in this file too), plus -extended schema validation, an outbound connection pool and log level info. +extended schema validation on every module, an outbound connection pool and log level info. Each module keeps its own tier identity. The same layout as the Helm chart's `config/single-network-layer-config.yaml`. diff --git a/quick-start/config/adapters/single-network-layer.yaml.tmpl b/quick-start/config/adapters/single-network-layer.yaml.tmpl index 670dfacf..b37d5b5e 100644 --- a/quick-start/config/adapters/single-network-layer.yaml.tmpl +++ b/quick-start/config/adapters/single-network-layer.yaml.tmpl @@ -115,7 +115,7 @@ modules: type: url location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml cacheTTL: '3600' - extendedSchema_enabled: 'true' + extendedSchema_enabled: "true" extendedSchema_cacheTTL: '86400' extendedSchema_maxCacheSize: '100' extendedSchema_downloadTimeout: '30' @@ -179,7 +179,7 @@ modules: type: url location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml cacheTTL: '3600' - extendedSchema_enabled: 'true' + extendedSchema_enabled: "true" extendedSchema_cacheTTL: '86400' extendedSchema_maxCacheSize: '100' extendedSchema_downloadTimeout: '30' @@ -242,7 +242,7 @@ modules: type: url location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml cacheTTL: '3600' - extendedSchema_enabled: 'true' + extendedSchema_enabled: "true" extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io extendedSchema_cacheTTL: '86400' extendedSchema_maxCacheSize: '100' @@ -348,7 +348,7 @@ modules: type: url location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml cacheTTL: '3600' - extendedSchema_enabled: 'true' + extendedSchema_enabled: "true" extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io extendedSchema_cacheTTL: '86400' extendedSchema_maxCacheSize: '100' From 2a9a966b1001c21d7802fa297e7d7268991c4aa0 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Sat, 10 Oct 2026 16:51:02 +0530 Subject: [PATCH 13/43] feat(adapter-service): remove default VISTAAR_TOKEN_URL substitution from values.yaml --- charts/adapter-service/values.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index ea9df2d4..7192650a 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -451,7 +451,5 @@ singleNetworkLayer: existingSecret: "" # Extra placeholders, as "/". - # config/single-network-layer-config.yaml uses __VISTAAR_TOKEN_URL__, so it is - # set by default: the provider tier's Secret must carry vistaarTokenUrl. - extraSubstitutions: - __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + # __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + extraSubstitutions: {} From 82c0ae438e7990ebff1353bc58c6f194719750f6 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 12:22:06 +0530 Subject: [PATCH 14/43] Add OpenAgriNet local environment configuration for single network layer - Introduced a new JSON configuration file for the OpenAgriNet local environment. - Defined key-value pairs for various services including discovery, registry, and keycloak hosts. - Included participant IDs, capability bindings, and mapping URLs relevant to the weather observation capability. - Ensured all configurations are enabled and documented for clarity. --- .gitignore | 4 + ...t.single-network-layer.api-collection.json | 736 ++++++++++++++ ...ingle-network-layer.local-environment.json | 146 +++ quick-start/.env.example | 8 + quick-start/Makefile | 11 +- quick-start/README.md | 124 ++- quick-start/bin/setup.py | 2 +- quick-start/bin/single-network-layer-up.sh | 94 -- quick-start/bin/stack.sh | 203 ++-- .../adapters/single-network-layer.yaml.tmpl | 2 +- .../docker-compose.single-network-layer.yml | 929 ------------------ quick-start/docker-compose.yml | 80 +- 12 files changed, 1210 insertions(+), 1129 deletions(-) create mode 100644 api-collection/OpenAgriNet.single-network-layer.api-collection.json create mode 100644 api-collection/OpenAgriNet.single-network-layer.local-environment.json delete mode 100755 quick-start/bin/single-network-layer-up.sh delete mode 100644 quick-start/docker-compose.single-network-layer.yml diff --git a/.gitignore b/.gitignore index ceaee1f4..f2bd38ba 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,7 @@ api-collection/OpenAgriNet.unified-network-layer.environment.json # Python bytecode __pycache__/ *.pyc +# Local demo and POC Postman collections (the committed ones are +# OpenAgriNet.api-collection.json and OpenAgriNet.single-network-layer.api-collection.json) +/api-collection/OpenAgriNet.single-network-layer-demo.* +/api-collection/OpenAgriNet.unified-network-layer.* diff --git a/api-collection/OpenAgriNet.single-network-layer.api-collection.json b/api-collection/OpenAgriNet.single-network-layer.api-collection.json new file mode 100644 index 00000000..3a58b691 --- /dev/null +++ b/api-collection/OpenAgriNet.single-network-layer.api-collection.json @@ -0,0 +1,736 @@ +{ + "info": { + "name": "OpenAgriNet - Single network layer", + "_postman_id": "0a4e1c00-0000-4000-8000-0000000snl01", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.single-network-layer.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, single-network-layer-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /network/ network module (internal)\n /provider/ provider module\n /provider/publish catalogue publishing\n\nPublish enters /provider/publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Fresh ids and a fresh timestamp on every send.", + "//", + "// messageId MUST be unique per request: discovery rejects a replay, and PoCRA", + "// caches its answer on this value for ten minutes while ignoring location --", + "// so a reused id silently returns the previous area's facilities.", + "const uuid = () => pm.variables.replaceIn('{{$guid}}');", + "", + "pm.collectionVariables.set('transactionId', uuid());", + "pm.collectionVariables.set('messageId', uuid());", + "pm.collectionVariables.set('timestamp', new Date().toISOString());" + ] + } + } + ], + "variable": [ + { + "key": "transactionId", + "value": "", + "type": "string" + }, + { + "key": "messageId", + "value": "", + "type": "string" + }, + { + "key": "timestamp", + "value": "", + "type": "string" + } + ], + "item": [ + { + "name": "MandiPrice Capability - MH", + "description": "273 Maharashtra markets from Agmarknet. Select fetches live prices, so it needs egress.", + "item": [ + { + "name": "Publish - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/provider/publish", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "provider", + "publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:mandi-price:MH\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MH\",\n \"name\": \"Maharashtra mandi prices\",\n \"shortDesc\": \"On-demand commodity price information for 2 agricultural markets in Maharashtra.\",\n \"longDesc\": \"Each Resource represents one market and advertises the commodities it trades. Consumers discover a market by location or commodity and invoke the Provider to obtain current prices as a Direct MandiPrice Resource.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\",\n \"shortDesc\": \"Provider of live agricultural market price information.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MARKET_1634\",\n \"name\": \"Rahuri APMC mandi prices\",\n \"shortDesc\": \"Current commodity prices for Rahuri APMC, Ahmednagar district, Maharashtra.\",\n \"longDesc\": \"Invoke this Resource with a commodity and applicable date to obtain minimum, maximum, and modal prices for Rahuri APMC. The result uses the same MandiPrice type in Direct information mode.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"agmarknet\",\n \"sourceName\": \"Agmarknet Vistaar\",\n \"sourceUri\": \"https://agmarknet.gov.in/\"\n },\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\",\n \"marketCode\": \"1634\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.66192086448265,\n 19.644527591487904\n ]\n }\n }\n },\n \"supportedCommodities\": [\n {\n \"code\": \"4\",\n \"name\": \"Maize\"\n },\n {\n \"code\": \"13\",\n \"name\": \"Soyabean\"\n },\n {\n \"code\": \"18\",\n \"name\": \"Orange\"\n },\n {\n \"code\": \"20\",\n \"name\": \"Mango\"\n },\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n },\n {\n \"code\": \"24\",\n \"name\": \"Potato\"\n },\n {\n \"code\": \"25\",\n \"name\": \"Garlic\"\n },\n {\n \"code\": \"28\",\n \"name\": \"Bajra(Pearl Millet/Cumbu)\"\n },\n {\n \"code\": \"31\",\n \"name\": \"Cauliflower\"\n },\n {\n \"code\": \"32\",\n \"name\": \"Brinjal\"\n },\n {\n \"code\": \"45\",\n \"name\": \"Arhar(Tur/Red Gram)(Whole)\"\n },\n {\n \"code\": \"46\",\n \"name\": \"Green Peas\"\n },\n {\n \"code\": \"59\",\n \"name\": \"Papaya\"\n },\n {\n \"code\": \"62\",\n \"name\": \"Guar\"\n },\n {\n \"code\": \"64\",\n \"name\": \"Mousambi(Sweet Lime)\"\n },\n {\n \"code\": \"65\",\n \"name\": \"Tomato\"\n },\n {\n \"code\": \"67\",\n \"name\": \"Bitter gourd\"\n },\n {\n \"code\": \"68\",\n \"name\": \"Bottle gourd\"\n },\n {\n \"code\": \"71\",\n \"name\": \"Bhindi(Ladies Finger)\"\n },\n {\n \"code\": \"73\",\n \"name\": \"Green Chilli\"\n },\n {\n \"code\": \"74\",\n \"name\": \"Chilly Capsicum\"\n },\n {\n \"code\": \"87\",\n \"name\": \"Ginger(Green)\"\n },\n {\n \"code\": \"124\",\n \"name\": \"Sweet Potato\"\n },\n {\n \"code\": \"126\",\n \"name\": \"Cabbage\"\n },\n {\n \"code\": \"131\",\n \"name\": \"Cucumbar(Kheera)\"\n },\n {\n \"code\": \"132\",\n \"name\": \"Ridgeguard(Tori)\"\n },\n {\n \"code\": \"140\",\n \"name\": \"Drumstick\"\n },\n {\n \"code\": \"145\",\n \"name\": \"Sweet Pumpkin\"\n },\n {\n \"code\": \"151\",\n \"name\": \"Lime\"\n },\n {\n \"code\": \"156\",\n \"name\": \"Guava\"\n },\n {\n \"code\": \"160\",\n \"name\": \"Pomegranate\"\n },\n {\n \"code\": \"162\",\n \"name\": \"Seetapal\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-16T23:59:59+05:30\"\n },\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.66192086448265,\n 19.644527591487904\n ]\n },\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n }\n ]\n }\n },\n {\n \"id\": \"resource:mandi-price:market:164\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MARKET_164\",\n \"name\": \"Pune APMC mandi prices\",\n \"shortDesc\": \"Current commodity prices for Pune APMC, Pune district, Maharashtra.\",\n \"longDesc\": \"Invoke this Resource with a commodity and applicable date to obtain minimum, maximum, and modal prices for Pune APMC. The result uses the same MandiPrice type in Direct information mode.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"agmarknet\",\n \"sourceName\": \"Agmarknet Vistaar\",\n \"sourceUri\": \"https://agmarknet.gov.in/\"\n },\n \"market\": {\n \"marketName\": \"Pune APMC\",\n \"district\": \"364\",\n \"state\": \"MH\",\n \"marketCode\": \"164\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.86850558,\n 18.49089838\n ]\n }\n }\n },\n \"supportedCommodities\": [\n {\n \"code\": \"3\",\n \"name\": \"Rice\"\n },\n {\n \"code\": \"4\",\n \"name\": \"Maize\"\n },\n {\n \"code\": \"5\",\n \"name\": \"Jowar(Sorghum)\"\n },\n {\n \"code\": \"8\",\n \"name\": \"Black Gram(Urd Beans)(Whole)\"\n },\n {\n \"code\": \"9\",\n \"name\": \"Green Gram(Moong)(Whole)\"\n },\n {\n \"code\": \"18\",\n \"name\": \"Orange\"\n },\n {\n \"code\": \"19\",\n \"name\": \"Banana\"\n },\n {\n \"code\": \"21\",\n \"name\": \"Pineapple\"\n },\n {\n \"code\": \"22\",\n \"name\": \"Grapes\"\n },\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n },\n {\n \"code\": \"24\",\n \"name\": \"Potato\"\n },\n {\n \"code\": \"25\",\n \"name\": \"Garlic\"\n },\n {\n \"code\": \"28\",\n \"name\": \"Bajra(Pearl Millet/Cumbu)\"\n },\n {\n \"code\": \"30\",\n \"name\": \"Ragi(Finger Millet)\"\n },\n {\n \"code\": \"31\",\n \"name\": \"Cauliflower\"\n },\n {\n \"code\": \"32\",\n \"name\": \"Brinjal\"\n },\n {\n \"code\": \"39\",\n \"name\": \"Coriander(Leaves)\"\n },\n {\n \"code\": \"42\",\n \"name\": \"Methi(Leaves)\"\n },\n {\n \"code\": \"46\",\n \"name\": \"Green Peas\"\n },\n {\n \"code\": \"52\",\n \"name\": \"Lentil(Masur)(Whole)\"\n },\n {\n \"code\": \"58\",\n \"name\": \"Chikoos(Sapota)\"\n },\n {\n \"code\": \"59\",\n \"name\": \"Papaya\"\n },\n {\n \"code\": \"60\",\n \"name\": \"Water Melon\"\n },\n {\n \"code\": \"61\",\n \"name\": \"Gur(Jaggery)\"\n },\n {\n \"code\": \"62\",\n \"name\": \"Guar\"\n },\n {\n \"code\": \"64\",\n \"name\": \"Mousambi(Sweet Lime)\"\n },\n {\n \"code\": \"65\",\n \"name\": \"Tomato\"\n },\n {\n \"code\": \"67\",\n \"name\": \"Bitter gourd\"\n },\n {\n \"code\": \"68\",\n \"name\": \"Bottle gourd\"\n },\n {\n \"code\": \"70\",\n \"name\": \"Pumpkin\"\n },\n {\n \"code\": \"71\",\n \"name\": \"Bhindi(Ladies Finger)\"\n },\n {\n \"code\": \"73\",\n \"name\": \"Green Chilli\"\n },\n {\n \"code\": \"74\",\n \"name\": \"Chilly Capsicum\"\n },\n {\n \"code\": \"80\",\n \"name\": \"Beans\"\n },\n {\n \"code\": \"87\",\n \"name\": \"Ginger(Green)\"\n },\n {\n \"code\": \"92\",\n \"name\": \"Corriander seed\"\n },\n {\n \"code\": \"124\",\n \"name\": \"Sweet Potato\"\n },\n {\n \"code\": \"126\",\n \"name\": \"Cabbage\"\n },\n {\n \"code\": \"128\",\n \"name\": \"Snakeguard\"\n },\n {\n \"code\": \"129\",\n \"name\": \"Beetroot\"\n },\n {\n \"code\": \"131\",\n \"name\": \"Cucumbar(Kheera)\"\n },\n {\n \"code\": \"132\",\n \"name\": \"Ridgeguard(Tori)\"\n },\n {\n \"code\": \"133\",\n \"name\": \"Raddish\"\n },\n {\n \"code\": \"140\",\n \"name\": \"Drumstick\"\n },\n {\n \"code\": \"144\",\n \"name\": \"Mango(Raw-Ripe)\"\n },\n {\n \"code\": \"145\",\n \"name\": \"Sweet Pumpkin\"\n },\n {\n \"code\": \"146\",\n \"name\": \"Peas Wet\"\n },\n {\n \"code\": \"151\",\n \"name\": \"Lime\"\n },\n {\n \"code\": \"153\",\n \"name\": \"Jack Fruit\"\n },\n {\n \"code\": \"155\",\n \"name\": \"Jamun(Narale Hannu)\"\n },\n {\n \"code\": \"156\",\n \"name\": \"Guava\"\n },\n {\n \"code\": \"157\",\n \"name\": \"Karbuja(Musk Melon)\"\n },\n {\n \"code\": \"160\",\n \"name\": \"Pomegranate\"\n },\n {\n \"code\": \"161\",\n \"name\": \"Tender Coconut\"\n },\n {\n \"code\": \"162\",\n \"name\": \"Seetapal\"\n },\n {\n \"code\": \"186\",\n \"name\": \"Rose(Loose))\"\n },\n {\n \"code\": \"187\",\n \"name\": \"Jasmine\"\n },\n {\n \"code\": \"188\",\n \"name\": \"Kakada\"\n },\n {\n \"code\": \"189\",\n \"name\": \"Chrysanthemum(Loose)\"\n },\n {\n \"code\": \"190\",\n \"name\": \"Astera\"\n },\n {\n \"code\": \"193\",\n \"name\": \"Marigold(Calcutta)\"\n },\n {\n \"code\": \"205\",\n \"name\": \"Rajgir\"\n },\n {\n \"code\": \"222\",\n \"name\": \"Ground Nut Seed\"\n },\n {\n \"code\": \"247\",\n \"name\": \"Elephant Yam(Suran)\"\n },\n {\n \"code\": \"253\",\n \"name\": \"Little gourd(Kundru)\"\n },\n {\n \"code\": \"254\",\n \"name\": \"Pointed gourd(Parval)\"\n },\n {\n \"code\": \"279\",\n \"name\": \"Plum\"\n },\n {\n \"code\": \"280\",\n \"name\": \"Pear(Marasebu)\"\n },\n {\n \"code\": \"281\",\n \"name\": \"Peach\"\n },\n {\n \"code\": \"303\",\n \"name\": \"Amla(Nelli Kai)\"\n },\n {\n \"code\": \"305\",\n \"name\": \"Onion Green\"\n },\n {\n \"code\": \"307\",\n \"name\": \"Mint(Pudina)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-16T23:59:59+05:30\"\n },\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.86850558,\n 18.49089838\n ]\n },\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:mandi-price:MH\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/MandiPrice/publish/ondemand-mh.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Discover - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/discover", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\" && @.resourceAttributes.supportedCommodities[*].code == \\\"23\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/MandiPrice/discover/discover.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Select", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/select", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\"\n },\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-09T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mandi-price:open-data\",\n \"resourceIds\": [\n \"resource:mandi-price:market:1634\"\n ],\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/MandiPrice/select/select.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "WeatherObservation Capability - MausamGram", + "description": "IMD MausamGram daily forecast. Select needs egress to mausamgram.imd.gov.in.", + "item": [ + { + "name": "Publish - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/provider/publish", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "provider", + "publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:weather-observation:mausamgram\",\n \"descriptor\": {\n \"code\": \"WEATHER_MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram daily weather forecast\",\n \"shortDesc\": \"On-demand five-day daily weather forecast for any point in the IMD numerical weather prediction domain.\",\n \"longDesc\": \"One Resource, not one per place. The Provider forecasts on a 0.25 degree grid and accepts any coordinate inside its domain, snapping it to the nearest grid centre, so there is no enumerable list of locations to publish. Consumers discover this Resource by falling inside its coverage area and then invoke the Provider with the point they care about to obtain a Direct WeatherObservation Resource per forecast day.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\",\n \"shortDesc\": \"Provider of IMD numerical weather prediction point forecasts.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"descriptor\": {\n \"code\": \"WEATHER_MAUSAMGRAM_DAILY_FORECAST\",\n \"name\": \"Five-day daily point weather forecast\",\n \"shortDesc\": \"Daily rainfall, temperature, humidity, wind and warning forecast for a requested coordinate.\",\n \"longDesc\": \"Invoke this Resource with a Point location to obtain one Direct WeatherObservation Resource per forecast day, each carrying that day's parameters. Values are bias-corrected model output, not station observations.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"source\": {\n \"sourceId\": \"mausamgram\",\n \"sourceName\": \"IMD Mausamgram NWP\",\n \"sourceUri\": \"https://mausamgram.imd.gov.in/\"\n },\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.5\n ],\n [\n 97.5,\n 6.5\n ],\n [\n 97.5,\n 37.5\n ],\n [\n 68.0,\n 37.5\n ],\n [\n 68.0,\n 6.5\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/WeatherObservation/publish/ondemand-mausamgram.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Discover - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/discover", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Weather\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/WeatherObservation/discover/discover.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Select", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/select", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:weather-observation:mausamgram:open-data\",\n \"resourceIds\": [\n \"resource:weather-observation:mausamgram:daily-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/WeatherObservation/select/select.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "KnowledgeAdvisory Capability - Bharat Vistaar", + "description": "Retrieval over the Bharat Vistaar corpus. A select answer is large -- four passages measured 15.7 KB.", + "item": [ + { + "name": "Publish - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/provider/publish", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "provider", + "publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:knowledge-advisory:knowledge\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_ADVISORY\",\n \"name\": \"OpenAgriNet knowledge advisory retrieval\",\n \"shortDesc\": \"On-demand retrieval of agricultural advisory passages from a curated document corpus.\",\n \"longDesc\": \"One Resource, not one per document. The Provider runs a retrieval query over an ingested corpus of extension literature, scheme documents and package-of-practice material, so there is no enumerable list of advisories to publish. Consumers discover this Resource and then invoke the Provider with the topics they are asking about, receiving one Direct KnowledgeAdvisory Resource per retrieved passage, each carrying the passage text as its recommendation and naming the document it came from as its source.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\",\n \"shortDesc\": \"Provider of retrieval over curated agricultural knowledge documents.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_ADVISORY_RETRIEVAL\",\n \"name\": \"Agricultural advisory retrieval\",\n \"shortDesc\": \"Ask a question about crops, practices, schemes, weather or markets and receive the relevant advisory passages.\",\n \"longDesc\": \"Invoke this Resource with the topics being asked about to obtain one Direct KnowledgeAdvisory Resource per retrieved passage. A request whose subjectCategories name Scheme and nothing else is answered from the narrower schemes corpus; any other combination is answered from the Provider's default corpus, because the request no longer states a question only the schemes corpus can answer.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\",\n \"Livestock\",\n \"Scheme\",\n \"Practice\",\n \"Weather\",\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"gu\"\n ],\n \"topics\": [\n \"Crop establishment\",\n \"Nutrient management\",\n \"Pest management\",\n \"Government schemes\",\n \"Package of practices\"\n ],\n \"source\": {\n \"sourceId\": \"knowledge-corpus\",\n \"sourceName\": \"OpenAgriNet curated knowledge corpus\",\n \"sourceUri\": \"https://openagrinet.github.io/network-specs/\"\n },\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.5\n ],\n [\n 97.5,\n 6.5\n ],\n [\n 97.5,\n 37.5\n ],\n [\n 68.0,\n 37.5\n ],\n [\n 68.0,\n 6.5\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/KnowledgeAdvisory/publish/ondemand-knowledge.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Discover - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/discover", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Crop\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/KnowledgeAdvisory/discover/discover.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Select", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/select", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"topics\": [\n \"cotton\",\n \"pink bollworm\",\n \"pest management\"\n ]\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:knowledge-advisory:knowledge:open-data\",\n \"resourceIds\": [\n \"resource:knowledge-advisory:knowledge:retrieval\"\n ],\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/KnowledgeAdvisory/select/select.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "AgricultureFacility Capability - PoCRA", + "description": "PoCRA facility locator, the one capability needing no credential. It takes ONE facility type per call and caches on messageId for ten minutes while ignoring location.", + "item": [ + { + "name": "Publish - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/provider/publish", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "provider", + "publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:agriculture-facility:pocra\",\n \"descriptor\": {\n \"code\": \"AGRI_FACILITY_POCRA\",\n \"name\": \"PoCRA agriculture facility locator\",\n \"shortDesc\": \"On-demand search for custom hiring centres, Krishi Vigyan Kendras, warehouses and soil testing laboratories across Maharashtra.\",\n \"longDesc\": \"One Resource, not one per facility. PoCRA aggregates facility registries from several Maharashtra departments and answers a search around a point, ranking results by distance, so there is no enumerable list of facilities to publish. Consumers discover this Resource by falling inside its coverage area and then invoke the Provider with a search origin and the facility types they want, receiving one Direct AgricultureFacility Resource per facility found.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\",\n \"shortDesc\": \"Maharashtra state aggregator for agricultural facility registries.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"descriptor\": {\n \"code\": \"AGRI_FACILITY_POCRA_SEARCH\",\n \"name\": \"Nearby agriculture facility search\",\n \"shortDesc\": \"Find custom hiring centres, Krishi Vigyan Kendras, warehouses and soil testing laboratories near a point.\",\n \"longDesc\": \"Invoke this Resource with a fulfillment stop carrying a Point location and the facility types wanted to obtain one Direct AgricultureFacility Resource per facility, each carrying its address, contact and, for warehouses, its reported storage capacity. Results are ranked by the Provider's own distance from the search origin; that distance is not carried on the Resource, because it is a property of the query rather than of the facility.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"languages\": [\n \"en\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"pocra\",\n \"sourceName\": \"PoCRA, Government of Maharashtra\",\n \"sourceUri\": \"https://mahapocra.gov.in/\"\n },\n \"supportedFacilityTypes\": [\n \"CustomHiringCentre\",\n \"KrishiVigyanKendra\",\n \"Warehouse\",\n \"SoilTestingFacility\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 72.6,\n 15.6\n ],\n [\n 80.9,\n 15.6\n ],\n [\n 80.9,\n 22.1\n ],\n [\n 72.6,\n 22.1\n ],\n [\n 72.6,\n 15.6\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/AgricultureFacility/publish/ondemand-pocra.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Discover - On-demand", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/discover", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Facility\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/AgricultureFacility/discover/discover.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + }, + { + "name": "Select", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{single-network-layer-host}}/select", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"supportedFacilityTypes\": [\n \"Warehouse\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agriculture-facility:pocra:open-data\",\n \"resourceIds\": [\n \"resource:agriculture-facility:pocra:facility-search\"\n ],\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Body is capability-examples/AgricultureFacility/select/select.json, unedited apart from the context variables." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));", + "", + "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", + "// status code alone does not tell you the call worked.", + "pm.test('not a NACK', () => {", + " const b = pm.response.json();", + " const err = (b.error) || (b.message && b.message.error);", + " if (err) throw new Error(`${err.code}: ${err.message}`);", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "Stack Health", + "description": "Run these first. Every one is a plain GET and none of them signs anything.", + "item": [ + { + "name": "single network layer adapter", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{single-network-layer-host}}/health", + "host": [ + "{{single-network-layer-host}}" + ], + "path": [ + "health" + ] + }, + "description": "One adapter process serves all three tiers, so there is one adapter health check. /health is registered by the adapter itself, outside the modules." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));" + ] + } + } + ] + }, + { + "name": "discovery", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{discovery-host}}/healthz", + "host": [ + "{{discovery-host}}" + ], + "path": [ + "healthz" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('HTTP 200', () => pm.response.to.have.status(200));" + ] + } + } + ] + } + ] + } + ] +} diff --git a/api-collection/OpenAgriNet.single-network-layer.local-environment.json b/api-collection/OpenAgriNet.single-network-layer.local-environment.json new file mode 100644 index 00000000..511511be --- /dev/null +++ b/api-collection/OpenAgriNet.single-network-layer.local-environment.json @@ -0,0 +1,146 @@ +{ + "id": "0a4e1c00-0000-4000-8000-0000000snle1", + "name": "OpenAgriNet local (single network layer)", + "values": [ + { + "key": "single-network-layer-host", + "value": "http://127.0.0.1:9200", + "enabled": true + }, + { + "key": "discovery-host", + "value": "http://127.0.0.1:8090", + "enabled": true, + "type": "default", + "description": "The catalogue index. Reached through an adapter, not directly, so only its health check is here. DISCOVERY_PORT in .env." + }, + { + "key": "registry-host", + "value": "http://127.0.0.1:8081", + "enabled": true, + "type": "default", + "description": "Participants and capability bindings, seeded by bin/setup.py. HOST ONLY: the API base is /api/v1, so a request spells it out. REGISTRY_PORT in .env." + }, + { + "key": "keycloak-host", + "value": "http://127.0.0.1:8080", + "enabled": true, + "type": "default", + "description": "Issues the registry write token, and authenticates nothing else -- it is Sunbird RC's, not the network's. KEYCLOAK_PORT in .env." + }, + { + "key": "keycloak-realm", + "value": "sunbird-rc", + "enabled": true, + "type": "default", + "description": "KEYCLOAK_REALM in .env." + }, + { + "key": "keycloak-client-id", + "value": "registry-frontend", + "enabled": true, + "type": "default", + "description": "The client that logs a user in. admin-api is the one that mints tokens and is not used from here. KEYCLOAK_CLIENT_ID in .env." + }, + { + "key": "registry-user", + "value": "no-user", + "enabled": true, + "type": "default", + "description": "REGISTRY_ADMIN_USER in .env. A shipped default, so it is public until changed." + }, + { + "key": "registry-password", + "value": "no-user-password", + "enabled": true, + "type": "default", + "description": "REGISTRY_ADMIN_PASSWORD in .env. A shipped default, so it is public until changed." + }, + { + "key": "consumer-participant-id", + "value": "consumer.oan.dev", + "enabled": true, + "type": "default", + "description": "The consumer adapter's network identity, and what its signatures verify against. CONSUMER_SUBSCRIBER_ID in .env." + }, + { + "key": "consumer-base-url", + "value": "https://consumer.oan.dev", + "enabled": true, + "type": "default", + "description": "The address PEERS are told to POST to, published in the registry row. Nothing in this stack resolves it, so a wrong value costs nothing locally and everything on a shared network -- and the registry cannot correct it. CONSUMER_BASE_URL in .env." + }, + { + "key": "consumer-participant-key", + "value": "", + "enabled": true, + "type": "default", + "description": "EMPTY ON PURPOSE. The public half lives in quick-start/keys/keys.json, which bin/setup.py generates; paste it in before registering. setup.py has already registered this participant, so it is normally not needed." + }, + { + "key": "mausamgram-participant-id", + "value": "mausamgram", + "enabled": true, + "type": "default", + "description": "IMD Mausamgram, the weather upstream. Half of its binding key. MAUSAMGRAM_PARTICIPANT_ID in .env." + }, + { + "key": "mausamgram-capability", + "value": "openagrinet:WeatherObservation", + "enabled": true, + "type": "default", + "description": "MAUSAMGRAM_CAPABILITY in .env." + }, + { + "key": "mausamgram-binding-key", + "value": "mausamgram|openagrinet:WeatherObservation", + "enabled": true, + "type": "default", + "description": "|. Used to look up the binding's osid. The provider adapter builds this from the incoming payload, so it must match the registry row exactly -- a mismatch is a 404." + }, + { + "key": "mausamgram-base-url", + "value": "https://mausamgram.imd.gov.in", + "enabled": true, + "type": "default", + "description": "MAUSAMGRAM_BASE_URL in .env." + }, + { + "key": "mausamgram-path", + "value": "/nwpapi/get-daily", + "enabled": true, + "type": "default", + "description": "MAUSAMGRAM_PATH in .env." + }, + { + "key": "mausamgram-mapping-url", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/0.0.1/quick-start/config/mappings/mausamgram/weather-observation.select.yaml", + "enabled": true, + "type": "default", + "description": "What the binding row publishes and the provider adapter fetches. A TAG ref, so what a running stack reads is fixed at 0.0.1 and a later push does not move it. MAUSAMGRAM_MAPPING_URL in .env." + }, + { + "key": "capability-code", + "value": "openagrinet:WeatherObservation", + "enabled": true, + "type": "default", + "description": "WHICH CAPABILITY THE 4. Capability Schemas FOLDER OPERATES ON -- the one variable to change to work on another, because the schemaUrl and the display name are derived from it rather than written down. The four this deployment serves are openagrinet:WeatherObservation, openagrinet:MandiPrice, openagrinet:KnowledgeAdvisory and openagrinet:AgricultureFacility; network-specs carries four more packs no capability uses yet." + }, + { + "key": "schema-pack-ref", + "value": "main", + "enabled": true, + "type": "default", + "description": "The ref in OpenAgriNet/network-specs that a derived schemaUrl points at. SCHEMA_PACK_REF in .env, which is what bin/setup.py seeds the rows from -- keep the two the same or a create here disagrees with a seeded row. MAIN IS A 404 FOR THESE PATHS TODAY; only schema-packs-v0.1 serves them. The registry checks the URL's shape and never fetches it, so a row stores either way, and main is the ref that becomes correct once the packs merge." + }, + { + "key": "schema-pack-version", + "value": "v0.1", + "enabled": true, + "type": "default", + "description": "The pack version, and the row's version field. All eight packs are v0.1 today. Per capability in setup.py rather than one setting for all four, because a capability can move to v0.2 while the others stay -- so if you point capability-code at one that has, change this too." + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_using": "hand-written, tracked in this repo" +} diff --git a/quick-start/.env.example b/quick-start/.env.example index 3b6d9a10..3736c2b8 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,6 +39,9 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 +# The single network layer adapter (the default topology). Inside the container +# it always listens on 9200; this only moves the host side. +SINGLE_NETWORK_LAYER_ADAPTER_PORT=9200 # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 @@ -211,3 +214,8 @@ POCRA_CAPABILITY=openagrinet:AgricultureFacility POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in POCRA_PATH=/search POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/0.0.1/quick-start/config/mappings/pocra/agriculture-facility.select.yaml + +# Which adapter topology a bare `docker compose up -d` starts: "single" (one +# adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes +# --profile explicitly and ignores this; it is for plain compose commands. +COMPOSE_PROFILES=single diff --git a/quick-start/Makefile b/quick-start/Makefile index 304d7e7e..e65f48ef 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -3,8 +3,10 @@ # startup order matters, and a good place to make the right order the shortest # thing to type. # -# make up the whole stack, in the order it has to start +# make up the whole stack (single network layer adapter), in order # make up-core the same minus the reverse proxy and hyperdx +# make up-multi the whole stack with three adapters instead of one +# make smoke publish, discover, select through what is running # make down stop everything, keep the data # make help the rest # @@ -13,7 +15,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core up-single-network-layer down-single-network-layer down destroy setup reverse-proxy observability pull restart restart-edge ps logs +.PHONY: help up up-core up-multi up-core-multi smoke down destroy setup reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -25,8 +27,9 @@ help: up: ; @$(STACK) up up-core: ; @$(STACK) up-core down: ; @$(STACK) down -up-single-network-layer: ; @$(STACK) up-single-network-layer -down-single-network-layer: ; @$(STACK) down-single-network-layer +up-multi: ; @$(STACK) up-multi +up-core-multi: ; @$(STACK) up-core-multi +smoke: ; @$(STACK) smoke destroy: ; @$(STACK) destroy # --------------------------------------------- the optional tiers, on their own diff --git a/quick-start/README.md b/quick-start/README.md index cbf67599..7405c877 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -1,7 +1,12 @@ # OAN quick-start -The whole OAN stack in Docker Compose: a registry, a discovery service, three -adapters, and two mock upstreams standing in for real provider APIs. +The whole OAN stack in Docker Compose: a registry, a discovery service, the +adapter tiers, and two mock upstreams standing in for real provider APIs. + +The three adapter tiers (consumer, network, provider) run as **one** container +by default: `single-network-layer-adapter`, the tiers as modules of one process +on port 9200. `make up-core-multi` runs them as three containers instead. Same +compose file, same keys, same registry → [Two topologies](#two-topologies). Nothing is built here — the images are pulled. Locally that is about ten minutes, most of it waiting for Keycloak. Discovery is the one exception, and @@ -25,6 +30,10 @@ where the reason lives. `publish` runs the other way: the provider adapter sends it to the network adapter, which files the catalogue in discovery. +In the default single topology each box above is a module of the one +`single-network-layer-adapter`, and the arrows between them are loopback calls +inside that container. + All three adapters read the registry — who signed this request, and where this capability's provider lives. @@ -43,6 +52,7 @@ below follow startup order, so they work top to bottom. | Observability | not started | optional, wants 2–4 GB | | Credentials | shipped defaults are ok | **must all be changed** | | Command | `make up-core` | `make up` | +| Three adapters instead | `make up-core-multi` | `make up-multi` | **Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. @@ -76,14 +86,15 @@ cp .env.example .env Locally, nothing in it needs changing. It publishes these on `localhost`: ``` -8081 registry 9200 provider adapter 9100 mock-imd -8080 keycloak 9201 network adapter 9101 mock-agmarknet -9990 keycloak admin 9202 consumer adapter -8090 discovery +8081 registry 9200 single network layer adapter (default) 9100 mock-imd +8080 keycloak or provider adapter (multi) 9101 mock-agmarknet +9990 keycloak admin 9201 network adapter (multi only) +8090 discovery 9202 consumer adapter (multi only) ``` If one is already taken, change it here — that is the only edit a local run -needs. Adapters reach each other by Compose service name, so it only changes +needs. The single adapter's host port is `SINGLE_NETWORK_LAYER_ADAPTER_PORT`; +in multi mode 9200 is `PROVIDER_ADAPTER_PORT`. Adapters reach each other by Compose service name, so it only changes what you type into Postman. ## Step 4 — Start it @@ -93,20 +104,31 @@ make up-core ``` Three tiers, in the only order that works: registry and discovery, then -`bin/setup.py`, then the mocks and the three adapters. Allow up to five minutes -the first time — Keycloak on a cold volume. +`bin/setup.py`, then the adapter: `single-network-layer-adapter` on 9200. +Allow up to five minutes the first time — Keycloak on a cold volume. -`setup.py` generates a keypair per adapter, registers seven participants and four -capability bindings, and renders the three adapter configs. Nothing needs -creating by hand. → Appendix C for why the order matters, Appendix D for what -it wrote. +Three adapters instead: `make up-core-multi`. Either target stops the other +topology's adapters first (both use 9200), so switching is just running the +other one. + +`setup.py` generates a keypair per tier, registers seven participants and four +capability bindings, and renders all four adapter configs (the three tier +configs and the single one). Nothing needs creating by hand. → Appendix C for +why the order matters, Appendix D for what it wrote. ```sh make ps +make smoke # publish, discover, select through whatever is running ``` ## Step 5 — Provider layer +> Steps 5–7 describe the three tiers. In the default single topology they are +> modules of one container, so read `single-network-layer-adapter` wherever a +> command names `provider-adapter`, `network-adapter` or `consumer-adapter` +> (e.g. `docker compose logs single-network-layer-adapter`). The `.env` keys +> are the same in both. + Answers `select`, and the only layer that calls an upstream. Serves all four capabilities from one adapter. @@ -154,12 +176,29 @@ straight to the provider layer, per `config/adapters/routing-consumer.yaml`. docker compose logs consumer-adapter | grep 'Server listening' ``` -`setup.py` renders all three configs from these keys. **Do not edit the +`setup.py` renders all four configs from these keys. **Do not edit the rendered `config/adapters/*.yaml`** — they are regenerated and hold private keys. Change `.env`, re-run `make up`. ## Step 8 — Verify end to end +Quickest, in either topology: + +```sh +make smoke +``` + +It starts the mocks, sends publish, discover and select to whichever adapters +are running, and prints each HTTP status and the tier hops from the adapter +log. All three should be 200. + +Two copies of the full collection, same requests, in `../api-collection/`: + +| Topology | Collection | Environment | +|---|---|---| +| single (`make up-core`, default) | `OpenAgriNet.single-network-layer.api-collection.json` | `OpenAgriNet.single-network-layer.local-environment.json` | +| multi (`make up-core-multi`) | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | + Import both files from `../api-collection/` into Postman — `OpenAgriNet.api-collection.json` and `OpenAgriNet.local-environment.json`. Select the environment, or every `{{...-host}}` resolves to nothing. Or: @@ -189,20 +228,32 @@ collection. → Appendix N. --- -## Single network layer (optional) +## Two topologies -The same network can run two ways. Both use the same registry, keys and data: +The three adapter tiers can run two ways. Same `docker-compose.yml`, same +image, same keys, registry and data; a compose profile picks the adapters. -| | Multi-adapter (default) | Single network layer | +| | Single network layer (default) | Multi-adapter | |---|---|---| -| What runs | `consumer-adapter`, `network-adapter`, `provider-adapter` | one `single-network-layer-adapter` (the single network layer adapter) | -| Compose file | `docker-compose.yml` | `docker-compose.single-network-layer.yml` | -| Start / stop | `make up` / `make down` | `make up-single-network-layer` / `make down-single-network-layer` | -| Tier-to-tier hops | HTTP between containers | HTTP over loopback (`http://localhost:9200`) inside the one container | -| Port | 9202 / 9201 / 9200 | `127.0.0.1:9200` (so only one mode runs at a time) | - -Use one or the other, not both: they share the project name and data volumes. -Both use `ADAPTER_IMAGE` from `.env`; nothing else needs setting. +| What runs | one `single-network-layer-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | +| Compose profile | `single` | `multi` | +| Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | +| Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | +| App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | +| Catalogue publishes to | `:9200/provider/publish` | `:9200/publish` | +| Tier-to-tier hops | loopback HTTP inside the container (`http://localhost:9200/...`) | HTTP between containers | + +Only one topology runs at a time (both use 9200). Switching is running the +other target: it stops the running topology's adapters first. `make down`, +`make ps` and `make logs` cover both. `MODE=multi` also works with `make up`, +`make up-core` and `make restart`. + +Plain compose, without make: `docker compose --profile single up -d` (or +`--profile multi`). A bare `docker compose up -d` follows `COMPOSE_PROFILES` +in `.env` (`single` in `.env.example`); without it no adapter starts. + +Signing is off in every quick-start template, in both topologies, so keep the +adapter ports on `127.0.0.1` (the compose file does). **Config.** `config/adapters/single-network-layer.yaml.tmpl`, beside the three tier templates. `bin/setup.py` fills it into `config/adapters/single-network-layer.yaml` @@ -223,8 +274,7 @@ The hops between modules are loopback HTTP calls to the same container. In the multi-adapter stack only the hosts differ. 9200 is the container's own listener and must match `http.port` in `single-network-layer.yaml.tmpl`. -Tools: -- `bin/single-network-layer-up.sh`: start and smoke-test (publish, discover, select). +Check it: `make smoke`. # Part 2 — Run it on a VM @@ -267,7 +317,9 @@ make up `make up`, not `make up-core`: two more tiers on top of Part 1's three — nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes -the VM reachable from the internet.** +the VM reachable from the internet.** It runs the single network layer +adapter; `make up-multi` runs the three adapters instead, and decides what the +proxy hosts in Appendix B forward to. ## Step V4 — Decide what is exposed @@ -286,7 +338,8 @@ ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ ``` The collection's defaults then work unchanged, since they already point at -loopback. +loopback. With the default single topology only 9200 is needed for the +adapter (`-L 9200:127.0.0.1:9200`); 9202 is for `make up-multi`. ## Step V6 — Observability (optional) @@ -398,6 +451,15 @@ Three hosts rather than one with path prefixes, so a rate limit or a block attaches to a hostname instead of a regex in a textarea, and each gets its own certificate. An unknown `Host` gets NPM's default page, not an adapter. +**Single topology (`make up`, the default).** The table above is for +`make up-multi`. With the single adapter there is no `consumer-adapter`, +`network-adapter` or `provider-adapter` to forward to: point the consumer host +at `single-network-layer-adapter`, port 9200, with the same Advanced paste. The +app's paths (`/discover`, `/select`, …) do not change. Do **not** make +`/network/…` or `/provider/publish` reachable from a public host: `/network/` is +an internal tier and `/provider/publish` publishes as the provider without +checking the caller, and signing is off in quick-start. + **A fourth host for HyperDX**, if the `observability` profile is up — this is what replaces `ssh -L 8085` for reading telemetry: @@ -550,8 +612,8 @@ loopback. A registry route would depend on that silently. ``` 1. registry and discovery (also sunbird-registry-postgres, sunbird-registry-keycloak, discovery-postgres) -2. bin/setup.py keys, seven participants, four bindings, three configs -3. mock upstreams, then the three adapters +2. bin/setup.py keys, seven participants, four bindings, four configs +3. the adapters single-network-layer-adapter (default), or the three with MODE=multi 4. nginx-proxy-manager the public edge — 80 and 443, all interfaces 5. hyperdx ClickStack ``` diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index dad0fdbc..4d04a5bd 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -640,7 +640,7 @@ def _render_one(name, template, substitutions): # ------------------------------------------------------ single network layer # # The single network layer runs the consumer, network and provider tiers as -# modules of ONE adapter (docker-compose.single-network-layer.yml). Its +# modules of ONE adapter (docker-compose.yml, profile "single"). Its # template, single-network-layer.yaml.tmpl, carries all three tiers' # placeholders: each module keeps its own tier identity, so both stacks use the # same keys and registry rows. diff --git a/quick-start/bin/single-network-layer-up.sh b/quick-start/bin/single-network-layer-up.sh deleted file mode 100755 index ccd93d0e..00000000 --- a/quick-start/bin/single-network-layer-up.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -# Start the single network layer -- ONE adapter for the consumer, network and -# provider tiers (the single network layer adapter), tier hops over loopback HTTP -- -# then prove it works. -# -# bin/single-network-layer-up.sh -# -# Image: ADAPTER_IMAGE from .env (the same as the multi-adapter stack). Port 9200. -# Uses docker-compose.single-network-layer.yml (and docker-compose.arm64.yml -# when present). Leaves the single network layer RUNNING. -# Stop it: make down-single-network-layer -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$ROOT" - -COMPOSE=(-f docker-compose.single-network-layer.yml) -[ -f docker-compose.arm64.yml ] && COMPOSE+=(-f docker-compose.arm64.yml) -envv() { grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-; } -PORT=9200 -IMAGE="$(envv ADAPTER_IMAGE)" - -step() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; } -die() { printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } - -[ -f .env ] || die ".env missing: cp .env.example .env and fill it in" -[ -n "$IMAGE" ] || die "no image: set ADAPTER_IMAGE in .env" -command -v python3 >/dev/null 2>&1 || die "python3 is not installed -- bin/setup.py needs it" -python3 -c 'import cryptography' >/dev/null 2>&1 \ - || die "the python 'cryptography' package is missing -- pip install cryptography" - -# --- 1. infrastructure ------------------------------------------------------ -step "1/4 registry, discovery, mocks (the mocks are for this script's local smoke test; make up-single-network-layer does not start them)" -docker compose "${COMPOSE[@]}" up -d sunbird-registry-service discovery-service mock-imd mock-agmarknet - -# --- 2. config -------------------------------------------------------------- -step "2/4 keys, registry entries, rendered config (bin/setup.py)" -python3 bin/setup.py >/dev/null -CFG=config/adapters/single-network-layer.yaml -[ -f "$CFG" ] || die "$CFG was not rendered (a directory there? rm -r it, then re-run)" -echo " $CFG rendered" - -# --- 3. adapter ------------------------------------------------------------- -step "3/4 single-network-layer-adapter on $IMAGE (loopback tier hops)" -# The multi-adapter containers share this project; only one topology at a time. -docker compose -f docker-compose.yml stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true -docker compose "${COMPOSE[@]}" up -d --force-recreate single-network-layer-adapter -for i in $(seq 1 60); do - curl -sf "http://127.0.0.1:$PORT/health" >/dev/null && break - [ "$i" = 60 ] && { docker logs --tail 30 single-network-layer-adapter; die "adapter not healthy after 60s"; } - sleep 1 -done -echo " image: $(docker inspect single-network-layer-adapter --format '{{.Config.Image}}')" -echo " health: $(curl -s "http://127.0.0.1:$PORT/health")" - -# --- 4. smoke test ---------------------------------------------------------- -step "4/4 publish, discover, select through the single network layer" -ROOT="$ROOT" PORT="$PORT" python3 - <<'PY' -import json, os, pathlib, sys, urllib.error, urllib.request, uuid -root = pathlib.Path(os.environ["ROOT"]); port = os.environ["PORT"] -env = {} -for line in (root / ".env").read_text().splitlines(): - if "=" in line and not line.lstrip().startswith("#"): - k, v = line.split("=", 1); env[k.strip()] = v.strip() -steps = (("publish", "provider/publish"), ("discover", "discover"), ("select", "select")) -rc = 0 -for step, path in steps: - f = next((root / "capability-examples" / "WeatherObservation" / step).glob("*.json")) - text = f.read_text() - for old, key in (("provider.oan.dev", "PROVIDER_SUBSCRIBER_ID"), - ("consumer.oan.dev", "CONSUMER_SUBSCRIBER_ID"), - ("network.oan.dev", "NETWORK_SUBSCRIBER_ID")): - text = text.replace(old, env[key]) - text = text.replace('"mausamgram"', json.dumps(env["MAUSAMGRAM_PARTICIPANT_ID"])) - body = json.loads(text) - body["context"]["transactionId"], body["context"]["messageId"] = str(uuid.uuid4()), str(uuid.uuid4()) - req = urllib.request.Request(f"http://127.0.0.1:{port}/{path}", json.dumps(body).encode(), - {"Content-Type": "application/json"}) - try: - with urllib.request.urlopen(req, timeout=20) as r: - status, data = r.status, r.read() - except urllib.error.HTTPError as e: - status, data = e.code, e.read() - print(f" {step:9} /{path:18} HTTP {status}") - if status != 200: - rc = 1 - print(" body:", data[:300].decode(errors="replace")) -sys.exit(rc) -PY -echo -echo " tier hops seen in the adapter log:" -docker logs --since 2m single-network-layer-adapter 2>&1 | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' - -printf '\n\033[1;32mSingle network layer running on %s. Port 127.0.0.1:%s\033[0m\n' "$IMAGE" "$PORT" diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 13fef204..9ffdc4a3 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -30,11 +30,37 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -# Both optional profiles, named explicitly. This matters for `down`: compose +# Every optional profile, named explicitly. This matters for `down`: compose # only acts on services whose profile is active, so a plain `docker compose -# down` leaves the gateway and hyperdx containers running and then reports -# success. Naming them on teardown is what makes "down" mean down. -PROFILES=(--profile reverse-proxy --profile observability) +# down` leaves the gateway, hyperdx and the adapters (both topologies are +# profiled) running and then reports success. Naming them on teardown is what +# makes "down" mean down. +PROFILES=(--profile reverse-proxy --profile observability --profile single --profile multi) + +# Adapter topology: "single" (default) runs single-network-layer-adapter, one +# process for all three tiers; "multi" runs provider-, network- and +# consumer-adapter. Both publish 127.0.0.1:9200, so only one runs at a time. +# Validated only by the commands that use it (check_mode), so an unrelated +# MODE in the shell never blocks `down`, `destroy` or `help`. +MODE="${MODE:-single}" +check_mode() { + case "$MODE" in single|multi) ;; *) die "MODE must be single or multi, not '$MODE'" ;; esac +} + +# The topology whose adapters are running now: single, multi, or empty. +running_mode() { + if [ "$(docker inspect -f '{{.State.Running}}' single-network-layer-adapter 2>/dev/null)" = true ]; then echo single + elif [ "$(docker inspect -f '{{.State.Running}}' consumer-adapter 2>/dev/null)" = true ]; then echo multi + fi +} + +adapters_for() { # $1 mode + case "$1" in + single) echo single-network-layer-adapter ;; + multi) echo provider-adapter network-adapter consumer-adapter ;; + esac +} +other_mode() { [ "$1" = single ] && echo multi || echo single; } # ------------------------------------------------------------------ output @@ -44,6 +70,10 @@ step() { printf '\n\033[1;36m==> [%s/%s] %s\033[0m\n' "$1" "$2" "$3"; } info() { printf ' %s\n' "$1"; } warn() { printf '\033[1;33mwarning: %s\033[0m\n' "$1" >&2; } die() { printf '\033[1;31mstack: %s\033[0m\n' "$1" >&2; exit 1; } +envv() { # $1 key, $2 default -- value from .env, else the default + local v; v=$(grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-) + echo "${v:-$2}" +} # -------------------------------------------------------------- preflight @@ -74,6 +104,7 @@ preflight() { # `up-core` below is the same thing minus steps 4 and 5, for when you want # neither a public port nor ClickHouse's memory. up() { + check_mode preflight up_registry_tier 5 up_setup 5 @@ -97,6 +128,7 @@ up() { # 16 GB -- this is the stack you can actually exercise, which is the reason the # profiles exist in the first place. up_core() { + check_mode preflight up_registry_tier 3 up_setup 3 @@ -125,9 +157,10 @@ up_registry_tier() { } # Generates the adapter keypairs, registers the three adapter identities, and -# renders config/adapters/{provider,network,consumer}.yaml from the .tmpl files -# beside them. Safe to re-run: keys come from keys/keys.json once it exists, -# and participants already registered are left alone. +# renders every adapter config (provider, network, consumer and +# single-network-layer) from the .tmpl files beside them. Safe to re-run: +# keys come from keys/keys.json once it exists, and participants already +# registered are left alone. up_setup() { step 2 "$1" "bin/setup.py -- keys, seven registry participants, adapter configs" python3 bin/setup.py @@ -135,17 +168,33 @@ up_setup() { # Only now do the bind-mounted config files exist. up_adapters() { - # The mocks are NOT started. All four capabilities point at real external - # upstreams, so starting them would pull two images and run two containers - # nothing calls. They are still defined in docker-compose.yml, so + # The mocks are NOT started here (`make smoke` starts them for its select + # test). All four capabilities point at real external upstreams, so + # starting them would pull two images and run two containers nothing + # calls. They are still defined in docker-compose.yml, so # `docker compose up -d mock-imd mock-agmarknet` brings them up for anyone # debugging a mapping without the real credentials -- but that also needs a # new participant id and base URL in .env plus a setup.py re-run, because # the registry cannot repoint an existing row. - step 3 "$1" "adapters (provider, network, consumer)" - # The single network layer adapter also publishes 9200: stop it first. - docker compose -f docker-compose.single-network-layer.yml stop single-network-layer-adapter >/dev/null 2>&1 || true - docker compose up -d provider-adapter network-adapter consumer-adapter + check_mode + local other f; other=$(other_mode "$MODE") + step 3 "$1" "adapters, mode $MODE ($(adapters_for "$MODE"))" + # The other topology also publishes 127.0.0.1:9200: stop it first. + # shellcheck disable=SC2046 + docker compose --profile "$other" stop $(adapters_for "$other") >/dev/null 2>&1 || true + # Never let compose create a directory where a bind-mounted config belongs. + for f in $( [ "$MODE" = single ] && echo single-network-layer || echo provider network consumer ); do + [ -f "config/adapters/$f.yaml" ] \ + || die "config/adapters/$f.yaml was not rendered (a directory there? rm -r it, then re-run)" + done + # shellcheck disable=SC2046 + docker compose --profile "$MODE" up -d --force-recreate $(adapters_for "$MODE") + # NPM resolves adapter names when nginx loads and caches the addresses, so + # a recreated or switched adapter 502s through it until it restarts. + if [ "$(docker inspect -f '{{.State.Running}}' nginx-proxy-manager 2>/dev/null)" = true ]; then + info "restarting nginx-proxy-manager so it re-resolves the adapter addresses" + docker compose --profile reverse-proxy restart nginx-proxy-manager >/dev/null + fi } done_banner() { @@ -167,46 +216,6 @@ done_banner() { NEXT } -# ----------------------------------------------- single network layer - -# The single network layer: ONE adapter runs the consumer, network and -# provider tiers as modules of one process (the single network layer adapter), with -# tier-to-tier hops over loopback HTTP inside the container. The multi-adapter -# containers are NOT started. `make up-single-network-layer` does not start the -# mocks either, as in `up`; bin/single-network-layer-up.sh does start them, for -# its local smoke test. -# -# Its stack is its own complete compose file (docker-compose.single-network-layer.yml), -# so docker-compose.yml is not touched. Passing -f overrides COMPOSE_FILE, so -# the Apple-silicon override is added here when it exists. -single_network_layer_compose() { - local files=(-f docker-compose.single-network-layer.yml) - [ -f docker-compose.arm64.yml ] && files+=(-f docker-compose.arm64.yml) - docker compose "${files[@]}" "$@" -} - -up_single_network_layer() { - preflight - step 1 3 "registry and discovery (single network layer compose file)" - single_network_layer_compose up -d sunbird-registry-service discovery-service - up_setup 3 - # Never let compose create a directory where the bind-mounted config file belongs. - [ -f config/adapters/single-network-layer.yaml ] \ - || die "config/adapters/single-network-layer.yaml was not rendered (is it a directory? rm -r it, then re-run)" - step 3 3 "single-network-layer-adapter (consumer + network + provider in one process)" - # The multi-adapter containers share this project; one topology at a time. - docker compose stop provider-adapter network-adapter consumer-adapter >/dev/null 2>&1 || true - single_network_layer_compose up -d --force-recreate single-network-layer-adapter - printf '\n\033[1;32m==> single network layer is up\033[0m on 127.0.0.1:9200\n' -} - -# Stops only the single network layer adapter; registry, discovery and data stay. -down_single_network_layer() { - step 1 1 "stopping single-network-layer-adapter only" - single_network_layer_compose stop single-network-layer-adapter - single_network_layer_compose rm -f single-network-layer-adapter -} - # ----------------------------------------------------------------- down # Containers and networks go; named volumes stay. So the registry's Postgres @@ -324,7 +333,6 @@ NEXT # It also excludes nginx-proxy-manager, whose routing table lives in a SQLite # database rather than in anything a restart would re-read. `restart-edge` is # the separate target for the one case that does need it. -APP_SERVICES=(registry discovery provider-adapter network-adapter consumer-adapter) # `restart`, not `up -d --force-recreate`. A restart keeps the container and # therefore its address, so NPM's cached proxy_pass targets stay valid -- a @@ -336,10 +344,14 @@ APP_SERVICES=(registry discovery provider-adapter network-adapter consumer-adapt # up is a changed image or a changed environment, both of which need the # container recreated -- use `make up` for those. restart_app() { - step 1 1 "restarting registry, discovery and the three adapters" + local running; running=$(running_mode) + [ -n "$running" ] && MODE=$running + check_mode + step 1 1 "restarting registry, discovery and the $MODE-mode adapters" info "keycloak, both databases and the edge are left alone" - docker compose restart "${APP_SERVICES[@]}" - docker compose ps --format 'table {{.Service}}\t{{.Status}}' + # shellcheck disable=SC2046 + docker compose --profile "$MODE" restart sunbird-registry-service discovery-service $(adapters_for "$MODE") + docker compose --profile "$MODE" ps --format 'table {{.Service}}\t{{.Status}}' } # ----------------------------------------------------------------- misc @@ -354,6 +366,66 @@ restart_edge() { docker compose --profile reverse-proxy restart nginx-proxy-manager } +# ---------------------------------------------------------------- smoke + +# publish -> discover -> select against whichever topology is running. Starts +# the mocks (select's upstream is mock-imd in the shipped .env). Exits non-zero +# on any non-200 and prints the tier hops from the adapter log. +smoke() { + local mode base_pub base_app svc s + mode=$(running_mode) + if [ "$mode" = single ]; then + base_app="http://127.0.0.1:$(envv SINGLE_NETWORK_LAYER_ADAPTER_PORT 9200)"; base_pub="$base_app/provider" + svc=single-network-layer-adapter + elif [ "$mode" = multi ]; then base_pub="http://127.0.0.1:$(envv PROVIDER_ADAPTER_PORT 9200)" + base_app="http://127.0.0.1:$(envv CONSUMER_ADAPTER_PORT 9202)" + svc="provider-adapter network-adapter consumer-adapter" + else + die "no adapter is running: make up-core (or make up-core-multi) first" + fi + step 1 2 "mocks (mock-imd, mock-agmarknet), and wait for the adapter" + docker compose up -d mock-imd mock-agmarknet >/dev/null + for s in $(seq 1 30); do curl -sf "$base_app/health" >/dev/null 2>&1 && break; sleep 1; done + curl -sf "$base_app/health" >/dev/null 2>&1 || die "$base_app/health does not answer after 30s: make logs SVC=${svc%% *}" + step 2 2 "publish, discover, select through the $mode topology" + ROOT="$ROOT" PUB="$base_pub" APP="$base_app" python3 - <<'PY' +import json, os, pathlib, sys, urllib.error, urllib.request, uuid +root = pathlib.Path(os.environ["ROOT"]) +env = {} +for line in (root / ".env").read_text().splitlines(): + if "=" in line and not line.lstrip().startswith("#"): + k, v = line.split("=", 1); env[k.strip()] = v.strip() +steps = (("publish", os.environ["PUB"] + "/publish"), + ("discover", os.environ["APP"] + "/discover"), + ("select", os.environ["APP"] + "/select")) +rc = 0 +for step, url in steps: + text = next((root / "capability-examples/WeatherObservation" / step).glob("*.json")).read_text() + for old, key in (("provider.oan.dev", "PROVIDER_SUBSCRIBER_ID"), + ("consumer.oan.dev", "CONSUMER_SUBSCRIBER_ID"), + ("network.oan.dev", "NETWORK_SUBSCRIBER_ID")): + text = text.replace(old, env[key]) + text = text.replace('"mausamgram"', json.dumps(env["MAUSAMGRAM_PARTICIPANT_ID"])) + body = json.loads(text) + body["context"]["transactionId"], body["context"]["messageId"] = str(uuid.uuid4()), str(uuid.uuid4()) + req = urllib.request.Request(url, json.dumps(body).encode(), {"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + status, data = r.status, r.read() + except urllib.error.HTTPError as e: + status, data = e.code, e.read() + except (urllib.error.URLError, ConnectionError) as e: + status, data = 0, str(e).encode() + print(f" {step:9} {url:42} HTTP {status}") + if status != 200 or b"NACK" in data: + rc = 1; print(" body:", data[:300].decode(errors="replace")) +sys.exit(rc) +PY + echo; echo " tier hops in the adapter log:" + for s in $svc; do docker logs --since 2m "$s" 2>&1; done \ + | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' || true +} + # Just step 2. Re-run it after editing a .tmpl, or to re-render configs that # were deleted. It is idempotent, so this is always safe. setup() { @@ -366,12 +438,14 @@ usage() { cat <<'USAGE' bin/stack.sh - up the whole stack: registry+discovery -> setup.py -> adapters + up the whole stack, single network layer (default): + registry+discovery -> setup.py -> single-network-layer-adapter -> gateway (public, 80/443) -> hyperdx (wants 16 GB) up-core steps 1-3 only. Nothing public, no ClickHouse. - up-single-network-layer docker-compose.single-network-layer.yml: registry+discovery - -> setup.py -> single-network-layer-adapter - down-single-network-layer stop and remove single-network-layer-adapter only + up-multi as up, with the three adapters (provider, network, consumer) + up-core-multi as up-core, with the three adapters + smoke publish, discover, select through whichever topology runs + (MODE=multi also works with up, up-core and restart) down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only @@ -389,8 +463,9 @@ USAGE case "${1:-}" in up) up ;; up-core) up_core ;; - up-single-network-layer) up_single_network_layer ;; - down-single-network-layer) down_single_network_layer ;; + up-multi) MODE=multi; up ;; + up-core-multi) MODE=multi; up_core ;; + smoke) smoke ;; down) down ;; destroy) destroy ;; setup) setup ;; diff --git a/quick-start/config/adapters/single-network-layer.yaml.tmpl b/quick-start/config/adapters/single-network-layer.yaml.tmpl index b37d5b5e..d070ff43 100644 --- a/quick-start/config/adapters/single-network-layer.yaml.tmpl +++ b/quick-start/config/adapters/single-network-layer.yaml.tmpl @@ -1,5 +1,5 @@ # Single network layer adapter: the consumer, network and provider tiers as -# four modules of ONE adapter process (docker-compose.single-network-layer.yml). +# four modules of ONE adapter process (docker-compose.yml, profile "single"). # # / the consumer (bap); the app posts here, unsigned # /network/ verifies and forwards to discovery diff --git a/quick-start/docker-compose.single-network-layer.yml b/quick-start/docker-compose.single-network-layer.yml deleted file mode 100644 index cdadc5a0..00000000 --- a/quick-start/docker-compose.single-network-layer.yml +++ /dev/null @@ -1,929 +0,0 @@ -# The whole OAN stack in one file, SINGLE NETWORK LAYER variant: registry, -# discovery, ONE adapter running the consumer, network and provider tiers as -# modules of one process (the single network layer adapter), the telemetry stack and -# the public edge. -# -# This is docker-compose.yml with the three adapter services replaced by one -# `single-network-layer-adapter`. Everything else is the same, so use ONE of -# the two files, not both: they share the project name, container names and -# data volumes. -# -# make up-single-network-layer (or: bin/single-network-layer-up.sh) -# docker compose -f docker-compose.single-network-layer.yml up -d -# -# Tier-to-tier hops go over loopback HTTP to the container's own listener -# (http://localhost:9200): they never leave the container. One routing file per -# module, as in the multi-adapter stack -- only the hosts differ. Any adapter -# image works. -# -# The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among -# them authenticates nothing but that registry, and shares its postgres. -# -# Read it in tiers -- the banner comments below are the structure: -# -# registry sunbird-registry-postgres, sunbird-registry-keycloak, -# sunbird-registry-service -# discovery discovery-postgres, discovery-service -# adapter single-network-layer-adapter (consumer + network + provider) -# observability hyperdx profile: observability -# edge nginx-proxy-manager profile: reverse-proxy -# -# The last two are behind profiles because neither is needed to exercise the -# stack, and HyperDX is the heaviest thing here. -# -# ---------------------------------------------------------------- bringing up -# -# 1. cp .env.example .env change every credential -- they are -# shipped defaults -# 2. docker compose -f docker-compose.single-network-layer.yml up -d sunbird-registry-service discovery-service -# everything EXCEPT the adapter -# 3. python3 bin/setup.py keys, the three tier identities, and -# config/adapters/single-network-layer.yaml -# 4. docker compose -f docker-compose.single-network-layer.yml up -d now the adapter -# 5. docker compose -f docker-compose.single-network-layer.yml --profile reverse-proxy --profile observability up -d -# the edge and the telemetry stack -# -# Naming registry and discovery in step 2 is not tidiness. An adapter config -# is a bind-mounted FILE, and Docker creates a DIRECTORY at any bind-mount -# source that does not exist yet -- so starting an adapter before step 3 both -# wedges that container on a directory it cannot parse and leaves a directory -# where step 3 needs to write a file. -# -# Nothing is built here. Every image is pulled from the tags named in .env. -# -# ------------------------------------------------------------- what is public -# -# Exactly one container publishes on a routable interface: Nginx Proxy -# Manager, on 80 and 443. Everything else -- both Postgres instances, -# Keycloak, the registry, discovery, the HyperDX UI, and NPM's own admin UI on -# 81 -- publishes on 127.0.0.1 and is reachable only through an SSH tunnel: -# -# The HyperDX UI is the one item on that list with a second way in: it also has -# a proxy host, so it is readable at https:/// with the -# Access List's username and password. Its loopback publish is unchanged -- -# routing it did not open a port here. -# -# ssh -L 81:127.0.0.1:81 -L 8080:127.0.0.1:8080 \ -# -L 8081:127.0.0.1:8081 -L 8085:127.0.0.1:8085 -N you@the-vm -# -# The loopback binds are written literally rather than taken from a BIND_ADDR -# variable, which is what they used to be. One variable that moves the whole -# stack onto a public interface is a footgun once an edge exists to do that -# job deliberately for the one tier that should be reachable: behind those -# ports are a Keycloak whose admin password ships as a default and a registry -# whose write token anyone who has read .env.example can mint. -# -# ------------------------------------------------------------------- networks -# -# Two, and with a UI-configured proxy at the edge the split stops being -# decoration and becomes the actual control. NPM's "Forward Hostname" is a -# free-text field, so anyone with the admin password can point a public route -# at `sunbird-registry-service` or `sunbird-registry-keycloak`. NPM sits only on -# oan-edge, where neither name resolves and neither address is routable -- single-network-layer-adapter and hyperdx -# are the only containers it shares a network with. Those three straddle both -# networks; everything else is internal-only. -# -# Neither is `internal: true`: the adapter and discovery fetch the Beckn spec -# from raw.githubusercontent.com at boot, the provider adapter calls an -# upstream API that lives outside this VM, and NPM has to reach Let's Encrypt. -# Cutting egress would break all three. - -# The stack's name, pinned rather than derived from the directory. Compose -# labels every container with it and prefixes every volume with it, so without -# this line the deployment is named after whatever the checkout is called: -# `quick-start_registry-data` names a directory, not a network. Pinning it also -# means the directory can be renamed without orphaning the registry's data. -name: openagrinet - -networks: - oan-internal: - name: oan-internal - oan-edge: - name: oan-edge - -volumes: - sunbird-registry-postgres-data: - discovery-postgres-data: - hyperdx-clickhouse-data: - nginx-proxy-manager-data: - nginx-proxy-manager-letsencrypt: - -x-logging: &logging - # Every container's stdout, tagged so the log console can name it. - # - # The driver is unchanged -- json-file is what Docker was already using. The - # two additions are: - # - # `labels`, which makes the json-file driver write the compose service name - # into each line's `attrs` field. Compose sets that label on every container - # already; without this opt it stays in the container's metadata, where the - # collector reading the log FILE cannot see it. config/observability/filelog.yaml - # turns it into service.name, and it is the only reason HyperDX can show - # "discovery-service" rather than 64 hex characters. - # - # `max-size`/`max-file`, which bound the files those logs accumulate in. - # Docker's default is unbounded, so today a chatty service fills the disk and - # takes Postgres down with it. Unrelated to telemetry and fixed here because - # this is the change that made the log files matter. - driver: json-file - options: - labels: com.docker.compose.service - max-size: "10m" - max-file: "3" - -x-adapter: &adapter - # Pulled, never built. Set ADAPTER_IMAGE in .env to the tag published for - # this environment -- `docker compose up -d` is the whole deployment step. - image: ${ADAPTER_IMAGE} - pull_policy: missing - restart: unless-stopped - # The adapter is the only service on oan-edge besides hyperdx, and therefore - # one of the only ones NPM can reach. It still needs oan-internal to read the - # registry and to call discovery, so it sits on both. - networks: [oan-internal, oan-edge] - logging: *logging - -x-adapter-env: &adapter-env - CONFIG_FILE: /app/config/adapter.yaml - # OpenTelemetry. These ARE read: the otelsetup plugin builds gRPC exporters - # and the SDK applies its OTEL_EXPORTER_OTLP_* environment config before any - # option the plugin passes. - # - # OTEL_EXPORTER_OTLP_INSECURE IS LOAD-BEARING, and the reason is worth - # knowing because the symptom names TLS and the cause does not. - # - # The SDK derives transport security from the endpoint's SCHEME: - # - # switch u.Scheme { case "http", "unix": WithInsecure(); default: WithSecure() } - # - # "default" includes an EMPTY scheme. OTLP_ENDPOINT is a bare host:port, - # because that is what the plugin's own otlpEndpoint config field takes -- - # so without the scheme prefix below, or this flag, the exporter dials TLS - # at hyperdx's plaintext 4317 and logs, once per export interval: - # - # failed to upload metrics: ... authentication handshake failed: - # tls: first record does not look like a TLS handshake - # - # The flag is set explicitly rather than left to the scheme, so a future - # edit to the URL cannot silently turn TLS back on. - # - # This does not weaken anything that was encrypted: the collector is on - # oan-internal, reachable from nowhere else, and serves plaintext OTLP. - # - # There is also a bug underneath, in the plugin rather than here: it passes - # insecure credentials via WithDialOption, and NewGRPCConfig appends its own - # default TLS credentials AFTER user options, so gRPC takes the TLS ones and - # the plugin's setting is silently discarded. The env var is what actually - # controls this until that is fixed upstream. - # - # OTEL_EXPORTER_OTLP_PROTOCOL is deliberately absent. The plugin constructs - # gRPC exporters in code, so a protocol preference here is ignored and only - # suggests the transport is configurable from this file. - OTEL_EXPORTER_OTLP_ENDPOINT: http://${OTLP_ENDPOINT:-hyperdx:4317} - OTEL_EXPORTER_OTLP_INSECURE: "true" - -services: - # ========================================================================== - # registry -- who is on the network, their public keys, and which upstream - # API answers which capability. - # ========================================================================== - sunbird-registry-postgres: - image: postgres:14 - container_name: sunbird-registry-postgres - restart: unless-stopped - logging: *logging - networks: [oan-internal] - environment: - POSTGRES_DB: ${REGISTRY_DB_NAME} - POSTGRES_USER: ${REGISTRY_DB_USER} - POSTGRES_PASSWORD: ${REGISTRY_DB_PASSWORD} - volumes: - - sunbird-registry-postgres-data:/var/lib/postgresql/data - # No published port at all, not even on loopback: psql runs through - # `docker compose exec sunbird-registry-postgres`, so publishing one would only widen - # the surface without adding a way in. - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${REGISTRY_DB_USER}"] - interval: 5s - timeout: 5s - retries: 20 - - sunbird-registry-keycloak: - image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest - container_name: sunbird-registry-keycloak - restart: unless-stopped - logging: *logging - networks: [oan-internal] - volumes: - - ./config/registry/imports:/opt/jboss/keycloak/imports - environment: - - DB_VENDOR=postgres - - DB_ADDR=sunbird-registry-postgres - - DB_PORT=5432 - - DB_DATABASE=${REGISTRY_DB_NAME} - - DB_USER=${REGISTRY_DB_USER} - - DB_PASSWORD=${REGISTRY_DB_PASSWORD} - - KEYCLOAK_USER=${KEYCLOAK_ADMIN_USER} - - KEYCLOAK_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD} - - KEYCLOAK_IMPORT=/opt/jboss/keycloak/imports/realm-export.json - - PROXY_ADDRESS_FORWARDING=true - ports: - # Loopback only. This is an admin console with an imported realm, and - # the tunnel is how you reach it. - - "127.0.0.1:${KEYCLOAK_PORT}:8080" - - "127.0.0.1:${KEYCLOAK_ADMIN_PORT}:9990" - depends_on: - sunbird-registry-postgres: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:9990/ || exit 1"] - interval: 10s - timeout: 10s - retries: 30 - - sunbird-registry-service: - image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_IMAGE_TAG} - container_name: sunbird-registry-service - restart: unless-stopped - logging: *logging - # oan-internal ONLY, and that is the decision, not an omission. NPM sits - # on oan-edge alone, so it cannot resolve the name `registry` -- a proxy - # host pointed here fails to start rather than quietly working. Attaching - # oan-edge is the one edit that would make the registry publishable, so it - # is the one edit to refuse. - # - # Why not route it. POST /api/v1/Participant/search takes NO token, which - # makes it tempting: it is the one call a network peer needs. But - # SunbirdRC uses POST for both reads and writes -- /Participant/search - # reads, /Participant creates -- so no method rule separates them, and a - # route publishes the write surface the moment anything can mint a token. - # Today nothing can, for one reason only: Keycloak publishes on - # 127.0.0.1. That is a decision forty lines up, and a route here would - # silently depend on it. - # - # What this costs: nothing outside the VM can write a registry row, which - # is why bin/setup.py seeds all seven participants and all four bindings from - # inside. It is also why the Postman collection has no registry request -- - # there is no way for one to work. - # - # Reaching it for debugging is an SSH tunnel to ${REGISTRY_PORT}. - networks: [oan-internal] - volumes: - # Schemas are read at startup, so a change here needs this service - # restarted before the registry will honour it. - - ./config/registry/schemas:/home/sunbirdrc/config/public/_schemas - environment: - - connectionInfo_uri=jdbc:postgresql://sunbird-registry-postgres:5432/${REGISTRY_DB_NAME} - - connectionInfo_username=${REGISTRY_DB_USER} - - connectionInfo_password=${REGISTRY_DB_PASSWORD} - - search_providerName=dev.sunbirdrc.registry.service.NativeSearchService - - authentication_enabled=true - - sunbird_sso_realm=${KEYCLOAK_REALM} - - sunbird_sso_url=http://sunbird-registry-keycloak:8080/auth - # Without this the service resolves its issuer to localhost -- which - # inside the container is itself -- and the security filter chain fails - # to build, so the whole registry never starts. - - OAUTH2_RESOURCES_0_URI=http://sunbird-registry-keycloak:8080/auth/realms/${KEYCLOAK_REALM} - - OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH=realm_access.roles - - identity_provider=dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl - - sunbird_sso_admin_client_id=${KEYCLOAK_ADMIN_CLIENT_ID} - - sunbird_sso_client_id=${KEYCLOAK_CLIENT_ID} - - sunbird_sso_admin_client_secret=${KEYCLOAK_ADMIN_CLIENT_SECRET} - - sunbird_keycloak_user_set_password=true - - sunbird_keycloak_user_password=${KEYCLOAK_DEFAULT_USER_PASSWORD} - - encryption_enabled=false - - event_enabled=false - - idgen_enabled=false - - claims_enabled=false - - did_enabled=false - - signature_enabled=false - - certificate_enabled=false - - filestorage_enabled=false - - notification_enabled=false - - notification_async_enabled=false - - async_enabled=false - - webhook_enabled=false - - registry_base_apis_enable=false - - manager_type=DefinitionsManager - - expand_reference=false - - swagger_title=OAN Registry - - logging.level.root=INFO - ports: - # Loopback only, and bin/setup.py talks to exactly this: it derives - # http://localhost:${REGISTRY_PORT} and runs on the VM. - - "127.0.0.1:${REGISTRY_PORT}:8081" - depends_on: - sunbird-registry-postgres: - condition: service_healthy - sunbird-registry-keycloak: - condition: service_healthy - healthcheck: - # This image ships no curl, wget, nc or bash, so the check reads the - # kernel's own table instead: 1F91 is 8081 in hex, and the service binds - # v6, hence tcp6. It proves the port is accepting connections, which is - # what everything downstream waits for. - test: ["CMD-SHELL", "grep -q ':1F91' /proc/net/tcp6"] - interval: 5s - timeout: 5s - retries: 60 - start_period: 40s - - # ========================================================================== - # discovery -- catalogue search, and its own Postgres. - # ========================================================================== - discovery-postgres: - # pgvector rather than plain postgres: the discovery service's HNSW index - # options arrived in 0.8. - image: pgvector/pgvector:0.8.0-pg16 - container_name: discovery-postgres - restart: unless-stopped - logging: *logging - networks: [oan-internal] - environment: - # Not read from .env, unlike the registry's pair, and so not covered by - # the credential rotation .env.example asks for. That is tolerable only - # because this database publishes no port and sits on an internal - # network; if it ever needs to be reachable, these move to .env first. - POSTGRES_USER: discovery - POSTGRES_PASSWORD: discovery - POSTGRES_DB: discovery - volumes: - - discovery-postgres-data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U discovery -d discovery"] - interval: 5s - timeout: 5s - retries: 20 - - discovery-service: - image: ${DISCOVERY_IMAGE} - pull_policy: missing - - # THE ONLY SERVICE HERE THAT CAN BE BUILT, and the exception is earned. It - # is never built by `make up` -- `image:` above still decides what runs, and - # a build only happens when you ask for one: - # - # DISCOVERY_IMAGE=discovery-service:local docker compose build discovery-service - # - # Why this one and not the adapters or the mocks: the published tag went - # stale in a way no error reports. Measured on 2026-09-10, - # ghcr.io/nisargabd/discovery-service:latest was built before the telemetry - # package existed -- its embedded build info lists no go.opentelemetry.io - # dependency at all. Against that image every OTEL_* setting below is - # accepted, nothing is exported, and the stack looks entirely healthy. A - # build you can run locally is what makes that falsifiable. - # - # The context defaults to a sibling checkout because that is where it sits - # for anyone who has both repos; point DISCOVERY_CONTEXT elsewhere if yours - # differs. Without that checkout the build fails on a missing directory, - # which is the correct failure -- there is no source to build. - build: - context: ${DISCOVERY_CONTEXT:-../../discovery-service} - - # PUBLIC BASE IMAGES, deliberately overriding the Dockerfile's own - # defaults. It defaults to dhi.io/* -- Docker Hardened Images, what CI - # and every shipped image use -- and those 401 on an anonymous pull. A - # quick-start reader is precisely the person without that subscription, - # so the public equivalents the Dockerfile documents are the right - # default HERE and the wrong one there. INSTALL_BUILD_TOOLS comes with - # them and is not optional: h3-go is a cgo binding around Uber's C - # library, DHI's -dev variant bundles a C toolchain and golang:alpine - # does not. - args: - BUILD_IMAGE: ${DISCOVERY_BUILD_IMAGE:-golang:1.27-alpine} - RUNTIME_IMAGE: ${DISCOVERY_RUNTIME_IMAGE:-gcr.io/distroless/static-debian12:nonroot} - INSTALL_BUILD_TOOLS: "true" - - # Bare names, so each is passed through from the environment if set and - # left to the Dockerfile's own default if not. No values invented here: - # compose cannot run `git describe`, and a stamp this file made up would - # be worse than an absent one. - # - # Unset, the binary reports `dev unknown 1970-01-01T00:00:00Z unknown` - # and its telemetry Resource carries that verbatim in build.commit, - # build.date and build.tree_state. Honest for a local build. To stamp - # one, build it in the discovery-service repo instead -- `make docker` - # is what derives all four -- or set them in the environment first. - # - # A key with NO VALUE is compose's pass-through form. It is not the same - # as `VERSION: ""`, which would pass an empty string and defeat the - # Dockerfile's default. - VERSION: - COMMIT: - BUILD_DATE: - TREE_STATE: - - container_name: discovery-service - restart: unless-stopped - logging: *logging - networks: [oan-internal] - environment: - DATABASE_URL: postgres://discovery:discovery@discovery-postgres:5432/discovery?sslmode=disable - DATABASE_AUTO_MIGRATE: "true" - # APP_NETWORK_ID is the discovery image's own variable, so the left side - # cannot be renamed with the rest -- only the .env name it reads from. - APP_NETWORK_ID: ${BECKN_NETWORK_ID} - SERVER_PORT: 8080 - VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} - - # ---- the publish geometry budget -------------------------------------- - # - # How many GeoJSON shapes ONE catalog may contribute to the spatial index. - # Past it the publish still answers 200, with status PARTIAL and a fault - # per shape naming its path -- so nothing is dropped in silence, but those - # shapes are not searchable. Set here rather than left to the image's own - # 256 because it is the one discovery setting whose right value depends - # entirely on what you publish, and the failure it causes reads as missing - # data rather than as configuration. - # - # Count SHAPES, not resources: the walker recognises GeoJSON by shape and - # not by field name, at any depth. A mandi catalog carrying a Point under - # both coverageAreas and market.location spends TWO per market -- 273 - # Maharashtra markets published that way need 546, and under 256 only the - # first 128 markets are findable by place. - GEO_MAX_GEOMETRIES_PER_CATALOG: ${DISCOVERY_MAX_GEOMETRIES_PER_CATALOG:-256} - - # ---- search ----------------------------------------------------------- - # - # OFF, against the image's own default of true. With it off a search - # matches on the structured index alone -- domain, capability and - # geometry -- and free-text terms stop contributing to what comes back. - # SEARCH_ENABLE_TEXT_SEARCH is the image's variable name, so the left - # side is its contract and only the value here is ours. - SEARCH_ENABLE_TEXT_SEARCH: "false" - - # ---- rate limit ------------------------------------------------------- - # - # EFFECTIVELY OFF, against the image's own default of 20 rps / burst 40. - # That default is right for a shared deployment and wrong for this one: - # a quick-start is read by someone running a scripted sweep -- setup.py, - # a Postman collection, a bulk publish, a benchmark loop -- and 20 rps is - # under all of them. What they get is AUT_RATE_LIMITED, and a 429 NACK - # carries no `catalogs`, so a throttled sweep reads as an empty result - # rather than as backpressure. Measured on 2026-09-12: a plain 20-request - # loop against /discover tripped it. - # - # NOT ZERO, and it cannot be: the boot requires burst >= rps > 0, on the - # same argument as the geometry budget above -- zero reads either as - # "unlimited" or as "refuse everything", and the config layer will not - # guess which. A number nothing reaches is the honest way to say off. - RATE_LIMIT_RPS: ${DISCOVERY_RATE_LIMIT_RPS:-100000} - RATE_LIMIT_BURST: ${DISCOVERY_RATE_LIMIT_BURST:-100000} - - # ---- telemetry -------------------------------------------------------- - # - # This block used to say the exporter was parsed and unread, and that - # `none` was therefore the honest default. That stopped being true when - # the telemetry package landed: the service exports spans and two pgxpool - # instruments over OTLP/gRPC, straight to hyperdx like the adapters. - # - # WHAT DOES NOT ARRIVE, so nobody goes looking for it: the five OAN - # `metric.code` streams -- the two *_total_count and the three *_percent. - # They are not instrumented in the service at all; they are derived from - # the exported span stream by a collector running the discovery-service - # repo's otel/collector.yaml, and this stack does not run one. Traces - # arrive, everything looks healthy, and those five are simply absent. - # That is a quick-start simplification, not the deployment shape -- see - # Appendix H. - OTEL_EXPORTER: ${DISCOVERY_OTEL_EXPORTER:-otlp} - - # NO SCHEME, and that is the opposite of the adapters two screens up -- - # worth reading beside the note on the x-adapter anchor, because the same - # bare host:port means opposite things on either side. - # - # This service does not consult the SDK's env lookup; it passes the - # endpoint in explicitly, having branched on whether it has a scheme: - # - # if hasScheme(endpoint) { WithEndpointURL(endpoint) } - # else { WithEndpoint(endpoint), WithInsecure() } - # - # So a bare host:port is plaintext HERE, where on the adapters a bare - # host:port is what silently selects TLS. Adding http:// to this line - # would not make it more explicit; it would route through the URL branch. - # OTEL_EXPORTER_OTLP_INSECURE is absent for the same reason: unread. - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-hyperdx:4317} - - # BOTH ARE REQUIRED once the exporter is otlp, and their absence fails - # the boot rather than degrading -- the service refuses to export a - # stream it cannot attribute. That refusal is why they are here: before - # this, flipping DISCOVERY_OTEL_EXPORTER=otlp would have stopped the container - # starting, naming a variable this file never set. - # - # APP_SUBSCRIBER_ID is the Resource's `producer` and every span's - # recipient.id. Empty means the whole stream lands under no participant, - # which is worse than no stream: an empty grouping key reads as data - # rather than as an absence. - # - # APP_DOMAIN is checked against the sector list the build declares, so a - # typo fails the boot instead of splitting this deployment's telemetry - # into a group of one. - APP_SUBSCRIBER_ID: ${DISCOVERY_SUBSCRIBER_ID} - APP_DOMAIN: ${DISCOVERY_DOMAIN:-Agriculture} - - # OTEL_SERVICE_NAME is deliberately absent, and it WAS SET HERE on both - # sides of this merge -- to `oan-discovery` before the container rename - # and to `discovery` after it. The service reads neither: `service.name` - # is a compile-time constant, `discovery-service`, because it says what - # SOFTWARE this is and so must not vary by deployment. Confirmed against - # a live stack on 2026-09-10 -- spans arrived as `discovery-service` with - # nothing set. The variable was worse than inert: it named a value you - # would then go hunting for in HyperDX and never find, and the rename - # gave it a second wrong value to hunt for. What varies per deployment is - # `producer`, from APP_SUBSCRIBER_ID above. - # config/common.yaml is baked into the image and is the reviewed default. - # To override a setting, copy config/discovery/instance.yaml.example to - # config/discovery/instance.yaml and uncomment the mount below -- compose - # creates a DIRECTORY if you mount a file that does not exist, which the - # service then fails to parse, so the file has to be there first. - # volumes: - # - ./config/discovery/instance.yaml:/app/config/instance.yaml:ro - ports: - # Loopback only. Discovery is reached by the network adapter over - # oan-internal; this publish exists for curl through the tunnel. - - "127.0.0.1:${DISCOVERY_PORT}:8080" - depends_on: - discovery-postgres: - condition: service_healthy - - # ========================================================================== - # adapter -- single-network-layer-adapter: the consumer, network and provider tiers as - # modules of one process, one image, one config. - # - # Its config is a bind-mounted FILE that does not exist until bin/setup.py - # renders it from the tier templates beside it, which is what the step ordering - # above is about. - # ========================================================================== - - # ========================================================================== - # mock upstreams -- stand-ins for the real provider APIs, so the stack can be - # exercised without their credentials. - # - # Pulled like everything else. The sources are in mock-server/ to be built and - # published once, not built here -- see mock-server/README.md. - # - # oan-internal only: an upstream is called BY the provider adapter and is - # never reached from outside, so NPM has no business seeing it. The published - # ports are for looking at the mock directly while debugging. - # - # Both reproduce the awkward parts of the services they stand in for on - # purpose. A tidy mock would let a mapping pass here and fail in the real - # deployment. - # ========================================================================== - mock-imd: - image: ${MOCKIMD_IMAGE} - pull_policy: missing - container_name: mock-imd - restart: unless-stopped - logging: *logging - networks: [oan-internal] - command: - - "-addr=:9100" - # How many forecast days it answers with. The mapping reads however many - # arrive, so this is the knob for checking that it does. - - "-days=${MOCKIMD_DAYS:-3}" - ports: - - "127.0.0.1:${MOCKIMD_PORT}:9100" - - mock-agmarknet: - image: ${MOCKAGMARKNET_IMAGE} - pull_policy: missing - container_name: mock-agmarknet - restart: unless-stopped - logging: *logging - networks: [oan-internal] - command: - - "-addr=:9101" - # It answers 401 without this token, which is what proves the adapter - # sent one rather than the call happening to work anyway. - - "-token=${MOCKAGMARKNET_TOKEN:-local-mandi-token}" - - "-days=${MOCKAGMARKNET_DAYS:-2}" - ports: - - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" - - # The single network layer: the consumer, network and provider tiers as - # modules of ONE adapter process (the single network layer adapter), on ONE - # config rendered by bin/setup.py from config/adapters/single-network-layer.yaml.tmpl - # (config/adapters/single-network-layer.yaml), with one routing file per module. Tier-to-tier hops go over loopback (http://localhost:9200), - # inside the container, so there is no consumer-adapter, network-adapter or - # provider-adapter here. - # - # :9200 inside and on 127.0.0.1 (ADAPTER_IMAGE, same as the multi-adapter stack). - # The multi-adapter provider-adapter uses 9200 too: run one stack at a time. - # / the consumer, the caller's entry (bap), takes unsigned requests - # /network/* hands publish on to discovery - # /provider/* calls the upstream, answers synchronously - # /provider/publish sends the provider's catalogue to /network/publish - # - # Each module carries its own tier identity (consumer, network, provider): - # the same keys and registry rows as the multi-adapter stack. Signing is off, - # as in the multi-adapter quick-start templates, so /network/ and /provider/ - # accept unsigned requests: keep :9200 on 127.0.0.1. - single-network-layer-adapter: - <<: *adapter - container_name: single-network-layer-adapter - depends_on: - sunbird-registry-service: - condition: service_healthy - discovery-service: - condition: service_started - environment: - <<: *adapter-env - OTEL_SERVICE_NAME: single-network-layer-adapter - # The weather upstream's credential, as ONE ready-made header value -- - # "Basic ", prefix included. Named here and read at call time, so - # the value is in neither a config file nor the registry. - MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} - # The mandi upstream's credentials, exchanged for a token at call time. - AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} - AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} - # Read only when MandiPrice is pointed back at mock-agmarknet, which takes - # a static token instead of issuing one. - MOCKAGMARKNET_TOKEN: ${MOCKAGMARKNET_TOKEN:-local-mandi-token} - # The knowledge provider's OAuth2 client credentials. The step fails fast - # and permanently when either is unset. - VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} - VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} - volumes: - - ./config/adapters/single-network-layer.yaml:/app/config/adapter.yaml:ro - # One routing file per module, as in the multi-adapter stack. The network - # module reuses the multi-adapter routing-network.yaml unchanged. - - ./config/adapters/routing-single-network-layer-consumer.yaml:/app/config/routing-single-network-layer-consumer.yaml:ro - - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - - ./config/adapters/routing-single-network-layer-publish.yaml:/app/config/routing-single-network-layer-publish.yaml:ro - ports: - - "127.0.0.1:9200:9200" - - # ========================================================================== - # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI - # over it. - # - # docker compose -f docker-compose.single-network-layer.yml --profile observability up -d - # - # Sizing: ClickHouse alone wants 2-4 GB. With this profile on, the VM needs - # 16 GB (t3.xlarge or m6i.xlarge); without it, 8 GB is workable. - # ========================================================================== - hyperdx: - # clickstack-local, not clickstack-all-in-one: local runs single-user with - # no team to create and no ingestion API key to mint, which is what makes - # `up -d` the whole setup step. It is also why it has NO login of its own - # -- so once this is routed (see networks, below) the Access List on the - # proxy host is the only authentication in front of every trace, log and - # metric the stack has emitted. It is not an optional hardening step. - image: clickhouse/clickstack-local:latest - container_name: hyperdx - restart: unless-stopped - logging: *logging - profiles: ["observability"] - - # oan-edge in addition to oan-internal, so NPM can resolve `hyperdx` and - # serve the UI on a hostname instead of an `ssh -L`. oan-internal stays, - # because that is the network the in-stack senders reach 4317/4318 on. - # - # This widens what a wrong click in NPM can reach, and the widening is - # worth stating rather than discovering: this container is also ClickHouse, - # so `hyperdx` is now a resolvable Forward Hostname for port 8123 -- an - # HTTP interface clickstack-local leaves open, over the same volume. The - # proxy host for the UI pins 8080 and is fine; nothing in NPM stops someone - # typing 8123 into a second one. Registry and Keycloak remain off oan-edge, - # so that bound is unchanged -- this is a deliberate trade of one name for - # not having to tunnel to read telemetry. - networks: [oan-internal, oan-edge] - env_file: - # Optional. Present so an existing .env.docker keeps working; compose - # errors on a missing env_file unless it is declared this way. - - path: .env.docker - required: false - environment: - # Self-instrumentation off: the telemetry stack reporting on itself is - # noise in the same tables the stack under test writes to, and at debug - # level it is most of the volume. - OTEL_SDK_DISABLED: "true" - HYPERDX_LOG_LEVEL: error - HYPERDX_USAGE_STATS_ENABLED: "false" - HYPERDX_USAGE_STATS_COLLECTION_ENABLED: "false" - OTEL_LOG_LEVEL: none - OTEL_TRACES_EXPORTER: none - OTEL_METRICS_EXPORTER: none - OTEL_LOGS_EXPORTER: none - USAGE_STATS_ENABLED: "false" - - # Container stdout -> this container's own collector. The supervisor - # appends this file to /etc/otelcol-contrib/config.yaml rather than - # replacing it (/etc/otel/supervisor.yaml.tmpl:27), so it adds a pipeline - # and changes nothing HyperDX configures for itself. The reasoning is in - # the file; the short version is that traces, metrics and AUDIT records - # arrive over OTLP and the console log did not arrive at all. - CUSTOM_OTELCOL_CONFIG_FILE: /etc/otelcol-contrib/filelog.yaml - ports: - # 8085, not the 8080 this image serves the UI on and not the 8081 it is - # usually published as: 8081 is REGISTRY_PORT here, and two services - # cannot claim one host port. - # - # Still 127.0.0.1 now that there is a route: the public path is NPM -> - # hyperdx:8080 across oan-edge, which is what makes the Access List - # unavoidable. Rebinding this to 0.0.0.0 would open a second door on the - # same UI with no credential on it, and the security group is the only - # thing that would be holding it shut. - - "127.0.0.1:${HYPERDX_UI_PORT:-8085}:8080" - # OTLP. Published on loopback only so a reverse tunnel can carry - # telemetry from a provider API running on someone's laptop; the - # in-stack senders reach hyperdx:4317/4318 over oan-internal and do not - # need these at all. - - "127.0.0.1:${HYPERDX_OTLP_GRPC_PORT:-4317}:4317" - - "127.0.0.1:${HYPERDX_OTLP_HTTP_PORT:-4318}:4318" - - # ClickHouse's HTTP interface, and it is published for one specific - # reason: clickstack-local is LOCAL mode, where the UI queries ClickHouse - # straight from the BROWSER with no API server in between. So the Host - # field in Team Settings is dialled by your laptop, not by the container - # -- its `http://localhost:8123` default means *your* machine, so with - # nothing listening there it fails with "Error connecting to ClickHouse - # server" however the proxy is set up. The route and the Access List are - # not implicated; the query never leaves the browser's own stack. - # - # A tunnel to 8123 is all it needs, and the UI can still be the public - # host -- the origin serving the UI and the address it queries are - # independent: - # - # ssh -L 8123:127.0.0.1:8123 -N you@the-vm - # then https://hyperdx./ with connection Host - # http://localhost:8123 (add -L 8085 if you want the UI local too) - # - # The catch is that this is PER VIEWER: a teammate who opens the public - # host without that tunnel gets the same error, and connections live in - # browser local storage, so each origin needs its own set up by hand. - # - # 127.0.0.1 and NOT negotiable, unlike the UI on 8085. This interface - # executes arbitrary SQL and clickstack-local leaves it open, so a - # 0.0.0.0 bind or an NPM route publishes the whole telemetry store -- - # and its own auth uses the same `Authorization: Basic` header an NPM - # Access List needs, so the two cannot be stacked. - # - # The real fix is clickstack-all-in-one: a server-side API queries - # ClickHouse, connections stop being browser state, and this publish and - # the tunnel both become unnecessary. See README Appendix H. - - "127.0.0.1:${CLICKHOUSE_HTTP_PORT:-8123}:8123" - volumes: - - hyperdx-clickhouse-data:/var/lib/clickhouse - - ./config/observability/filelog.yaml:/etc/otelcol-contrib/filelog.yaml:ro - - # Docker's own log files, READ-ONLY. This is the container that reads - # every other container's stdout, which is a real widening of what a - # compromise here reaches -- stated rather than buried, because it is the - # cost of the feature. - # - # The path is the daemon's, not the host's. On Docker Desktop for macOS - # it lives inside the LinuxKit VM and the bind still resolves (verified - # on 28.2.2). On rootless Docker, Podman or colima it does not, and the - # collector simply finds no files -- the stack comes up, the pipeline - # runs, and the log console stays empty. - # - # "No logs arrive" has TWO causes that look identical from outside, so - # check them in this order -- the second is the more common one and the - # cheaper test: - # - # 1. The collector never built the pipeline, because something in - # filelog.yaml did not parse or names a component the merged config - # no longer supplies. The container stays UP and the UI keeps - # serving; only the collector is down. `docker compose ps` says - # unhealthy, because the healthcheck below probes 13133 as well - # as 8080. Then: docker compose logs hyperdx | grep -i error - # - # 2. This mount resolved to nothing, so the receiver has no files to - # read. The container is HEALTHY -- the collector is fine, it just - # has no input -- which is what makes this one silent: - # - # docker exec hyperdx ls /var/lib/docker/containers | head - # - # Healthy-but-empty is (2). Unhealthy-and-empty is (1). - - /var/lib/docker/containers:/var/lib/docker/containers:ro - healthcheck: - # wget, not curl: this image has busybox. And a GET, not `--spider`, - # which sends HEAD -- Next.js answers HEAD / with a 404 while serving - # GET / perfectly, so the probe failed against a UI that was working. - # - # 127.0.0.1 rather than localhost, because busybox resolves localhost to - # ::1 first and these listeners are v4. - # - # Two checks, because this container is two things. 8080 is the Next.js - # UI -- what 8085 publishes and what you actually open. 13133 is the - # collector's health_check extension, and the collector is what receives - # telemetry on 4317/4318: it can die while the UI keeps serving, which - # loses data silently and is the failure worth catching. - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/ && wget -q -O /dev/null http://127.0.0.1:13133/"] - interval: 10s - timeout: 5s - # ClickHouse creates its system tables on a cold volume, which takes - # minutes rather than the 80s the previous 5/30s pair allowed. Docker - # does not re-evaluate once a container has failed past `retries`, so a - # window that is merely tight leaves a permanent wrong label. - retries: 10 - start_period: 180s - - # ========================================================================== - # edge -- Nginx Proxy Manager. - # - # docker compose -f docker-compose.single-network-layer.yml --profile reverse-proxy up -d - # - # NPM rather than a hand-written nginx.conf because it owns the part that is - # genuinely tedious to do by hand -- ACME. It requests, installs and renews - # Let's Encrypt certificates from a UI rather than from a certbot invocation - # someone has to remember. In exchange, the routing table stops being a file - # in this repo and becomes rows in NPM's SQLite database under the nginx-proxy-manager-data - # volume: reviewable only by opening the UI, and restorable only from that - # volume. See README for the proxy hosts to create. - # ========================================================================== - nginx-proxy-manager: - image: jc21/nginx-proxy-manager:latest - container_name: nginx-proxy-manager - restart: unless-stopped - logging: *logging - profiles: ["reverse-proxy"] - - # oan-edge ONLY, and this is the whole security argument for putting a - # UI-configured proxy in front of this stack. - # - # NPM's "Forward Hostname" is a free-text field. Anyone with the admin - # password can type `sunbird-registry-service`, `sunbird-registry-keycloak` or `discovery-postgres` into it. On - # this network none of those names resolve and none of those addresses are - # routable from here -- single-network-layer-adapter, and hyperdx when the - # observability profile is up, are the only containers NPM shares a network - # with. So the blast radius of a wrong click, or of the admin UI being - # reached by someone who should not have it, is bounded to that set. - # - # Putting NPM on oan-internal to "make things easier" would remove that - # bound entirely and put the registry's write API one form field away from - # the internet. Do not. - networks: [oan-edge] - - ports: - # Public. Both of these must be reachable from anywhere, not just from - # your address: Let's Encrypt validates HTTP-01 by fetching - # http:///.well-known/acme-challenge/... from its own servers, and - # those come from addresses you do not get to enumerate. An SG rule - # scoped to your IP will make certificate issuance fail with a challenge - # timeout. Use DNS-01 instead if 80 must stay closed -- see README. - - "0.0.0.0:${REVERSE_PROXY_HTTP_PORT:-80}:80" - - "0.0.0.0:${REVERSE_PROXY_HTTPS_PORT:-443}:443" - - # The admin UI, on loopback -- deliberately not the "publish it and - # restrict it in the firewall" arrangement. Two reasons. It ships with - # a known default login (admin@example.com / changeme) that is live from - # first boot until someone changes it, and it is the one surface here - # that can mint certificates and re-point every public route. A security - # group is a second system to keep in step with that; a loopback bind is - # not. Reach it over the tunnel: - # - # ssh -L 81:127.0.0.1:81 -N you@the-vm then http://127.0.0.1:81 - - "127.0.0.1:${NPM_ADMIN_PORT:-81}:81" - - volumes: - - nginx-proxy-manager-data:/data - - nginx-proxy-manager-letsencrypt:/etc/letsencrypt - - # Version-controlled nginx that NPM includes on its own. /data is a - # named volume and this bind mounts over one directory inside it, which - # works because the more specific mount wins. - # - # Only what is genuinely context-wide belongs here -- see the README for - # which of these NPM loads automatically and which is a paste job. - # - # NOT :ro, however much it wants to be. NPM's s6 init runs 50-ipv6.sh - # over everything under /data/nginx, which writes a .tmp beside each file - # and then chowns it. On a read-only mount both fail, the `prepare` - # service exits 1, s6 aborts the rest of the chain, and nginx never - # starts -- the container still reports Up, because s6 itself is alive, - # so the symptom is an empty reply on 81 and a healthcheck that never - # passes rather than anything that says "permissions". - # - # What NPM actually rewrites is `listen` directives, and neither file - # here has one, so the pass is a no-op on content. It does take - # ownership of the files, which is cosmetic and not tracked by git. - - ./config/reverse-proxy/npm-custom:/data/nginx/custom - - depends_on: - # Ordering only; the adapter image has no healthcheck. - # - # And ordering is not the whole problem. NPM writes a literal - # proxy_pass hostname per proxy host, which nginx resolves at reload and - # then caches -- so if you RECREATE an adapter (not merely restart it; - # a restart keeps the address) NPM keeps proxying to an address nothing - # answers on. The fix is one command, and it is worth knowing before you - # spend an afternoon on a 502: - # - # docker compose restart nginx-proxy-manager - # - # hyperdx is deliberately absent even though it is routed: it is behind - # the `observability` profile, and a depends_on would make it start - # whenever NPM does. That means the restart above is not optional after - # `make observability` -- NPM was started before the name existed, so the - # HyperDX host 502s until it re-resolves. - single-network-layer-adapter: - condition: service_started - - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:81/api/ || exit 1"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 9568abc3..d4ca23bd 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -11,7 +11,8 @@ # registry sunbird-registry-postgres, sunbird-registry-keycloak, # sunbird-registry-service # discovery discovery-postgres, discovery-service -# adapters provider, network, consumer +# adapters single-network-layer-adapter (profile single, default) | +# provider, network, consumer (profile multi) # observability hyperdx profile: observability # edge nginx-proxy-manager profile: reverse-proxy # @@ -24,9 +25,10 @@ # shipped defaults # 2. docker compose up -d sunbird-registry-service discovery-service # everything EXCEPT the adapters -# 3. python3 bin/setup.py keys, the three adapter entries, and -# the adapter configs they mount -# 4. docker compose up -d now the adapters +# 3. python3 bin/setup.py keys, the three tier identities, and +# every adapter config they mount +# 4. docker compose --profile single up -d the single network layer adapter +# (or --profile multi: provider-, network-, consumer-adapter) # 5. docker compose --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack # @@ -533,7 +535,19 @@ services: condition: service_healthy # ========================================================================== - # adapters -- consumer, network and provider. Same image, three configs. + # adapters -- TWO topologies, chosen with a compose profile. Same image. + # + # profile "single" (the default; `make up`) + # single-network-layer-adapter: the consumer, network and provider tiers + # as four modules of ONE process, one config, one port (127.0.0.1:9200). + # Tier-to-tier hops are loopback HTTP inside the container. + # + # profile "multi" (`make up-multi`) + # provider-adapter, network-adapter, consumer-adapter: one container per + # tier, tier-to-tier hops over the compose network. + # + # Both publish 127.0.0.1:9200, so run one profile at a time (bin/stack.sh + # stops the other first). Both use the same keys and registry rows. # # Their configs are bind-mounted FILES that do not exist until bin/setup.py # renders them from the .tmpl beside them, which is what the step ordering @@ -586,12 +600,57 @@ services: ports: - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" + # The single network layer: the consumer, network and provider tiers as + # modules of ONE adapter process, on ONE config rendered by bin/setup.py from + # config/adapters/single-network-layer.yaml.tmpl, with one routing file per + # module. + # + # / the consumer, the caller's entry (bap), takes unsigned requests + # /network/* hands discover and publish on to discovery + # /provider/* calls the upstream, answers synchronously + # /provider/publish sends the provider's catalogue to /network/publish + # + # Each module carries its own tier identity (consumer, network, provider): + # the same keys and registry rows as the multi-adapter services. Signing is + # off, as in every quick-start template, so /network/ and /provider/ accept + # unsigned requests: keep :9200 on 127.0.0.1. + single-network-layer-adapter: + <<: *adapter + profiles: ["single"] + container_name: single-network-layer-adapter + depends_on: + sunbird-registry-service: + condition: service_healthy + discovery-service: + condition: service_started + environment: + <<: *adapter-env + OTEL_SERVICE_NAME: single-network-layer-adapter + # Upstream credentials, read at call time -- the same variables and + # reasons as provider-adapter below. + MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} + AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} + AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} + MOCKAGMARKNET_TOKEN: ${MOCKAGMARKNET_TOKEN:-local-mandi-token} + VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} + VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} + volumes: + - ./config/adapters/single-network-layer.yaml:/app/config/adapter.yaml:ro + # One routing file per module. The network module reuses the + # multi-adapter routing-network.yaml unchanged. + - ./config/adapters/routing-single-network-layer-consumer.yaml:/app/config/routing-single-network-layer-consumer.yaml:ro + - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro + - ./config/adapters/routing-single-network-layer-publish.yaml:/app/config/routing-single-network-layer-publish.yaml:ro + ports: + - "127.0.0.1:${SINGLE_NETWORK_LAYER_ADAPTER_PORT:-9200}:9200" + # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, # so repointing it at a different upstream is a registry write, not a # change here or a restart. provider-adapter: <<: *adapter + profiles: ["multi"] container_name: provider-adapter # Registry only. The mocks are deliberately NOT dependencies: every # capability points at a real external upstream, and depends_on would start @@ -637,6 +696,7 @@ services: # Verifies the caller, hands discovery on, re-signs as itself. network-adapter: <<: *adapter + profiles: ["multi"] container_name: network-adapter depends_on: sunbird-registry-service: @@ -658,6 +718,7 @@ services: # is also why the edge rate-limits this host and nothing else. consumer-adapter: <<: *adapter + profiles: ["multi"] container_name: consumer-adapter depends_on: network-adapter: @@ -938,12 +999,21 @@ services: # whenever NPM does. That means the restart above is not optional after # `make observability` -- NPM was started before the name existed, so the # HyperDX host 502s until it re-resolves. + # + # required: false -- only one topology's adapters exist at a time + # (profiles "single" / "multi"); NPM starts with whichever is running. + single-network-layer-adapter: + condition: service_started + required: false consumer-adapter: condition: service_started + required: false network-adapter: condition: service_started + required: false provider-adapter: condition: service_started + required: false healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:81/api/ || exit 1"] From 96b1648ecada50df46293b2302f8b0c7fd78c7af Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 12:50:18 +0530 Subject: [PATCH 15/43] feat(adapter-service): add single network layer adapter configuration and routing - Introduced a new single network layer adapter role that consolidates consumer, network, and provider tiers into a single process. - Added configuration files for the single network layer adapter, including resource requests, limits, and autoscaling settings. - Implemented routing rules for the consumer and provider modules to facilitate internal communication via loopback HTTP. - Created example YAML files for deploying the single network layer adapter. - Enhanced the existing templates to support the new adapter role and ensure proper key management and routing. --- ...ction.json => OpenAgriNet.network-adapter.api-collection.json} | 0 ...nt.json => OpenAgriNet.network-adapter.local-environment.json} | 0 ...ngle-network-layer-values.yaml => network-adapter-values.yaml} | 0 ...ngle-network-layer-config.yaml => network-adapter-config.yaml} | 0 .../examples/{single-network-layer.yaml => network-adapter.yaml} | 0 .../templates/{_single-network-layer.tpl => _network-adapter.tpl} | 0 .../{single-network-layer.yaml.tmpl => network-adapter.yaml.tmpl} | 0 ...-layer-consumer.yaml => routing-network-adapter-consumer.yaml} | 0 ...rk-layer-publish.yaml => routing-network-adapter-publish.yaml} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename api-collection/{OpenAgriNet.single-network-layer.api-collection.json => OpenAgriNet.network-adapter.api-collection.json} (100%) rename api-collection/{OpenAgriNet.single-network-layer.local-environment.json => OpenAgriNet.network-adapter.local-environment.json} (100%) rename charts/adapter-service/ci/{single-network-layer-values.yaml => network-adapter-values.yaml} (100%) rename charts/adapter-service/config/{single-network-layer-config.yaml => network-adapter-config.yaml} (100%) rename charts/adapter-service/examples/{single-network-layer.yaml => network-adapter.yaml} (100%) rename charts/adapter-service/templates/{_single-network-layer.tpl => _network-adapter.tpl} (100%) rename quick-start/config/adapters/{single-network-layer.yaml.tmpl => network-adapter.yaml.tmpl} (100%) rename quick-start/config/adapters/{routing-single-network-layer-consumer.yaml => routing-network-adapter-consumer.yaml} (100%) rename quick-start/config/adapters/{routing-single-network-layer-publish.yaml => routing-network-adapter-publish.yaml} (100%) diff --git a/api-collection/OpenAgriNet.single-network-layer.api-collection.json b/api-collection/OpenAgriNet.network-adapter.api-collection.json similarity index 100% rename from api-collection/OpenAgriNet.single-network-layer.api-collection.json rename to api-collection/OpenAgriNet.network-adapter.api-collection.json diff --git a/api-collection/OpenAgriNet.single-network-layer.local-environment.json b/api-collection/OpenAgriNet.network-adapter.local-environment.json similarity index 100% rename from api-collection/OpenAgriNet.single-network-layer.local-environment.json rename to api-collection/OpenAgriNet.network-adapter.local-environment.json diff --git a/charts/adapter-service/ci/single-network-layer-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml similarity index 100% rename from charts/adapter-service/ci/single-network-layer-values.yaml rename to charts/adapter-service/ci/network-adapter-values.yaml diff --git a/charts/adapter-service/config/single-network-layer-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml similarity index 100% rename from charts/adapter-service/config/single-network-layer-config.yaml rename to charts/adapter-service/config/network-adapter-config.yaml diff --git a/charts/adapter-service/examples/single-network-layer.yaml b/charts/adapter-service/examples/network-adapter.yaml similarity index 100% rename from charts/adapter-service/examples/single-network-layer.yaml rename to charts/adapter-service/examples/network-adapter.yaml diff --git a/charts/adapter-service/templates/_single-network-layer.tpl b/charts/adapter-service/templates/_network-adapter.tpl similarity index 100% rename from charts/adapter-service/templates/_single-network-layer.tpl rename to charts/adapter-service/templates/_network-adapter.tpl diff --git a/quick-start/config/adapters/single-network-layer.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl similarity index 100% rename from quick-start/config/adapters/single-network-layer.yaml.tmpl rename to quick-start/config/adapters/network-adapter.yaml.tmpl diff --git a/quick-start/config/adapters/routing-single-network-layer-consumer.yaml b/quick-start/config/adapters/routing-network-adapter-consumer.yaml similarity index 100% rename from quick-start/config/adapters/routing-single-network-layer-consumer.yaml rename to quick-start/config/adapters/routing-network-adapter-consumer.yaml diff --git a/quick-start/config/adapters/routing-single-network-layer-publish.yaml b/quick-start/config/adapters/routing-network-adapter-publish.yaml similarity index 100% rename from quick-start/config/adapters/routing-single-network-layer-publish.yaml rename to quick-start/config/adapters/routing-network-adapter-publish.yaml From 762c4a3c7c3bf6f3bc9cdba97625eb67b390eeb5 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 12:50:24 +0530 Subject: [PATCH 16/43] Refactor adapter service from single-network-layer to network-adapter-layer - Updated deployment and ingress templates to reflect the new network-adapter-layer role. - Changed Helm chart values and configurations to support the network-adapter-layer. - Modified quick-start documentation and scripts to align with the new naming and structure. - Adjusted routing configurations for the network-adapter-layer. - Updated Docker Compose files to use the network-adapter-layer service. --- .gitignore | 7 +- ...griNet.network-adapter.api-collection.json | 58 ++++++++--------- ...Net.network-adapter.local-environment.json | 4 +- charts/adapter-service/CHANGELOG.md | 12 ++-- charts/adapter-service/Chart.yaml | 10 +-- charts/adapter-service/README.md | 65 +++++++++++-------- .../ci/network-adapter-values.yaml | 14 ++-- .../config/network-adapter-config.yaml | 16 ++--- .../examples/network-adapter.yaml | 20 +++--- charts/adapter-service/templates/NOTES.txt | 16 ++--- charts/adapter-service/templates/_helpers.tpl | 20 +++--- .../templates/_network-adapter.tpl | 36 +++++----- .../adapter-service/templates/configmap.yaml | 26 ++++---- .../adapter-service/templates/deployment.yaml | 18 ++--- charts/adapter-service/templates/ingress.yaml | 16 ++--- charts/adapter-service/values.yaml | 18 ++--- charts/openbao-secrets/CHANGELOG.md | 10 +-- charts/openbao-secrets/values.yaml | 12 ++-- quick-start/.env.example | 4 +- quick-start/.gitignore | 8 +-- quick-start/Makefile | 2 +- quick-start/README.md | 51 ++++++++------- quick-start/bin/setup.py | 14 ++-- quick-start/bin/stack.sh | 41 +++++++----- .../config/adapters/network-adapter.yaml.tmpl | 22 +++---- .../routing-network-adapter-consumer.yaml | 7 +- .../routing-network-adapter-publish.yaml | 4 +- quick-start/docker-compose.yml | 26 ++++---- 28 files changed, 293 insertions(+), 264 deletions(-) diff --git a/.gitignore b/.gitignore index f2bd38ba..ade35b3b 100644 --- a/.gitignore +++ b/.gitignore @@ -71,14 +71,11 @@ secrets.json # Generated local Postman exports for the unified network layer; keep them out of source # control even though they are tracked alongside the repo's API collection files. -api-collection/OpenAgriNet.unified-network-layer.collection.json -api-collection/OpenAgriNet.unified-network-layer.environment.json /dev_docs # Python bytecode __pycache__/ *.pyc # Local demo and POC Postman collections (the committed ones are -# OpenAgriNet.api-collection.json and OpenAgriNet.single-network-layer.api-collection.json) -/api-collection/OpenAgriNet.single-network-layer-demo.* -/api-collection/OpenAgriNet.unified-network-layer.* +# OpenAgriNet.api-collection.json and OpenAgriNet.network-adapter.api-collection.json) +/api-collection/OpenAgriNet.network-adapter-demo.* diff --git a/api-collection/OpenAgriNet.network-adapter.api-collection.json b/api-collection/OpenAgriNet.network-adapter.api-collection.json index 3a58b691..433ae866 100644 --- a/api-collection/OpenAgriNet.network-adapter.api-collection.json +++ b/api-collection/OpenAgriNet.network-adapter.api-collection.json @@ -1,9 +1,9 @@ { "info": { - "name": "OpenAgriNet - Single network layer", + "name": "OpenAgriNet - Network Adapter Layer", "_postman_id": "0a4e1c00-0000-4000-8000-0000000snl01", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.single-network-layer.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, single-network-layer-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /network/ network module (internal)\n /provider/ provider module\n /provider/publish catalogue publishing\n\nPublish enters /provider/publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." + "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.network-adapter.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, network-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /network/ network module (internal)\n /provider/ provider module\n /provider/publish catalogue publishing\n\nPublish enters /provider/publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." }, "event": [ { @@ -58,9 +58,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/provider/publish", + "raw": "{{network-adapter-host}}/provider/publish", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "provider", @@ -109,9 +109,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/discover", + "raw": "{{network-adapter-host}}/discover", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "discover" @@ -159,9 +159,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/select", + "raw": "{{network-adapter-host}}/select", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "select" @@ -215,9 +215,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/provider/publish", + "raw": "{{network-adapter-host}}/provider/publish", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "provider", @@ -266,9 +266,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/discover", + "raw": "{{network-adapter-host}}/discover", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "discover" @@ -316,9 +316,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/select", + "raw": "{{network-adapter-host}}/select", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "select" @@ -372,9 +372,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/provider/publish", + "raw": "{{network-adapter-host}}/provider/publish", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "provider", @@ -423,9 +423,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/discover", + "raw": "{{network-adapter-host}}/discover", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "discover" @@ -473,9 +473,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/select", + "raw": "{{network-adapter-host}}/select", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "select" @@ -529,9 +529,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/provider/publish", + "raw": "{{network-adapter-host}}/provider/publish", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "provider", @@ -580,9 +580,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/discover", + "raw": "{{network-adapter-host}}/discover", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "discover" @@ -630,9 +630,9 @@ } ], "url": { - "raw": "{{single-network-layer-host}}/select", + "raw": "{{network-adapter-host}}/select", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "select" @@ -676,14 +676,14 @@ "description": "Run these first. Every one is a plain GET and none of them signs anything.", "item": [ { - "name": "single network layer adapter", + "name": "Network Adapter", "request": { "method": "GET", "header": [], "url": { - "raw": "{{single-network-layer-host}}/health", + "raw": "{{network-adapter-host}}/health", "host": [ - "{{single-network-layer-host}}" + "{{network-adapter-host}}" ], "path": [ "health" diff --git a/api-collection/OpenAgriNet.network-adapter.local-environment.json b/api-collection/OpenAgriNet.network-adapter.local-environment.json index 511511be..bc72f824 100644 --- a/api-collection/OpenAgriNet.network-adapter.local-environment.json +++ b/api-collection/OpenAgriNet.network-adapter.local-environment.json @@ -1,9 +1,9 @@ { "id": "0a4e1c00-0000-4000-8000-0000000snle1", - "name": "OpenAgriNet local (single network layer)", + "name": "OpenAgriNet local (Network Adapter Layer)", "values": [ { - "key": "single-network-layer-host", + "key": "network-adapter-host", "value": "http://127.0.0.1:9200", "enabled": true }, diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index a876cb80..fb3ea441 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -8,19 +8,19 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ## [0.5.3] - 2026-10-09 ### Added -- `role: single-network-layer`: all three tiers in one adapter (four modules: +- `role: network-adapter-layer`: all three tiers in one adapter (four modules: `/` (consumer), `/network/`, `/provider/`, `/provider/publish`), as in the - quick-start's single-network-layer compose stack. Its config is - `config/single-network-layer-config.yaml` (extended schema, an outbound + quick-start's `single` topology (compose profile). Its config is + `config/network-adapter-config.yaml` (extended schema, an outbound connection pool and log level info on every module); its routing is generated at render time from `config/routing-*.yaml`. Tier hops are loopback HTTP inside the pod. Signs as three identities from - `singleNetworkLayer.keys.{consumer,network,provider}`; `routing.config` fails the + `networkAdapterLayer.keys.{consumer,network,provider}`; `routing.config` fails the render for this role. -- single-network-layer ingress may expose only single-segment `/` and +- network-adapter-layer ingress may expose only single-segment `/` and `/provider/`, as literal paths with `pathType` Prefix or Exact and no `use-regex` / `rewrite-target` annotation. Example - `examples/single-network-layer.yaml`. + `examples/network-adapter.yaml`. - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index f6390ffc..0782d1df 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -2,9 +2,11 @@ apiVersion: v2 name: adapter-service description: >- One chart for the OAN Beckn adapters: provider, network and consumer (one - release per tier), or single-network-layer (all three tiers in one adapter). Same - image and config format; role selects the handler role, the step list and - the routing target. Stateless: an adapter's + release per tier, the multi topology), or network-adapter-layer: the Network + Adapter Layer runs the consumer, network and provider tiers as four modules + of one process (the single topology; tier-to-tier calls are loopback HTTP and + each module keeps its own tier identity). Same image and config format; role + selects the handler role, the step list and the routing target. Stateless: an adapter's identity is a keypair held in a Secret, and everything else it reads from the registry. type: application @@ -23,7 +25,7 @@ keywords: - provider - network - consumer - - single-network-layer + - network-adapter-layer home: https://github.com/OpenAgriNet/helmcharts sources: - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index d845c8c9..eab17ea0 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -1,8 +1,8 @@ # adapter-service One chart for the Beckn adapters, in two modes. Multi-adapter: three roles, -`provider`, `network` and `consumer`, one release per tier. Network layer: -`single-network-layer`, all three tiers in one adapter. All roles are the same image +`provider`, `network` and `consumer`, one release per tier. The Network Adapter +Layer: `network-adapter-layer`, all three tiers in one adapter. All roles are the same image and the same config format, so `role` selects the handler role, the step list and where requests are routed. @@ -15,7 +15,7 @@ and where requests are routed. ``` ``` - single-network-layer (one adapter, one pod): + network-adapter-layer (one adapter, one pod): / ──lo──▶ /network/ ──▶ discovery └──────lo────▶ /provider/ ──▶ upstream APIs /provider/publish ──lo──▶ /network/publish @@ -26,7 +26,7 @@ and where requests are routed. | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | | `network` | `bpp` | `validateSign → addRoute → sign` | discovery | | `consumer` | `bap` | `addRoute → sign` | network | -| `single-network-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | +| `network-adapter-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | `consumer` is the only one that accepts **unsigned** requests: the consumer app is inside the trust boundary, so there is no network signature to check. @@ -68,7 +68,7 @@ render error rather than a default: | Value | Why there is no default | |---|---| | `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | -| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `single-network-layer` uses `singleNetworkLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | +| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `network-adapter-layer` uses `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | | `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | | `role` | It decides the handler role, the step list and the routing target. A default would give one adapter another one's behaviour | | `routing.rules` | An adapter with no route accepts requests and has nowhere to send them | @@ -131,58 +131,71 @@ the **public** half of the keypair. Missing, or carrying a different key, and every peer rejects what this adapter signs while the pod stays perfectly healthy. -## Choosing a mode: multi-adapter or network layer +## Choosing a mode: multi-adapter or Network Adapter Layer | Mode | Releases (release = namespace) | Callers use | |---|---|---| | multi | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | -| network layer | `single-network-layer-adapter` (examples/single-network-layer.yaml) | `single-network-layer-adapter:9200/`, `single-network-layer-adapter:9200/provider/publish` | +| Network Adapter Layer | `network-adapter` (examples/network-adapter.yaml) | `network-adapter:9200/`, `network-adapter:9200/provider/publish` | -`role: single-network-layer` runs all three tiers in one adapter, as four -modules of one process. Its config is `config/single-network-layer-config.yaml` +`role: network-adapter-layer` runs all three tiers in one adapter, as four +modules of one process. Its config is `config/network-adapter-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change to a tier file that should apply to both modes goes in both files). Its routing is **generated** at render time from `config/routing-*.yaml`, so routes stay in one place. Tier hops are loopback HTTP inside the pod (any image). It signs as the same three identities, from the same three Secrets, so the registry is the same in both modes. An environment's `config:` override for a multi role is NOT -seen by network-layer; give the single-network-layer release its own `config:` if -needed (it replaces `single-network-layer-config.yaml` whole). +seen by the Network Adapter Layer; give its release its own `config:` if +needed (it replaces `network-adapter-config.yaml` whole). -Pick the mode per environment with ONE switch where releases are chosen. For a -helmfile (adapt to infra-automation's layout): +Pick the topology per environment with ONE value where releases are chosen, +`networkAdapterTopology: single | multi` (a helmfile environment value; no chart +reads it). `single` installs one release, `network-adapter` (the Network Adapter +Layer); `multi` installs the three tier releases. For a helmfile (adapt to +infra-automation's layout): ```yaml -# environments: values carry adapters: {singleNetworkLayerAdapter: false} +# environments: values carry networkAdapterTopology: multi # or: single +{{- if not (has .Values.networkAdapterTopology (list "single" "multi")) }} +{{- fail (printf "networkAdapterTopology must be single or multi, not %q" .Values.networkAdapterTopology) }} +{{- end }} releases: {{- range $tier := list "consumer" "network" "provider" }} - name: {{ $tier }}-adapter namespace: {{ $tier }}-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ not $.Values.adapters.singleNetworkLayerAdapter }} + installed: {{ eq $.Values.networkAdapterTopology "multi" }} values: [values/{{ $tier }}-adapter.yaml] {{- end }} - - name: single-network-layer-adapter - namespace: single-network-layer-adapter + - name: network-adapter + namespace: network-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ .Values.adapters.singleNetworkLayerAdapter }} - values: [values/single-network-layer-adapter.yaml] + installed: {{ eq .Values.networkAdapterTopology "single" }} + values: [values/network-adapter.yaml] ``` -With Argo CD, gate the four Applications on the same boolean (for example an +With Argo CD, gate the same releases on the same value (for example an ApplicationSet list generator, or `{{ if }}` in an app-of-apps chart). -Switching an environment: install `single-network-layer-adapter` alongside the multi -releases (both modes sign with the same identities), repoint the app and the -catalogue publisher to the paths above, then remove the three multi releases. -Reverse the steps to switch back. +**The two topologies share the name `network-adapter`.** The Network Adapter +Layer is the network adapter as a whole, so its release, namespace and Service +are `network-adapter`, which is also the multi network tier's. They therefore +cannot be installed side by side under that name (the Service port differs too: +9200 for the layer, 9201 for the multi network tier). Switch an environment in +place: for a trial, install the layer in another namespace +(`-n network-adapter-trial`); to switch for real, remove the multi +`network-adapter` release, install the layer, repoint the app and the +catalogue publisher to the paths above, then remove the other two multi +releases. Both topologies sign with the same identities and the same registry +rows, so nothing in the registry changes. Reverse the steps to switch back. ## Values See `values.yaml` — every field is commented with what it does and what breaks without it. `examples/{provider,network,consumer}.yaml` are working dev deployments for the -multi-adapter mode, and `examples/single-network-layer.yaml` for the network layer; -`ci/` holds the files lint renders, including `ci/single-network-layer-values.yaml`. +multi-adapter mode, and `examples/network-adapter.yaml` for the Network Adapter Layer; +`ci/` holds the files lint renders, including `ci/network-adapter-values.yaml`. ## Related diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 829892cb..c14581a5 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -1,10 +1,10 @@ -# charts/adapter-service/ci/single-network-layer-values.yaml -# role single-network-layer: the single network layer adapter (consumer, network and +# charts/adapter-service/ci/network-adapter-values.yaml +# role network-adapter-layer: the Network Adapter Layer (consumer, network and # provider tiers as modules of one process). Renders -# config/single-network-layer-config.yaml, the loopback routing and the three +# config/network-adapter-config.yaml, the loopback routing and the three # tier identities. -role: single-network-layer -fullnameOverride: single-network-layer-adapter +role: network-adapter-layer +fullnameOverride: network-adapter image: registry: ghcr.io @@ -18,7 +18,7 @@ service: port: 9200 targetPort: 9200 -singleNetworkLayer: +networkAdapterLayer: keys: consumer: existingSecret: consumer-adapter-keys @@ -29,7 +29,7 @@ singleNetworkLayer: extraSubstitutions: __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl -# Scale defaults for the single network layer adapter: roughly the three tiers' +# Scale defaults for the Network Adapter Layer: roughly the three tiers' # requests in one pod, at least one replica per node. resources: requests: diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index baf27416..7137905c 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -1,4 +1,4 @@ -# OAN single network layer adapter (role single-network-layer). The consumer, network +# OAN Network Adapter Layer (role network-adapter-layer). The consumer, network # and provider tiers as four modules of ONE adapter process: # # / the consumer (bap); the app posts here, unsigned @@ -12,7 +12,7 @@ # # Differences from the three tier files: # - one listener (9200) and one appName; tier hops are loopback HTTP to this -# same listener (see the routing-single-network-layer-*.yaml files, which the chart +# same listener (see the routing-network-adapter-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; @@ -21,9 +21,9 @@ # This file is mounted verbatim. Each module signs as its OWN tier: the # __CONSUMER_*__, __NETWORK_*__ and __PROVIDER_*__ placeholders are substituted # by the init container from that tier's Secret -# (singleNetworkLayer.keys..existingSecret) at pod start, so no key is in this +# (networkAdapterLayer.keys..existingSecret) at pod start, so no key is in this # file, in the ConfigMap, or in git. -appName: "single-network-layer-adapter" +appName: "network-adapter" log: level: info @@ -51,7 +51,7 @@ plugins: otelsetup: id: otelsetup config: - serviceName: "oan-single-network-layer-adapter" + serviceName: "oan-network-adapter" environment: "dev" otlpEndpoint: "clickstack-otel-collector.observability.svc.cluster.local:4317" enableMetrics: "true" @@ -117,7 +117,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-single-network-layer-consumer.yaml + routingConfig: /app/config/routing-network-adapter-consumer.yaml middleware: - id: reqpreprocessor config: @@ -280,7 +280,7 @@ modules: bharat-vistaar: authScheme: oauth2 # Substituted from the provider Secret (vistaarTokenUrl) via - # singleNetworkLayer.extraSubstitutions. + # networkAdapterLayer.extraSubstitutions. tokenUrl: "__VISTAAR_TOKEN_URL__" clientIdEnv: VISTAAR_CLIENT_ID clientSecretEnv: VISTAAR_CLIENT_SECRET @@ -353,7 +353,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-single-network-layer-publish.yaml + routingConfig: /app/config/routing-network-adapter-publish.yaml middleware: - id: reqpreprocessor config: diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index c8807a15..c5e25731 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -1,5 +1,5 @@ # =========================================================================== -# single-network-layer role -- the single network layer adapter. +# network-adapter-layer role -- the Network Adapter. # # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: @@ -13,18 +13,18 @@ # (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the # other in an environment, not both. # -# helm install single-network-layer-adapter . -n single-network-layer-adapter --create-namespace \ -# -f examples/single-network-layer.yaml +# helm install network-adapter . -n network-adapter --create-namespace \ +# -f examples/network-adapter.yaml # -# The config is config/single-network-layer-config.yaml; the routing is +# The config is config/network-adapter-config.yaml; the routing is # generated from config/routing-*.yaml. Tier hops are loopback HTTP inside the # pod, so any adapter image works. # =========================================================================== -role: single-network-layer +role: network-adapter-layer # The Service name and DNS name callers use: -# single-network-layer-adapter.single-network-layer-adapter.svc.cluster.local:9200 -fullnameOverride: single-network-layer-adapter +# network-adapter.network-adapter.svc.cluster.local:9200 +fullnameOverride: network-adapter image: registry: ghcr.io @@ -36,12 +36,12 @@ registry: # The same port the quick-start publishes. targetPort is also the loopback port # the generated routing uses, and must equal http.port in -# config/single-network-layer-config.yaml (the render checks). +# config/network-adapter-config.yaml (the render checks). service: port: 9200 targetPort: 9200 -singleNetworkLayer: +networkAdapterLayer: # The three tier Secrets the multi-adapter releases use. openbao-secrets # reflects them into this namespace. keys: @@ -108,7 +108,7 @@ ingress: enabled: false className: "" hosts: - - host: single-network-layer.oan.example.com + - host: network-adapter.oan.example.com paths: - path: /discover pathType: Exact diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index 423ef25a..babfbf0b 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,6 +1,6 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} -{{ if include "adapter-service.isSingleNetworkLayer" . }} - role single-network-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) +{{ if include "adapter-service.isNetworkAdapterLayer" . }} + role network-adapter-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) hops loopback HTTP between tiers (http://localhost:{{ .Values.service.targetPort }}/) {{- else }} role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) @@ -9,7 +9,7 @@ image {{ include "adapter-service.image" . }} service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} registry {{ .Values.registry.url }} - routes {{ if include "adapter-service.isSingleNetworkLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} + routes {{ if include "adapter-service.isNetworkAdapterLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. @@ -26,10 +26,10 @@ chart's appVersion. TWO THINGS THIS CHART CANNOT CHECK FOR YOU -{{ if include "adapter-service.isSingleNetworkLayer" . -}} +{{ if include "adapter-service.isNetworkAdapterLayer" . -}} 1. The Participant rows. This adapter signs as three identities -- consumer, - network and provider -- from Secrets {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, - {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the + network and provider -- from Secrets {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, + {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the multi-adapter releases use. Each must carry its tier's public key. 2. Key rotation. Changing any of those Secrets restarts nothing: @@ -62,8 +62,8 @@ on and nothing listening the exporter logs a failure every interval. {{- if .Values.ingress.enabled }} Ingress is ON. -{{- if include "adapter-service.isSingleNetworkLayer" . }} -This is the single-network-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is +{{- if include "adapter-service.isNetworkAdapterLayer" . }} +This is the network-adapter-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is inside the trust boundary and has no validateSign step, so an Ingress on it exposes an unauthenticated entry point to the network. Rate-limit this host. Rule: only single-segment / paths and /provider/ may be exposed (the render refuses other paths), diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl index fc62e6db..e02f0084 100644 --- a/charts/adapter-service/templates/_helpers.tpl +++ b/charts/adapter-service/templates/_helpers.tpl @@ -6,7 +6,7 @@ # identity, upstream and config-rendering wiring an adapter needs. # # One chart, four roles. provider, network and consumer (one release per tier) -# and single-network-layer (all three tiers in one adapter) are the same image and the +# and network-adapter-layer (all three tiers in one adapter) are the same image and the # same config format; the role decides the handler role, the step list and # where requests are routed. Install once per role, and keep the RELEASE name # per-role -- the release name is what becomes the Service DNS name that the @@ -15,7 +15,7 @@ */}} {{/* -The adapter's role: provider | network | consumer | single-network-layer. +The adapter's role: provider | network | consumer | network-adapter-layer. Required, with no default. A default would silently give one role's step list and handler role to a different adapter -- which renders, starts, reports Ready @@ -24,20 +24,23 @@ and then mis-handles every request, at the far end, in a peer's logs. {{- define "adapter-service.role" -}} {{- $role := .Values.role | default "" -}} {{- if not $role -}} -{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or single-network-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} +{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or network-adapter-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} {{- end -}} -{{- if not (has $role (list "provider" "network" "consumer" "single-network-layer")) -}} -{{- fail (printf "%s: role must be one of provider, network, consumer, single-network-layer (got %q)." .Chart.Name $role) -}} +{{- if not (has $role (list "provider" "network" "consumer" "network-adapter-layer")) -}} +{{- fail (printf "%s: role must be one of provider, network, consumer, network-adapter-layer (got %q)." .Chart.Name $role) -}} {{- end -}} {{- $role -}} {{- end }} {{/* The name the adapter calls itself in its config, its logs and its module. -Defaults to -adapter, matching the compose stack's service names. +Defaults to -adapter, matching the compose stack's service names. The +Network Adapter Layer (role network-adapter-layer) is the network adapter as +a whole, so it is called network-adapter, not network-adapter-layer-adapter. */}} {{- define "adapter-service.appName" -}} -{{- .Values.appName | default (printf "%s-adapter" (include "adapter-service.role" .)) -}} +{{- $role := include "adapter-service.role" . -}} +{{- .Values.appName | default (ternary "network-adapter" (printf "%s-adapter" $role) (eq $role "network-adapter-layer")) -}} {{- end }} {{/* @@ -45,7 +48,8 @@ OTEL service name. Defaults to oan--adapter, matching OTEL_SERVICE_NAME in the compose stack so traces from either deployment line up. */}} {{- define "adapter-service.otelServiceName" -}} -{{- .Values.otel.serviceName | default (printf "oan-%s-adapter" (include "adapter-service.role" .)) -}} +{{- $role := include "adapter-service.role" . -}} +{{- .Values.otel.serviceName | default (ternary "oan-network-adapter" (printf "oan-%s-adapter" $role) (eq $role "network-adapter-layer")) -}} {{- end }} {{/* diff --git a/charts/adapter-service/templates/_network-adapter.tpl b/charts/adapter-service/templates/_network-adapter.tpl index 478902f5..20b8cd10 100644 --- a/charts/adapter-service/templates/_network-adapter.tpl +++ b/charts/adapter-service/templates/_network-adapter.tpl @@ -1,9 +1,9 @@ {{/* # ============================================================================ -# role: single-network-layer -- the single network layer adapter. +# role: network-adapter-layer -- the Network Adapter Layer. # # The consumer, network and provider tiers as four modules of ONE adapter -# process. The adapter config is config/single-network-layer-config.yaml +# process. The adapter config is config/network-adapter-config.yaml # (configmap.yaml mounts it like any role's file). The routing is GENERATED at # render time from config/routing-.yaml, the same files the three # multi-adapter roles mount, with tier hosts pointed at this pod's own listener. @@ -11,31 +11,31 @@ */}} {{/* -"true" when this release runs the single network layer adapter, else empty. +"true" when this release runs the Network Adapter Layer, else empty. */}} -{{- define "adapter-service.isSingleNetworkLayer" -}} -{{- if eq (include "adapter-service.role" .) "single-network-layer" -}}true{{- end -}} +{{- define "adapter-service.isNetworkAdapterLayer" -}} +{{- if eq (include "adapter-service.role" .) "network-adapter-layer" -}}true{{- end -}} {{- end }} {{/* -Fails unless every private-key VALUE in a single-network-layer config is one of the +Fails unless every private-key VALUE in a network-adapter-layer config is one of the tier placeholders. Parsed, not grepped: it reads each module's keyManager. -Usage: include "adapter-service.singleNetworkLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) +Usage: include "adapter-service.networkAdapterLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) */}} -{{- define "adapter-service.singleNetworkLayer.checkKeys" -}} +{{- define "adapter-service.networkAdapterLayer.checkKeys" -}} {{- range $m := .cfg.modules -}} {{- $km := dig "handler" "plugins" "keyManager" "config" dict $m -}} {{- range $field := list "signingPrivateKey" "encrPrivateKey" -}} {{- $v := index $km $field | default "" | toString -}} {{- if and $v (not (regexMatch "^__(CONSUMER|NETWORK|PROVIDER)_(SIGNING|ENCR)_PRIVATE__$" $v)) -}} -{{- fail (printf "%s: single-network-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} +{{- fail (printf "%s: network-adapter-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} {{- end -}} {{- end -}} {{- end -}} {{- end }} {{/* -Routing for the single network layer adapter, as YAML: {: }. +Routing for the Network Adapter Layer, as YAML: {: }. routing-network.yaml is the multi-adapter file, unchanged (discover and publish to discovery). The other two are generated from routing-consumer.yaml and @@ -47,25 +47,25 @@ kept as it is, and a rule with no target is left alone (the same nil guard as configmap.yaml). So the two modes are one routing source, and splitting the modules across pods again changes only the hosts. */}} -{{- define "adapter-service.singleNetworkLayer.routing" -}} +{{- define "adapter-service.networkAdapterLayer.routing" -}} {{- $ctx := . -}} {{- $loopback := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} {{- /* Whole URL, anchored both ends: only a tier host (optionally with a namespace suffix and port) followed by nothing or a path is rewritten. */ -}} {{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} {{- $out := dict -}} -{{- range $pair := list (list "routing-consumer.yaml" "routing-single-network-layer-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-single-network-layer-publish.yaml") -}} +{{- range $pair := list (list "routing-consumer.yaml" "routing-network-adapter-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-network-adapter-publish.yaml") -}} {{- $src := index $pair 0 -}} {{- $doc := $ctx.Files.Get (printf "config/%s" $src) | fromYaml -}} {{- if not $doc.routingRules -}} -{{- fail (printf "%s: config/%s parsed to no routingRules; the single-network-layer routing is generated from it." $ctx.Chart.Name $src) -}} +{{- fail (printf "%s: config/%s parsed to no routingRules; the network-adapter-layer routing is generated from it." $ctx.Chart.Name $src) -}} {{- end -}} {{- range $i, $rule := $doc.routingRules -}} {{- if $rule.target -}} {{- $url := $rule.target.url | default "" -}} {{- $url = regexReplaceAll $tierHost $url $loopback -}} {{- if hasSuffix "/" $url -}} -{{- fail (printf "%s: single-network-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} +{{- fail (printf "%s: network-adapter-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} {{- end -}} {{- $_ := set $rule.target "url" $url -}} {{- end -}} @@ -78,12 +78,12 @@ modules across pods again changes only the hosts. {{/* Secret holding one tier's keypair. Fails naming the tier: a missing identity would otherwise start, report Ready, and fail every signature that tier makes. -Usage: include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" $ "tier" "network") +Usage: include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" $ "tier" "network") */}} -{{- define "adapter-service.singleNetworkLayer.tierSecret" -}} -{{- $name := (index .ctx.Values.singleNetworkLayer.keys .tier).existingSecret | default "" -}} +{{- define "adapter-service.networkAdapterLayer.tierSecret" -}} +{{- $name := (index .ctx.Values.networkAdapterLayer.keys .tier).existingSecret | default "" -}} {{- if not $name -}} -{{- fail (printf "%s: singleNetworkLayer.keys.%s.existingSecret is required -- the %s tier signs as its own identity, from its own Secret (the six keys under keys.existingSecret)." .ctx.Chart.Name .tier .tier) -}} +{{- fail (printf "%s: networkAdapterLayer.keys.%s.existingSecret is required -- the %s tier signs as its own identity, from its own Secret (the six keys under keys.existingSecret)." .ctx.Chart.Name .tier .tier) -}} {{- end -}} {{- $name -}} {{- end }} diff --git a/charts/adapter-service/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml index 1ad09b01..ea9534fa 100644 --- a/charts/adapter-service/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -10,8 +10,8 @@ Everything that differs between the provider, network and consumer adapters is a value: they are one image with one config format, and the role decides the identity, the handler role, the step list and where requests are routed. -role single-network-layer reads config/single-network-layer-config.yaml; its routing -files are generated from the tier routing files (_single-network-layer.tpl), and it +role network-adapter-layer reads config/network-adapter-config.yaml; its routing +files are generated from the tier routing files (_network-adapter.tpl), and it renders more than one routing file. */}} {{/* @@ -28,23 +28,23 @@ at pod start, which is why no rendered config ever holds a key. A raw config with a literal private key in it would put that key in git, in a ConfigMap, and in `helm get values` -- so the render refuses it. */}} -{{- $singleNetworkLayer := include "adapter-service.isSingleNetworkLayer" . }} +{{- $networkAdapterLayer := include "adapter-service.isNetworkAdapterLayer" . }} {{- $nlCfg := "" }} -{{- if $singleNetworkLayer }} -{{- /* role single-network-layer: the config is config/single-network-layer-config.yaml, +{{- if $networkAdapterLayer }} +{{- /* role network-adapter-layer: the config is config/network-adapter-config.yaml, unless `config` replaces it whole. Either way every private key must be a tier placeholder; checked on the parsed config. */}} -{{- $nlCfg = .Values.config | default (.Files.Get "config/single-network-layer-config.yaml") }} +{{- $nlCfg = .Values.config | default (.Files.Get "config/network-adapter-config.yaml") }} {{- if not $nlCfg }} -{{- fail (printf "%s: role single-network-layer reads config/single-network-layer-config.yaml, which is missing from the chart." .Chart.Name) }} +{{- fail (printf "%s: role network-adapter-layer reads config/network-adapter-config.yaml, which is missing from the chart." .Chart.Name) }} {{- end }} {{- $nlParsed := $nlCfg | fromYaml }} {{- if or (not $nlParsed) (hasKey $nlParsed "Error") }} -{{- fail (printf "%s: the single-network-layer adapter config (`config` override or config/single-network-layer-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} +{{- fail (printf "%s: the network-adapter-layer adapter config (`config` override or config/network-adapter-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} {{- end }} -{{- include "adapter-service.singleNetworkLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} +{{- include "adapter-service.networkAdapterLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} {{- if ne (toString (dig "http" "port" "" $nlParsed)) (toString .Values.service.targetPort) }} -{{- fail (printf "%s: the single-network-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} +{{- fail (printf "%s: the network-adapter-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} {{- end }} {{- else }} {{- $cfgFile := printf "config/%s-config.yaml" (include "adapter-service.role" .) }} @@ -68,13 +68,13 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: - {{- if $singleNetworkLayer }} + {{- if $networkAdapterLayer }} adapter.yaml: | {{- $nlCfg | nindent 4 }} {{- if .Values.routing.config }} - {{- fail (printf "%s: routing.config is not used by the single-network-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} + {{- fail (printf "%s: routing.config is not used by the network-adapter-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} {{- end }} - {{- range $file, $doc := include "adapter-service.singleNetworkLayer.routing" . | fromYaml }} + {{- range $file, $doc := include "adapter-service.networkAdapterLayer.routing" . | fromYaml }} {{ $file }}: | # GENERATED by adapter-service from config/routing-*.yaml. Do not edit; edit those files. {{- toYaml $doc | nindent 4 }} diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index 1e5fa7b1..b8664fc2 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -86,8 +86,8 @@ spec: - -c - | set -eu - {{- if include "adapter-service.isSingleNetworkLayer" . }} - # single-network-layer: adapter.yaml plus one routing file per routed module. + {{- if include "adapter-service.isNetworkAdapterLayer" . }} + # network-adapter-layer: adapter.yaml plus one routing file per routed module. cp /config-template/*.yaml /rendered/ {{- else }} cp /config-template/adapter.yaml /rendered/adapter.yaml @@ -97,7 +97,7 @@ spec: subst() { # $1 placeholder, $2 file holding the value if [ ! -s "$2" ]; then - {{- if include "adapter-service.isSingleNetworkLayer" . }} + {{- if include "adapter-service.isNetworkAdapterLayer" . }} # three tier Secrets share key names: name the tier too (e.g. network/keyId) echo "render-config: ${2#/keys/} is missing or empty in its tier's keys Secret" >&2 {{- else }} @@ -111,7 +111,7 @@ spec: # -- the sentinel contains a colon -- or an adapter signing with an # identity nothing can verify. if head -c 9 "$2" | grep -q "<&2 {{- else }} echo "render-config: $(basename "$2") is still unresolved." >&2 @@ -123,7 +123,7 @@ spec: fi sed -i "s|$1|$(cat "$2")|g" /rendered/adapter.yaml } - {{- if include "adapter-service.isSingleNetworkLayer" . }} + {{- if include "adapter-service.isNetworkAdapterLayer" . }} {{- /* Three identities: each tier's module signs as that tier, from that tier's Secret mounted at /keys/. */}} @@ -137,7 +137,7 @@ spec: subst __{{ $T }}_ENCR_PRIVATE__ /keys/{{ $tier }}/{{ $k.encrPrivateKey }} subst __{{ $T }}_ENCR_PUBLIC__ /keys/{{ $tier }}/{{ $k.encrPublicKey }} {{- end }} - {{- range $ph, $path := .Values.singleNetworkLayer.extraSubstitutions }} + {{- range $ph, $path := .Values.networkAdapterLayer.extraSubstitutions }} subst {{ $ph }} /keys/{{ $path }} {{- end }} {{- else }} @@ -170,7 +170,7 @@ spec: - name: config-template mountPath: /config-template readOnly: true - {{- if include "adapter-service.isSingleNetworkLayer" . }} + {{- if include "adapter-service.isNetworkAdapterLayer" . }} {{- range $tier := list "consumer" "network" "provider" }} - name: keys-{{ $tier }} mountPath: /keys/{{ $tier }} @@ -243,11 +243,11 @@ spec: - name: config-template configMap: name: {{ include "adapter-service.fullname" . }}-config - {{- if include "adapter-service.isSingleNetworkLayer" . }} + {{- if include "adapter-service.isNetworkAdapterLayer" . }} {{- range $tier := list "consumer" "network" "provider" }} - name: keys-{{ $tier }} secret: - secretName: {{ include "adapter-service.singleNetworkLayer.tierSecret" (dict "ctx" $ "tier" $tier) }} + secretName: {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" $ "tier" $tier) }} defaultMode: 0400 {{- end }} {{- else }} diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index 2bfed931..29f14834 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,6 +1,6 @@ {{- if .Values.ingress.enabled }} -{{- if include "adapter-service.isSingleNetworkLayer" . }} -{{- /* role single-network-layer: only single-segment / paths (the +{{- if include "adapter-service.isNetworkAdapterLayer" . }} +{{- /* role network-adapter-layer: only single-segment / paths (the consumer module is mounted at /) and individual /provider/ paths may be public. "/" or "/provider" with Prefix would also expose /network (an internal tier) and /provider/publish, which has no validateSign and @@ -11,25 +11,25 @@ be literal, and regex or rewrite annotations are refused. */}} {{- $ann := merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict) }} {{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString) "true" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the single-network-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} {{- end }} {{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the single-network-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-adapter-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} {{- end }} {{- range .Values.ingress.hosts }} {{- range .paths }} {{- $p := .path | default "" }} {{- $pt := .pathType | default "Prefix" }} {{- if not (has $pt (list "Prefix" "Exact")) }} -{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the single-network-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} +{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-adapter-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} {{- end }} {{- if or (not (regexMatch "^/[A-Za-z0-9/_.-]*$" $p)) (contains ".." $p) (contains "//" $p) }} -{{- fail (printf "%s: ingress path %q is not allowed for the single-network-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} +{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} {{- end }} {{- $consumer := and (regexMatch "^/[A-Za-z0-9_.-]+$" $p) (not (has $p (list "/network" "/provider"))) }} {{- $provider := and (hasPrefix "/provider/" $p) (ne $p "/provider/") (not (hasPrefix "/provider/publish" $p)) }} {{- if not (or $consumer $provider) }} -{{- fail (printf "%s: ingress path %q is not allowed for the single-network-layer role. Allowed: single-segment / (the consumer, e.g. /discover) and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} +{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. Allowed: single-segment / (the consumer, e.g. /discover) and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} {{- end }} {{- end }} {{- end }} @@ -39,7 +39,7 @@ Off by default, and worth a thought before turning on. The provider and network roles take SIGNED requests -- validateSign is their first step -- so publishing them is not the same risk as publishing an adapter -that does not. The consumer role, and the single-network-layer consumer module +that does not. The consumer role, and the network-adapter-layer consumer module at /, take UNSIGNED requests (NOTES says so when ingress is on). Either way it is still a Beckn peer endpoint, and the thing that makes it safe is that every caller must hold a key the registry publishes. That is a property of the diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 7192650a..f0a94627 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -2,7 +2,7 @@ # adapter-service # # One chart for the Beckn adapters: provider, network and consumer (one release -# per tier), or single-network-layer (all three tiers in one adapter). Same image and +# per tier), or network-adapter-layer (all three tiers in one adapter). Same image and # config format; `role` below decides the handler role, the step list and # where requests are routed. # @@ -14,7 +14,7 @@ # # OR, all three tiers in one adapter (one release instead of the three above): # -# helm install single-network-layer-adapter . -f examples/single-network-layer.yaml +# helm install network-adapter . -f examples/network-adapter.yaml # # The release name is what the Service is called, which is the DNS name the # other adapters address. Rename a release and you must update whatever routes @@ -30,7 +30,7 @@ # =========================================================================== # --------------------------------------------------------------------------- -# Role: provider | network | consumer | single-network-layer +# Role: provider | network | consumer | network-adapter-layer # # Required, with no default. A default would hand one role's handler role and # step list to a different adapter, which renders, starts, reports Ready and @@ -417,11 +417,11 @@ extraEnv: [] envFromSecrets: [] # --------------------------------------------------------------------------- -# role: single-network-layer -- the single network layer adapter. +# role: network-adapter-layer -- the Network Adapter Layer. # # The consumer, network and provider tiers as four modules of ONE process # (/ for the consumer, /network/, /provider/, /provider/publish). Its adapter config is -# config/single-network-layer-config.yaml, mounted verbatim like any role's +# config/network-adapter-config.yaml, mounted verbatim like any role's # file: log level, timeouts, telemetry, extended schema and the outbound # connection pool are set there. The routing is generated at render time from # the tier routing files, with tier-to-tier hops pointed at this pod's own @@ -432,12 +432,12 @@ envFromSecrets: [] # # NOT USED by this role: `routing.config` (setting it fails the render, because # the routing is generated) and `handler.*` (role, steps and path are in the -# config file). `config` replaces config/single-network-layer-config.yaml +# config file). `config` replaces config/network-adapter-config.yaml # WHOLE: the private-key guard still runs on it, and it must point its routers -# at /app/config/routing-single-network-layer-consumer.yaml, routing-network.yaml and -# routing-single-network-layer-publish.yaml. +# at /app/config/routing-network-adapter-consumer.yaml, routing-network.yaml and +# routing-network-adapter-publish.yaml. # --------------------------------------------------------------------------- -singleNetworkLayer: +networkAdapterLayer: # Three identities, one per tier: the same Secrets (six keys each, named as # under keys.existingSecret) and the same registry rows the multi-adapter # releases use, so either mode runs against the same registry. They must diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index 63209e41..bd26ff79 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -8,10 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.1] - 2026-10-09 ### Added -- `single-network-layer-adapter` in the namespaces of `consumer-adapter-keys`, - `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for - `adapter-service` role `single-network-layer` (all three tiers in one adapter, three - identities). Inert where that namespace does not exist. +- `network-adapter` as a target namespace of `consumer-adapter-keys`, + `provider-adapter-keys` and `provider-upstream` (`network-adapter-keys` + already reflects there), for `adapter-service` role `network-adapter-layer`: + the Network Adapter Layer runs all three tiers in one release named + `network-adapter`, so it reads all three identities. Inert where that + namespace does not exist. ## [0.1.0] - 2026-09-27 diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index 3bdd9b13..3559c8b6 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -57,16 +57,16 @@ secrets: # -- adapter identities. keyId inside these is filled in by # resolve-key-ids.sh AFTER the registry has been seeded, so these entries # change once per environment after the first bring-up. - # single-network-layer-adapter (role single-network-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. + # network-adapter (role network-adapter-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. - name: consumer-adapter-keys - namespaces: [consumer-adapter, single-network-layer-adapter] + namespaces: [consumer-adapter, network-adapter] - name: network-adapter-keys - namespaces: [network-adapter, single-network-layer-adapter] + namespaces: [network-adapter] - name: provider-adapter-keys - namespaces: [provider-adapter, single-network-layer-adapter] + namespaces: [provider-adapter, network-adapter] # The provider adapter's upstream credentials, named as environment variables - # by its config rather than substituted into it. single-network-layer-adapter runs + # by its config rather than substituted into it. network-adapter runs # the provider tier too, so it reads them as well. - name: provider-upstream - namespaces: [provider-adapter, single-network-layer-adapter] + namespaces: [provider-adapter, network-adapter] diff --git a/quick-start/.env.example b/quick-start/.env.example index 3736c2b8..a4d7dda2 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,9 +39,9 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 -# The single network layer adapter (the default topology). Inside the container +# The Network Adapter (the default topology). Inside the container # it always listens on 9200; this only moves the host side. -SINGLE_NETWORK_LAYER_ADAPTER_PORT=9200 +NETWORK_ADAPTER_LAYER_PORT=9200 # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 diff --git a/quick-start/.gitignore b/quick-start/.gitignore index cbb83f8e..2351f021 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -16,9 +16,9 @@ config/adapters/experience.yaml config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml -config/adapters/single-network-layer.yaml -bin/demo-single-network-layer.sh -bin/bench-single-network-layer.sh +config/adapters/network-adapter.yaml +bin/demo-network-adapter.sh +bin/bench-network-adapter.sh poc-flow.py bin/poc # Apple-silicon-only override that forces amd64 emulation for images with no @@ -42,5 +42,5 @@ config/discovery/instance.yaml # nothing here to ignore -- and nothing here to review either. Back those # volumes up; they are the only copy. -# bin/bench-single-network-layer.sh scratch: generated bodies and results +# bin/bench-network-adapter.sh scratch: generated bodies and results .bench/ \ No newline at end of file diff --git a/quick-start/Makefile b/quick-start/Makefile index e65f48ef..026b2d8a 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -3,7 +3,7 @@ # startup order matters, and a good place to make the right order the shortest # thing to type. # -# make up the whole stack (single network layer adapter), in order +# make up the whole stack (Network Adapter), in order # make up-core the same minus the reverse proxy and hyperdx # make up-multi the whole stack with three adapters instead of one # make smoke publish, discover, select through what is running diff --git a/quick-start/README.md b/quick-start/README.md index 7405c877..67ee3b3c 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -4,8 +4,8 @@ The whole OAN stack in Docker Compose: a registry, a discovery service, the adapter tiers, and two mock upstreams standing in for real provider APIs. The three adapter tiers (consumer, network, provider) run as **one** container -by default: `single-network-layer-adapter`, the tiers as modules of one process -on port 9200. `make up-core-multi` runs them as three containers instead. Same +by default, the Network Adapter Layer (container `network-adapter`): the tiers +as modules of one process on port 9200. `make up-core-multi` runs them as three containers instead. Same compose file, same keys, same registry → [Two topologies](#two-topologies). Nothing is built here — the images are pulled. Locally that is about ten @@ -31,7 +31,7 @@ where the reason lives. adapter, which files the catalogue in discovery. In the default single topology each box above is a module of the one -`single-network-layer-adapter`, and the arrows between them are loopback calls +`network-adapter`, and the arrows between them are loopback calls inside that container. All three adapters read the registry — who signed this request, and where this @@ -86,14 +86,14 @@ cp .env.example .env Locally, nothing in it needs changing. It publishes these on `localhost`: ``` -8081 registry 9200 single network layer adapter (default) 9100 mock-imd +8081 registry 9200 Network Adapter Layer (default) 9100 mock-imd 8080 keycloak or provider adapter (multi) 9101 mock-agmarknet 9990 keycloak admin 9201 network adapter (multi only) 8090 discovery 9202 consumer adapter (multi only) ``` If one is already taken, change it here — that is the only edit a local run -needs. The single adapter's host port is `SINGLE_NETWORK_LAYER_ADAPTER_PORT`; +needs. The single adapter's host port is `NETWORK_ADAPTER_LAYER_PORT`; in multi mode 9200 is `PROVIDER_ADAPTER_PORT`. Adapters reach each other by Compose service name, so it only changes what you type into Postman. @@ -104,16 +104,17 @@ make up-core ``` Three tiers, in the only order that works: registry and discovery, then -`bin/setup.py`, then the adapter: `single-network-layer-adapter` on 9200. +`bin/setup.py`, then the Network Adapter Layer (container `network-adapter`) on 9200. Allow up to five minutes the first time — Keycloak on a cold volume. -Three adapters instead: `make up-core-multi`. Either target stops the other -topology's adapters first (both use 9200), so switching is just running the -other one. +Three adapters instead: `make up-core-multi`. Either target removes the other +topology's adapters first (both use 9200, and both have a container called +`network-adapter`: the Network Adapter Layer in one, the network tier in the +other), so switching is just running the other one. `setup.py` generates a keypair per tier, registers seven participants and four capability bindings, and renders all four adapter configs (the three tier -configs and the single one). Nothing needs creating by hand. → Appendix C for +configs and the Network Adapter Layer's). Nothing needs creating by hand. → Appendix C for why the order matters, Appendix D for what it wrote. ```sh @@ -124,9 +125,9 @@ make smoke # publish, discover, select through whatever is running ## Step 5 — Provider layer > Steps 5–7 describe the three tiers. In the default single topology they are -> modules of one container, so read `single-network-layer-adapter` wherever a +> modules of one container, so read `network-adapter` wherever a > command names `provider-adapter`, `network-adapter` or `consumer-adapter` -> (e.g. `docker compose logs single-network-layer-adapter`). The `.env` keys +> (e.g. `docker compose logs network-adapter`). The `.env` keys > are the same in both. Answers `select`, and the only layer that calls an upstream. Serves all four @@ -196,7 +197,7 @@ Two copies of the full collection, same requests, in `../api-collection/`: | Topology | Collection | Environment | |---|---|---| -| single (`make up-core`, default) | `OpenAgriNet.single-network-layer.api-collection.json` | `OpenAgriNet.single-network-layer.local-environment.json` | +| single (`make up-core`, default) | `OpenAgriNet.network-adapter.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` | | multi (`make up-core-multi`) | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | Import both files from `../api-collection/` into Postman — @@ -233,9 +234,9 @@ collection. → Appendix N. The three adapter tiers can run two ways. Same `docker-compose.yml`, same image, same keys, registry and data; a compose profile picks the adapters. -| | Single network layer (default) | Multi-adapter | +| | Network Adapter Layer (default) | Multi-adapter | |---|---|---| -| What runs | one `single-network-layer-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | +| What runs | one container, `network-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | | Compose profile | `single` | `multi` | | Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | | Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | @@ -244,7 +245,7 @@ image, same keys, registry and data; a compose profile picks the adapters. | Tier-to-tier hops | loopback HTTP inside the container (`http://localhost:9200/...`) | HTTP between containers | Only one topology runs at a time (both use 9200). Switching is running the -other target: it stops the running topology's adapters first. `make down`, +other target: it removes the running topology's adapters first. `make down`, `make ps` and `make logs` cover both. `MODE=multi` also works with `make up`, `make up-core` and `make restart`. @@ -255,24 +256,24 @@ in `.env` (`single` in `.env.example`); without it no adapter starts. Signing is off in every quick-start template, in both topologies, so keep the adapter ports on `127.0.0.1` (the compose file does). -**Config.** `config/adapters/single-network-layer.yaml.tmpl`, beside the three -tier templates. `bin/setup.py` fills it into `config/adapters/single-network-layer.yaml` +**Config.** `config/adapters/network-adapter.yaml.tmpl`, beside the three +tier templates. `bin/setup.py` fills it into `config/adapters/network-adapter.yaml` like the others. Each module is its tier's module from the tier template (a change to a tier template that should apply here goes in this file too), plus extended schema validation on every module, an outbound connection pool and log level info. Each module keeps its own tier identity. The same layout as the Helm chart's -`config/single-network-layer-config.yaml`. +`config/network-adapter-config.yaml`. | Module | Path | From | Routing file | Routes | |---|---|---|---|---| -| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-single-network-layer-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | +| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | | `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (multi file, reused) | discover, publish to discovery | | `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | -| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-single-network-layer-publish.yaml` | to `localhost:9200/network/publish` | +| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/network/publish` | The hops between modules are loopback HTTP calls to the same container. In the multi-adapter stack only the hosts differ. 9200 is the container's own listener -and must match `http.port` in `single-network-layer.yaml.tmpl`. +and must match `http.port` in `network-adapter.yaml.tmpl`. Check it: `make smoke`. @@ -317,7 +318,7 @@ make up `make up`, not `make up-core`: two more tiers on top of Part 1's three — nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes -the VM reachable from the internet.** It runs the single network layer +the VM reachable from the internet.** It runs the Network Adapter Layer adapter; `make up-multi` runs the three adapters instead, and decides what the proxy hosts in Appendix B forward to. @@ -454,7 +455,7 @@ certificate. An unknown `Host` gets NPM's default page, not an adapter. **Single topology (`make up`, the default).** The table above is for `make up-multi`. With the single adapter there is no `consumer-adapter`, `network-adapter` or `provider-adapter` to forward to: point the consumer host -at `single-network-layer-adapter`, port 9200, with the same Advanced paste. The +at `network-adapter`, port 9200, with the same Advanced paste. The app's paths (`/discover`, `/select`, …) do not change. Do **not** make `/network/…` or `/provider/publish` reachable from a public host: `/network/` is an internal tier and `/provider/publish` publishes as the provider without @@ -613,7 +614,7 @@ loopback. A registry route would depend on that silently. ``` 1. registry and discovery (also sunbird-registry-postgres, sunbird-registry-keycloak, discovery-postgres) 2. bin/setup.py keys, seven participants, four bindings, four configs -3. the adapters single-network-layer-adapter (default), or the three with MODE=multi +3. the adapters network-adapter (default), or the three with MODE=multi 4. nginx-proxy-manager the public edge — 80 and 443, all interfaces 5. hyperdx ClickStack ``` diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 4d04a5bd..48b24ec5 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -637,19 +637,19 @@ def _render_one(name, template, substitutions): _write(f"{name}.yaml", _fill(name, template, substitutions)) -# ------------------------------------------------------ single network layer +# ------------------------------------------------------ Network Adapter Layer # -# The single network layer runs the consumer, network and provider tiers as +# The Network Adapter Layer runs the consumer, network and provider tiers as # modules of ONE adapter (docker-compose.yml, profile "single"). Its -# template, single-network-layer.yaml.tmpl, carries all three tiers' +# template, network-adapter.yaml.tmpl, carries all three tiers' # placeholders: each module keeps its own tier identity, so both stacks use the # same keys and registry rows. -def _render_single_network_layer(identities, shared): +def _render_network_adapter_layer(identities, shared): substitutions = shared for role in ROLES: substitutions = _role_substitutions(role, identities[role]) + substitutions - _render_one("single-network-layer", - (ADAPTERS / "single-network-layer.yaml.tmpl").read_text(), + _render_one("network-adapter", + (ADAPTERS / "network-adapter.yaml.tmpl").read_text(), substitutions) @@ -660,7 +660,7 @@ def render(identities): template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() _render_one(role, template, _role_substitutions(role, identities[role]) + shared) - _render_single_network_layer(identities, shared) + _render_network_adapter_layer(identities, shared) if __name__ == "__main__": diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 9ffdc4a3..06d67c33 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -37,9 +37,12 @@ cd "$ROOT" # makes "down" mean down. PROFILES=(--profile reverse-proxy --profile observability --profile single --profile multi) -# Adapter topology: "single" (default) runs single-network-layer-adapter, one -# process for all three tiers; "multi" runs provider-, network- and -# consumer-adapter. Both publish 127.0.0.1:9200, so only one runs at a time. +# Adapter topology: "single" (default) runs the Network Adapter Layer (compose +# service network-adapter-layer, container network-adapter), one process for +# all three tiers; "multi" runs provider-, network- and consumer-adapter. Both +# publish 127.0.0.1:9200, and the single container shares the multi network +# tier's name, so only one topology exists at a time: the other is REMOVED +# (not just stopped) before this one starts. # Validated only by the commands that use it (check_mode), so an unrelated # MODE in the shell never blocks `down`, `destroy` or `help`. MODE="${MODE:-single}" @@ -48,15 +51,19 @@ check_mode() { } # The topology whose adapters are running now: single, multi, or empty. +# Both topologies have a container called network-adapter, so single is told +# apart by its compose service label. running_mode() { - if [ "$(docker inspect -f '{{.State.Running}}' single-network-layer-adapter 2>/dev/null)" = true ]; then echo single - elif [ "$(docker inspect -f '{{.State.Running}}' consumer-adapter 2>/dev/null)" = true ]; then echo multi + if [ "$(docker inspect -f '{{.State.Running}}' consumer-adapter 2>/dev/null)" = true ]; then echo multi + elif [ "$(docker inspect -f '{{.State.Running}}' network-adapter 2>/dev/null)" = true ] \ + && [ "$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' network-adapter 2>/dev/null)" = network-adapter-layer ]; then + echo single fi } adapters_for() { # $1 mode case "$1" in - single) echo single-network-layer-adapter ;; + single) echo network-adapter-layer ;; multi) echo provider-adapter network-adapter consumer-adapter ;; esac } @@ -157,8 +164,8 @@ up_registry_tier() { } # Generates the adapter keypairs, registers the three adapter identities, and -# renders every adapter config (provider, network, consumer and -# single-network-layer) from the .tmpl files beside them. Safe to re-run: +# renders every adapter config (provider, network, consumer and the +# Network Adapter Layer's network-adapter.yaml) from the .tmpl files beside them. Safe to re-run: # keys come from keys/keys.json once it exists, and participants already # registered are left alone. up_setup() { @@ -179,11 +186,15 @@ up_adapters() { check_mode local other f; other=$(other_mode "$MODE") step 3 "$1" "adapters, mode $MODE ($(adapters_for "$MODE"))" - # The other topology also publishes 127.0.0.1:9200: stop it first. + # The other topology also publishes 127.0.0.1:9200, and the single container + # shares the multi network tier's name: remove it (stopped is not enough). # shellcheck disable=SC2046 - docker compose --profile "$other" stop $(adapters_for "$other") >/dev/null 2>&1 || true + docker compose --profile "$other" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true + # Containers from before the rename (single-network-layer-adapter, and the + # earlier network-layer-adapter) would hold 127.0.0.1:9200. + docker rm -f single-network-layer-adapter network-layer-adapter >/dev/null 2>&1 || true # Never let compose create a directory where a bind-mounted config belongs. - for f in $( [ "$MODE" = single ] && echo single-network-layer || echo provider network consumer ); do + for f in $( [ "$MODE" = single ] && echo network-adapter || echo provider network consumer ); do [ -f "config/adapters/$f.yaml" ] \ || die "config/adapters/$f.yaml was not rendered (a directory there? rm -r it, then re-run)" done @@ -375,8 +386,8 @@ smoke() { local mode base_pub base_app svc s mode=$(running_mode) if [ "$mode" = single ]; then - base_app="http://127.0.0.1:$(envv SINGLE_NETWORK_LAYER_ADAPTER_PORT 9200)"; base_pub="$base_app/provider" - svc=single-network-layer-adapter + base_app="http://127.0.0.1:$(envv NETWORK_ADAPTER_LAYER_PORT 9200)"; base_pub="$base_app/provider" + svc=network-adapter elif [ "$mode" = multi ]; then base_pub="http://127.0.0.1:$(envv PROVIDER_ADAPTER_PORT 9200)" base_app="http://127.0.0.1:$(envv CONSUMER_ADAPTER_PORT 9202)" svc="provider-adapter network-adapter consumer-adapter" @@ -438,8 +449,8 @@ usage() { cat <<'USAGE' bin/stack.sh - up the whole stack, single network layer (default): - registry+discovery -> setup.py -> single-network-layer-adapter + up the whole stack, Network Adapter Layer (default): + registry+discovery -> setup.py -> network-adapter (Network Adapter Layer) -> gateway (public, 80/443) -> hyperdx (wants 16 GB) up-core steps 1-3 only. Nothing public, no ClickHouse. up-multi as up, with the three adapters (provider, network, consumer) diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index d070ff43..09022c17 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -1,4 +1,4 @@ -# Single network layer adapter: the consumer, network and provider tiers as +# Network Adapter: the consumer, network and provider tiers as # four modules of ONE adapter process (docker-compose.yml, profile "single"). # # / the consumer (bap); the app posts here, unsigned @@ -9,11 +9,11 @@ # Each module is its tier's module from consumer.yaml.tmpl, network.yaml.tmpl # and provider.yaml.tmpl. A change to one of those templates that should also # apply here must be made in this file too. The same layout as the Helm chart's -# charts/adapter-service/config/single-network-layer-config.yaml. +# charts/adapter-service/config/network-adapter-config.yaml. # # Differences from the three tier templates: # - one listener (9200) and one appName; tier hops are loopback HTTP to this -# same listener (routing-single-network-layer-*.yaml beside this file); +# same listener (routing-network-adapter-*.yaml beside this file); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; # - log level info. @@ -22,13 +22,13 @@ # sign or validateSign, so /network/ and /provider/ accept unsigned requests # from anyone who can reach :9200. Keep it on 127.0.0.1 (as the compose file # does) and never put /network/ or /provider/ behind the reverse proxy. The -# Helm chart's single-network-layer-config.yaml has signing on. +# Helm chart's network-adapter-config.yaml has signing on. # # bin/setup.py fills the placeholders -- each module's own tier identity # (the CONSUMER_, NETWORK_ and PROVIDER_ ones) and the shared ones -- and -# writes config/adapters/single-network-layer.yaml (mode 600, gitignored: it +# writes config/adapters/network-adapter.yaml (mode 600, gitignored: it # holds private keys). -appName: single-network-layer-adapter +appName: network-adapter log: level: info @@ -54,7 +54,7 @@ plugins: otelsetup: id: otelsetup config: - serviceName: single-network-layer-adapter + serviceName: network-adapter environment: "__OTEL_ENVIRONMENT__" otlpEndpoint: "__OTLP_ENDPOINT__" enableMetrics: "__OTEL_ENABLED__" @@ -66,7 +66,7 @@ modules: # From consumer.yaml.tmpl. Mounted at /, so the app calls /discover, /select, # ... as on the multi-adapter consumer-adapter. Accepts unsigned requests # under the consumer identity (signing is off here), routes discover to /network and transactions to - # /provider (routing-single-network-layer-consumer.yaml). + # /provider (routing-network-adapter-consumer.yaml). - name: consumer path: / handler: @@ -123,7 +123,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-single-network-layer-consumer.yaml + routingConfig: /app/config/routing-network-adapter-consumer.yaml steps: - validateSchema - addRoute @@ -301,7 +301,7 @@ modules: # ---------------------------------------------------------- provider-publish # From provider.yaml.tmpl, module 2. The catalogue publishes here, under the # provider identity (signing is off here); routed to /network/publish - # (routing-single-network-layer-publish.yaml). + # (routing-network-adapter-publish.yaml). - name: provider-publish path: /provider/publish handler: @@ -356,7 +356,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-single-network-layer-publish.yaml + routingConfig: /app/config/routing-network-adapter-publish.yaml steps: - validateSchema - addRoute diff --git a/quick-start/config/adapters/routing-network-adapter-consumer.yaml b/quick-start/config/adapters/routing-network-adapter-consumer.yaml index 73ef9336..b07d7119 100644 --- a/quick-start/config/adapters/routing-network-adapter-consumer.yaml +++ b/quick-start/config/adapters/routing-network-adapter-consumer.yaml @@ -1,6 +1,5 @@ -# What the consumer layer does with each action, in the single network layer -# (the single network layer adapter: consumer, network and provider as modules of one -# adapter). +# What the consumer layer does with each action, in the Network Adapter Layer +# (consumer, network and provider as modules of one adapter). # # Both targets are THIS SAME adapter on another path. In the multi-adapter # stack they are two other containers; here the hop is a loopback call from one @@ -10,7 +9,7 @@ # # targetType "url" appends the action to the path, so the base here is the # module mount point and /discover or /select is added to it. localhost:9200 is -# the container's own listener and must match `http.port` in single-network-layer.yaml.tmpl. +# the container's own listener and must match `http.port` in network-adapter.yaml.tmpl. routingRules: # Discovery is answered from this network's own discovery service, which the # network module fronts. diff --git a/quick-start/config/adapters/routing-network-adapter-publish.yaml b/quick-start/config/adapters/routing-network-adapter-publish.yaml index 8a04cb11..19838113 100644 --- a/quick-start/config/adapters/routing-network-adapter-publish.yaml +++ b/quick-start/config/adapters/routing-network-adapter-publish.yaml @@ -1,5 +1,5 @@ -# This network's catalogue system publishing inward, in the single network -# layer (the single network layer adapter). +# This network's catalogue system publishing inward, in the Network Adapter +# Layer. # # It goes to the NETWORK module rather than straight to discovery, so a publish # takes the same path a publish from anywhere else takes -- one place that diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index d4ca23bd..5d22ce43 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -11,7 +11,7 @@ # registry sunbird-registry-postgres, sunbird-registry-keycloak, # sunbird-registry-service # discovery discovery-postgres, discovery-service -# adapters single-network-layer-adapter (profile single, default) | +# adapters network-adapter-layer (profile single, default) | # provider, network, consumer (profile multi) # observability hyperdx profile: observability # edge nginx-proxy-manager profile: reverse-proxy @@ -27,7 +27,7 @@ # everything EXCEPT the adapters # 3. python3 bin/setup.py keys, the three tier identities, and # every adapter config they mount -# 4. docker compose --profile single up -d the single network layer adapter +# 4. docker compose --profile single up -d the Network Adapter # (or --profile multi: provider-, network-, consumer-adapter) # 5. docker compose --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack @@ -538,7 +538,7 @@ services: # adapters -- TWO topologies, chosen with a compose profile. Same image. # # profile "single" (the default; `make up`) - # single-network-layer-adapter: the consumer, network and provider tiers + # network-adapter-layer (container network-adapter): the consumer, network and provider tiers # as four modules of ONE process, one config, one port (127.0.0.1:9200). # Tier-to-tier hops are loopback HTTP inside the container. # @@ -600,9 +600,9 @@ services: ports: - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" - # The single network layer: the consumer, network and provider tiers as + # The Network Adapter Layer: the consumer, network and provider tiers as # modules of ONE adapter process, on ONE config rendered by bin/setup.py from - # config/adapters/single-network-layer.yaml.tmpl, with one routing file per + # config/adapters/network-adapter.yaml.tmpl, with one routing file per # module. # # / the consumer, the caller's entry (bap), takes unsigned requests @@ -614,10 +614,10 @@ services: # the same keys and registry rows as the multi-adapter services. Signing is # off, as in every quick-start template, so /network/ and /provider/ accept # unsigned requests: keep :9200 on 127.0.0.1. - single-network-layer-adapter: + network-adapter-layer: <<: *adapter profiles: ["single"] - container_name: single-network-layer-adapter + container_name: network-adapter depends_on: sunbird-registry-service: condition: service_healthy @@ -625,7 +625,7 @@ services: condition: service_started environment: <<: *adapter-env - OTEL_SERVICE_NAME: single-network-layer-adapter + OTEL_SERVICE_NAME: network-adapter # Upstream credentials, read at call time -- the same variables and # reasons as provider-adapter below. MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} @@ -635,14 +635,14 @@ services: VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} volumes: - - ./config/adapters/single-network-layer.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/network-adapter.yaml:/app/config/adapter.yaml:ro # One routing file per module. The network module reuses the # multi-adapter routing-network.yaml unchanged. - - ./config/adapters/routing-single-network-layer-consumer.yaml:/app/config/routing-single-network-layer-consumer.yaml:ro + - ./config/adapters/routing-network-adapter-consumer.yaml:/app/config/routing-network-adapter-consumer.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - - ./config/adapters/routing-single-network-layer-publish.yaml:/app/config/routing-single-network-layer-publish.yaml:ro + - ./config/adapters/routing-network-adapter-publish.yaml:/app/config/routing-network-adapter-publish.yaml:ro ports: - - "127.0.0.1:${SINGLE_NETWORK_LAYER_ADAPTER_PORT:-9200}:9200" + - "127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9200}:9200" # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, @@ -1002,7 +1002,7 @@ services: # # required: false -- only one topology's adapters exist at a time # (profiles "single" / "multi"); NPM starts with whichever is running. - single-network-layer-adapter: + network-adapter-layer: condition: service_started required: false consumer-adapter: From 74d2b4a0c99845de10fd4a09f017aafd6b5c61a0 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 12:56:33 +0530 Subject: [PATCH 17/43] feat(adapter-service): update documentation for network-adapter-layer and topologies --- charts/adapter-service/CHANGELOG.md | 55 ++++++++++++++----- charts/adapter-service/Chart.yaml | 13 +++-- charts/adapter-service/README.md | 30 +++++----- .../examples/network-adapter.yaml | 3 +- charts/adapter-service/values.yaml | 31 +++++++---- charts/openbao-secrets/CHANGELOG.md | 6 +- quick-start/README.md | 17 +++--- 7 files changed, 98 insertions(+), 57 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index fb3ea441..6d3ef452 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -8,23 +8,50 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ## [0.5.3] - 2026-10-09 ### Added -- `role: network-adapter-layer`: all three tiers in one adapter (four modules: - `/` (consumer), `/network/`, `/provider/`, `/provider/publish`), as in the - quick-start's `single` topology (compose profile). Its config is - `config/network-adapter-config.yaml` (extended schema, an outbound - connection pool and log level info on every module); its routing is generated - at render time from `config/routing-*.yaml`. Tier hops are loopback HTTP - inside the pod. Signs as three identities from - `networkAdapterLayer.keys.{consumer,network,provider}`; `routing.config` fails the - render for this role. -- network-adapter-layer ingress may expose only single-segment `/` and - `/provider/`, as literal paths with `pathType` Prefix or Exact and - no `use-regex` / `rewrite-target` annotation. Example - `examples/network-adapter.yaml`. +- **Two topologies, with single as the default.** `role: network-adapter-layer` + is the Network Adapter Layer: the consumer, network and provider tiers as four + modules of one process (`/` for the consumer, `/network/`, `/provider/`, + `/provider/publish`), one release called `network-adapter`. It is the default; + the existing roles `provider`, `network` and `consumer` (one release per + tier) are the optional multi topology and are unchanged. The README's + "Choosing a topology" has the helmfile snippet with + `networkAdapterTopology: single | multi` (default `single`; a deploy-layer + value, not a chart value). +- `config/network-adapter-config.yaml`: the layer's adapter config, mounted + like any role's file (extended schema, an outbound connection pool and log + level info on every module). Each module is its tier's module, so a change to + a tier file that should apply to both topologies is made in both. Its routing + is generated at render time from `config/routing-*.yaml` + (`routing-network-adapter-{consumer,publish}.yaml`), with tier hosts rewritten + to `http://localhost:`. Tier hops are loopback HTTP inside the + pod. The render fails if `http.port` differs from `service.targetPort`. +- Three identities: each module signs as its own tier from + `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` + (`networkAdapterLayer.extraSubstitutions` for extra placeholders). A literal + private key in the config, including a `config:` override, fails the render. +- Ingress for the layer may expose only single-segment `/` and + `/provider/`, as literal paths with `pathType` Prefix or Exact and no + `use-regex` / `rewrite-target` annotation; `/`, `/network*` and + `/provider/publish` are refused. `routing.config` fails the render for this + role. +- `examples/network-adapter.yaml` and `ci/network-adapter-values.yaml` + (HPA 3-10, PDB, spread, graceful shutdown). - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. +### Changed +- Docs, `values.yaml` and the example headers say single is the default + topology and multi is optional. +- Naming: the layer's release, namespace, Service and app name are + `network-adapter` (telemetry service `oan-network-adapter`); "layer" appears + only in the role, the values key and helper names, where it must differ from + the `network` role. The `-adapter` default for app name and telemetry + service name is special-cased for it. The layer shares the name + `network-adapter` with the multi network tier, so they cannot be installed + under that name together: switch an environment in place, or trial the layer + in its own namespace. + ### Fixed - Liveness and readiness probes now render. Values used `probes.liveness` / `probes.readiness`, which `common.probes` never read, so no adapter pod had a @@ -34,7 +61,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h - **Breaking:** the `probes.liveness` and `probes.readiness` values keys, renamed to `livenessProbe` / `readinessProbe` (and a new `startupProbe`, off). The timings are unchanged. An environment that overrides `probes.*` must rename - those keys. + those keys; the old keys never took effect. ### Unchanged - Roles `provider`, `network`, `consumer`: same config, routing and manifests diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index 0782d1df..b8e2f239 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -1,12 +1,13 @@ apiVersion: v2 name: adapter-service description: >- - One chart for the OAN Beckn adapters: provider, network and consumer (one - release per tier, the multi topology), or network-adapter-layer: the Network - Adapter Layer runs the consumer, network and provider tiers as four modules - of one process (the single topology; tier-to-tier calls are loopback HTTP and - each module keeps its own tier identity). Same image and config format; role - selects the handler role, the step list and the routing target. Stateless: an adapter's + One chart for the OAN Beckn adapters, in two topologies. single (the + default): the Network Adapter Layer (role network-adapter-layer) runs the + consumer, network and provider tiers as four modules of one process; + tier-to-tier calls are loopback HTTP and each module keeps its own tier + identity. multi (optional): provider, network and consumer, one release per + tier. Same image and config format; role selects the handler role, the step + list and the routing target. Stateless: an adapter's identity is a keypair held in a Secret, and everything else it reads from the registry. type: application diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index eab17ea0..cb3cbfe5 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -1,9 +1,10 @@ # adapter-service -One chart for the Beckn adapters, in two modes. Multi-adapter: three roles, -`provider`, `network` and `consumer`, one release per tier. The Network Adapter -Layer: `network-adapter-layer`, all three tiers in one adapter. All roles are the same image -and the same config format, so `role` selects the handler role, the step list +One chart for the Beckn adapters, in two topologies. **single (the default):** +the Network Adapter Layer, role `network-adapter-layer`, all three tiers in one +adapter. **multi (optional):** three roles, `provider`, `network` and +`consumer`, one release per tier. All roles are the same image and the same +config format, so `role` selects the handler role, the step list and where requests are routed. ``` @@ -131,12 +132,12 @@ the **public** half of the keypair. Missing, or carrying a different key, and every peer rejects what this adapter signs while the pod stays perfectly healthy. -## Choosing a mode: multi-adapter or Network Adapter Layer +## Choosing a topology: single (default) or multi (optional) -| Mode | Releases (release = namespace) | Callers use | +| Topology | Releases (release = namespace) | Callers use | |---|---|---| -| multi | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | -| Network Adapter Layer | `network-adapter` (examples/network-adapter.yaml) | `network-adapter:9200/`, `network-adapter:9200/provider/publish` | +| **single (default)**: Network Adapter Layer | `network-adapter` (examples/network-adapter.yaml) | `network-adapter:9200/`, `network-adapter:9200/provider/publish` | +| multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | `role: network-adapter-layer` runs all three tiers in one adapter, as four modules of one process. Its config is `config/network-adapter-config.yaml` @@ -151,27 +152,28 @@ needed (it replaces `network-adapter-config.yaml` whole). Pick the topology per environment with ONE value where releases are chosen, `networkAdapterTopology: single | multi` (a helmfile environment value; no chart -reads it). `single` installs one release, `network-adapter` (the Network Adapter +reads it; **default `single`**). `single` installs one release, `network-adapter` (the Network Adapter Layer); `multi` installs the three tier releases. For a helmfile (adapt to infra-automation's layout): ```yaml -# environments: values carry networkAdapterTopology: multi # or: single -{{- if not (has .Values.networkAdapterTopology (list "single" "multi")) }} -{{- fail (printf "networkAdapterTopology must be single or multi, not %q" .Values.networkAdapterTopology) }} +# environments: values may carry networkAdapterTopology: multi # default: single +{{- $topology := .Values.networkAdapterTopology | default "single" }} +{{- if not (has $topology (list "single" "multi")) }} +{{- fail (printf "networkAdapterTopology must be single or multi, not %q" $topology) }} {{- end }} releases: {{- range $tier := list "consumer" "network" "provider" }} - name: {{ $tier }}-adapter namespace: {{ $tier }}-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ eq $.Values.networkAdapterTopology "multi" }} + installed: {{ eq $topology "multi" }} values: [values/{{ $tier }}-adapter.yaml] {{- end }} - name: network-adapter namespace: network-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ eq .Values.networkAdapterTopology "single" }} + installed: {{ eq $topology "single" }} values: [values/network-adapter.yaml] ``` diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index c5e25731..c716dc09 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -1,5 +1,6 @@ # =========================================================================== -# network-adapter-layer role -- the Network Adapter. +# network-adapter-layer role -- the Network Adapter Layer: the DEFAULT topology +# (single). The optional alternative is multi: examples/{consumer,network,provider}.yaml. # # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index f0a94627..855f68d2 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -1,24 +1,31 @@ # =========================================================================== # adapter-service # -# One chart for the Beckn adapters: provider, network and consumer (one release -# per tier), or network-adapter-layer (all three tiers in one adapter). Same image and -# config format; `role` below decides the handler role, the step list and -# where requests are routed. +# One chart for the Beckn adapters, in two topologies. Same image and config +# format; `role` below decides the handler role, the step list and where +# requests are routed. # -# INSTALL ONCE PER ROLE, and keep the RELEASE name per-role: +# single (the DEFAULT): ONE release, the Network Adapter Layer, serving all +# three tiers as modules of one process (role network-adapter-layer): # -# helm install provider-adapter . -f examples/provider.yaml -# helm install network-adapter . -f examples/network.yaml -# helm install consumer-adapter . -f examples/consumer.yaml +# helm install network-adapter . -f examples/network-adapter.yaml # -# OR, all three tiers in one adapter (one release instead of the three above): +# multi (OPTIONAL): one release per tier. INSTALL ONCE PER ROLE, and keep the +# RELEASE name per-role: # -# helm install network-adapter . -f examples/network-adapter.yaml +# helm install provider-adapter . -f examples/provider.yaml +# helm install network-adapter . -f examples/network.yaml +# helm install consumer-adapter . -f examples/consumer.yaml +# +# Run one topology per environment. The Network Adapter Layer is released as +# `network-adapter`, the same name as the multi network tier, so they cannot be +# installed under that name together. # # The release name is what the Service is called, which is the DNS name the -# other adapters address. Rename a release and you must update whatever routes -# to it -- consumer routes to the network adapter, provider is routed to by network. +# other adapters address. In multi, rename a release and you must update +# whatever routes to it -- consumer routes to the network adapter, provider is +# routed to by network. The single topology has no such addresses: its tiers +# call each other over loopback inside the pod. # # Two things every role needs that this chart cannot give it: # diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index bd26ff79..a8c098d9 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -11,9 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `network-adapter` as a target namespace of `consumer-adapter-keys`, `provider-adapter-keys` and `provider-upstream` (`network-adapter-keys` already reflects there), for `adapter-service` role `network-adapter-layer`: - the Network Adapter Layer runs all three tiers in one release named - `network-adapter`, so it reads all three identities. Inert where that - namespace does not exist. + the Network Adapter Layer (the default topology) runs all three tiers in + one release named `network-adapter`, so it reads all three identities. Inert + where that namespace does not exist. ## [0.1.0] - 2026-09-27 diff --git a/quick-start/README.md b/quick-start/README.md index 67ee3b3c..53b41585 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -4,9 +4,11 @@ The whole OAN stack in Docker Compose: a registry, a discovery service, the adapter tiers, and two mock upstreams standing in for real provider APIs. The three adapter tiers (consumer, network, provider) run as **one** container -by default, the Network Adapter Layer (container `network-adapter`): the tiers -as modules of one process on port 9200. `make up-core-multi` runs them as three containers instead. Same -compose file, same keys, same registry → [Two topologies](#two-topologies). +by default: the Network Adapter Layer (container `network-adapter`), the tiers +as modules of one process on port 9200. **Single is the default topology; +multi is optional:** `make up-core-multi` runs the tiers as three containers +instead. Same compose file, same keys, same registry → +[Two topologies](#two-topologies). Nothing is built here — the images are pulled. Locally that is about ten minutes, most of it waiting for Keycloak. Discovery is the one exception, and @@ -93,9 +95,9 @@ Locally, nothing in it needs changing. It publishes these on `localhost`: ``` If one is already taken, change it here — that is the only edit a local run -needs. The single adapter's host port is `NETWORK_ADAPTER_LAYER_PORT`; -in multi mode 9200 is `PROVIDER_ADAPTER_PORT`. Adapters reach each other by Compose service name, so it only changes -what you type into Postman. +needs. The single adapter's host port is `NETWORK_ADAPTER_LAYER_PORT`; in +multi mode 9200 is `PROVIDER_ADAPTER_PORT`. Adapters reach each other by +Compose service name, so it only changes what you type into Postman. ## Step 4 — Start it @@ -233,8 +235,9 @@ collection. → Appendix N. The three adapter tiers can run two ways. Same `docker-compose.yml`, same image, same keys, registry and data; a compose profile picks the adapters. +**single is the default; multi is optional.** -| | Network Adapter Layer (default) | Multi-adapter | +| | single (default): Network Adapter Layer | multi (optional) | |---|---|---| | What runs | one container, `network-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | | Compose profile | `single` | `multi` | From ee5b6c35436529af3199e9c6126d7ac6bcd14da6 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 13:43:19 +0530 Subject: [PATCH 18/43] feat(adapter-service): update documentation and configurations for network-adapter-layer --- charts/adapter-service/CHANGELOG.md | 17 +++++------ charts/adapter-service/README.md | 28 +++++++++++-------- .../examples/network-adapter.yaml | 7 +++-- charts/adapter-service/values.yaml | 9 ++++-- charts/openbao-secrets/CHANGELOG.md | 14 ++++++---- charts/openbao-secrets/values.yaml | 22 ++++++++++----- quick-start/.env.example | 6 ++-- quick-start/README.md | 16 ++++++----- quick-start/bin/setup.py | 2 +- quick-start/bin/stack.sh | 16 +++++++---- .../config/adapters/network-adapter.yaml.tmpl | 2 +- quick-start/docker-compose.yml | 10 +++---- 12 files changed, 87 insertions(+), 62 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 6d3ef452..213e7867 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -43,14 +43,15 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ### Changed - Docs, `values.yaml` and the example headers say single is the default topology and multi is optional. -- Naming: the layer's release, namespace, Service and app name are - `network-adapter` (telemetry service `oan-network-adapter`); "layer" appears - only in the role, the values key and helper names, where it must differ from - the `network` role. The `-adapter` default for app name and telemetry - service name is special-cased for it. The layer shares the name - `network-adapter` with the multi network tier, so they cannot be installed - under that name together: switch an environment in place, or trial the layer - in its own namespace. +- Naming: the layer's release, Service and app name are `network-adapter` + (telemetry service `oan-network-adapter`); its namespace is + `network-adapter-layer`. "layer" appears only in the namespace, the role, + the values key and helper names, where it must differ from the multi + `network` tier. The `-adapter` default for app name and telemetry + service name is special-cased for it. The separate namespace keeps the + consumer and provider keys, which openbao-secrets mirrors to the layer, out + of the multi network tier's namespace, and lets both topologies run side by + side during a switch-over. ### Fixed - Liveness and readiness probes now render. Values used `probes.liveness` / diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index cb3cbfe5..e80e363b 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -152,8 +152,9 @@ needed (it replaces `network-adapter-config.yaml` whole). Pick the topology per environment with ONE value where releases are chosen, `networkAdapterTopology: single | multi` (a helmfile environment value; no chart -reads it; **default `single`**). `single` installs one release, `network-adapter` (the Network Adapter -Layer); `multi` installs the three tier releases. For a helmfile (adapt to +reads it; **default `single`**). `single` installs one release, `network-adapter` +in namespace `network-adapter-layer` (the Network Adapter Layer); `multi` +installs the three tier releases. For a helmfile (adapt to infra-automation's layout): ```yaml @@ -171,7 +172,7 @@ releases: values: [values/{{ $tier }}-adapter.yaml] {{- end }} - name: network-adapter - namespace: network-adapter + namespace: network-adapter-layer chart: ../helmcharts/charts/adapter-service installed: {{ eq $topology "single" }} values: [values/network-adapter.yaml] @@ -180,15 +181,18 @@ releases: With Argo CD, gate the same releases on the same value (for example an ApplicationSet list generator, or `{{ if }}` in an app-of-apps chart). -**The two topologies share the name `network-adapter`.** The Network Adapter -Layer is the network adapter as a whole, so its release, namespace and Service -are `network-adapter`, which is also the multi network tier's. They therefore -cannot be installed side by side under that name (the Service port differs too: -9200 for the layer, 9201 for the multi network tier). Switch an environment in -place: for a trial, install the layer in another namespace -(`-n network-adapter-trial`); to switch for real, remove the multi -`network-adapter` release, install the layer, repoint the app and the -catalogue publisher to the paths above, then remove the other two multi +**Namespace `network-adapter-layer`.** The Network Adapter Layer is the network +adapter as a whole, so its release, Service and app name are `network-adapter`, +the same as the multi network tier's. Its **namespace** is its own, +`network-adapter-layer`, for two reasons: openbao-secrets mirrors all three +tier identities into it, which must not happen in the multi network tier's +namespace `network-adapter`; and the two topologies can then run side by side +during a switch-over. In-cluster address of the layer: +`network-adapter.network-adapter-layer.svc.cluster.local:9200`. + +To switch an environment: install the layer (namespace `network-adapter-layer`) +next to the multi releases, repoint the app (`…:9200/`) and the +catalogue publisher (`…:9200/provider/publish`), then remove the three multi releases. Both topologies sign with the same identities and the same registry rows, so nothing in the registry changes. Reverse the steps to switch back. diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index c716dc09..49c11fbe 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -14,7 +14,7 @@ # (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the # other in an environment, not both. # -# helm install network-adapter . -n network-adapter --create-namespace \ +# helm install network-adapter . -n network-adapter-layer --create-namespace \ # -f examples/network-adapter.yaml # # The config is config/network-adapter-config.yaml; the routing is @@ -23,8 +23,9 @@ # =========================================================================== role: network-adapter-layer -# The Service name and DNS name callers use: -# network-adapter.network-adapter.svc.cluster.local:9200 +# The Service name and DNS name callers use. The namespace is its own, +# network-adapter-layer (not the multi network tier's network-adapter): +# network-adapter.network-adapter-layer.svc.cluster.local:9200 fullnameOverride: network-adapter image: diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 855f68d2..e0d62eab 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -8,7 +8,8 @@ # single (the DEFAULT): ONE release, the Network Adapter Layer, serving all # three tiers as modules of one process (role network-adapter-layer): # -# helm install network-adapter . -f examples/network-adapter.yaml +# helm install network-adapter . -n network-adapter-layer --create-namespace \ +# -f examples/network-adapter.yaml # # multi (OPTIONAL): one release per tier. INSTALL ONCE PER ROLE, and keep the # RELEASE name per-role: @@ -18,8 +19,10 @@ # helm install consumer-adapter . -f examples/consumer.yaml # # Run one topology per environment. The Network Adapter Layer is released as -# `network-adapter`, the same name as the multi network tier, so they cannot be -# installed under that name together. +# `network-adapter` but in its own namespace, `network-adapter-layer`, so the +# tier identities openbao-secrets mirrors into it never land in the multi +# network tier's namespace, and the two topologies can coexist during a +# switch-over. # # The release name is what the Service is called, which is the DNS name the # other adapters address. In multi, rename a release and you must update diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index a8c098d9..2a89a8ff 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -8,12 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.1] - 2026-10-09 ### Added -- `network-adapter` as a target namespace of `consumer-adapter-keys`, - `provider-adapter-keys` and `provider-upstream` (`network-adapter-keys` - already reflects there), for `adapter-service` role `network-adapter-layer`: - the Network Adapter Layer (the default topology) runs all three tiers in - one release named `network-adapter`, so it reads all three identities. Inert - where that namespace does not exist. +- `network-adapter-layer` as a target namespace of `consumer-adapter-keys`, + `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for + `adapter-service` role `network-adapter-layer`: the Network Adapter Layer + (the default topology) runs all three tiers, so it reads all three + identities. It has a namespace of its own rather than the multi network + tier's `network-adapter`, so multi environments do not get the consumer and + provider keys or the upstream credentials copied next to the network tier. + Inert where that namespace does not exist. ## [0.1.0] - 2026-09-27 diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index 3559c8b6..f55e05df 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -57,16 +57,24 @@ secrets: # -- adapter identities. keyId inside these is filled in by # resolve-key-ids.sh AFTER the registry has been seeded, so these entries # change once per environment after the first bring-up. - # network-adapter (role network-adapter-layer) runs all three tiers, so it reads all three identities. Reflector mirrors only into namespaces that exist, so this is inert where that release is not installed. + # + # network-adapter-layer is the namespace of the Network Adapter Layer (the + # single topology, role network-adapter-layer): it runs all three tiers, so + # it reads all three identities. It is a namespace of its own, NOT the multi + # network tier's `network-adapter`, so a multi environment never gets the + # consumer and provider keys copied next to the network tier. Reflector + # mirrors only into namespaces that exist, so this is inert where the layer + # is not installed. - name: consumer-adapter-keys - namespaces: [consumer-adapter, network-adapter] + namespaces: [consumer-adapter, network-adapter-layer] - name: network-adapter-keys - namespaces: [network-adapter] + namespaces: [network-adapter, network-adapter-layer] - name: provider-adapter-keys - namespaces: [provider-adapter, network-adapter] + namespaces: [provider-adapter, network-adapter-layer] # The provider adapter's upstream credentials, named as environment variables - # by its config rather than substituted into it. network-adapter runs - # the provider tier too, so it reads them as well. + # by its config rather than substituted into it. The Network Adapter Layer + # (namespace network-adapter-layer) runs the provider tier too, so it reads + # them as well. - name: provider-upstream - namespaces: [provider-adapter, network-adapter] + namespaces: [provider-adapter, network-adapter-layer] diff --git a/quick-start/.env.example b/quick-start/.env.example index a4d7dda2..999172a8 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -215,7 +215,7 @@ POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in POCRA_PATH=/search POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/0.0.1/quick-start/config/mappings/pocra/agriculture-facility.select.yaml -# Which adapter topology a bare `docker compose up -d` starts: "single" (one -# adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes +# Which adapter topology a bare `docker compose up -d` starts: "network" (the +# single topology: one adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes # --profile explicitly and ignores this; it is for plain compose commands. -COMPOSE_PROFILES=single +COMPOSE_PROFILES=network diff --git a/quick-start/README.md b/quick-start/README.md index 53b41585..551411c5 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -128,9 +128,11 @@ make smoke # publish, discover, select through whatever is running > Steps 5–7 describe the three tiers. In the default single topology they are > modules of one container, so read `network-adapter` wherever a -> command names `provider-adapter`, `network-adapter` or `consumer-adapter` -> (e.g. `docker compose logs network-adapter`). The `.env` keys -> are the same in both. +> command names `provider-adapter`, `network-adapter` or `consumer-adapter`, +> and use `docker logs `: the container is `network-adapter` in both +> topologies, but the compose service is `network-adapter-layer` in single and +> `network-adapter` in multi (so `make logs SVC=network-adapter-layer` in +> single). The `.env` keys are the same in both. Answers `select`, and the only layer that calls an upstream. Serves all four capabilities from one adapter. @@ -165,7 +167,7 @@ re-signs as itself. service behind it — a `discover` naming a different network finds nothing. ```sh -docker compose logs network-adapter | grep 'Server listening' +docker logs network-adapter 2>&1 | grep 'Server listening' # either topology ``` ## Step 7 — Consumer layer @@ -240,7 +242,7 @@ image, same keys, registry and data; a compose profile picks the adapters. | | single (default): Network Adapter Layer | multi (optional) | |---|---|---| | What runs | one container, `network-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | -| Compose profile | `single` | `multi` | +| Compose profile | `network` | `multi` | | Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | | Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | | App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | @@ -252,9 +254,9 @@ other target: it removes the running topology's adapters first. `make down`, `make ps` and `make logs` cover both. `MODE=multi` also works with `make up`, `make up-core` and `make restart`. -Plain compose, without make: `docker compose --profile single up -d` (or +Plain compose, without make: `docker compose --profile network up -d` (or `--profile multi`). A bare `docker compose up -d` follows `COMPOSE_PROFILES` -in `.env` (`single` in `.env.example`); without it no adapter starts. +in `.env` (`network` in `.env.example`); without it no adapter starts. Signing is off in every quick-start template, in both topologies, so keep the adapter ports on `127.0.0.1` (the compose file does). diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 48b24ec5..cf3607e2 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -640,7 +640,7 @@ def _render_one(name, template, substitutions): # ------------------------------------------------------ Network Adapter Layer # # The Network Adapter Layer runs the consumer, network and provider tiers as -# modules of ONE adapter (docker-compose.yml, profile "single"). Its +# modules of ONE adapter (docker-compose.yml, profile "network"). Its # template, network-adapter.yaml.tmpl, carries all three tiers' # placeholders: each module keeps its own tier identity, so both stacks use the # same keys and registry rows. diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 06d67c33..d77e412d 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -35,7 +35,7 @@ cd "$ROOT" # down` leaves the gateway, hyperdx and the adapters (both topologies are # profiled) running and then reports success. Naming them on teardown is what # makes "down" mean down. -PROFILES=(--profile reverse-proxy --profile observability --profile single --profile multi) +PROFILES=(--profile reverse-proxy --profile observability --profile network --profile multi) # Adapter topology: "single" (default) runs the Network Adapter Layer (compose # service network-adapter-layer, container network-adapter), one process for @@ -69,6 +69,10 @@ adapters_for() { # $1 mode } other_mode() { [ "$1" = single ] && echo multi || echo single; } +# The compose profile for a topology: single runs under profile "network", +# multi under profile "multi". +profile_for() { [ "$1" = single ] && echo network || echo multi; } + # ------------------------------------------------------------------ output # Steps are numbered in the output because the whole point of this script is @@ -189,7 +193,7 @@ up_adapters() { # The other topology also publishes 127.0.0.1:9200, and the single container # shares the multi network tier's name: remove it (stopped is not enough). # shellcheck disable=SC2046 - docker compose --profile "$other" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true + docker compose --profile "$(profile_for "$other")" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true # Containers from before the rename (single-network-layer-adapter, and the # earlier network-layer-adapter) would hold 127.0.0.1:9200. docker rm -f single-network-layer-adapter network-layer-adapter >/dev/null 2>&1 || true @@ -199,7 +203,7 @@ up_adapters() { || die "config/adapters/$f.yaml was not rendered (a directory there? rm -r it, then re-run)" done # shellcheck disable=SC2046 - docker compose --profile "$MODE" up -d --force-recreate $(adapters_for "$MODE") + docker compose --profile "$(profile_for "$MODE")" up -d --force-recreate $(adapters_for "$MODE") # NPM resolves adapter names when nginx loads and caches the addresses, so # a recreated or switched adapter 502s through it until it restarts. if [ "$(docker inspect -f '{{.State.Running}}' nginx-proxy-manager 2>/dev/null)" = true ]; then @@ -361,8 +365,8 @@ restart_app() { step 1 1 "restarting registry, discovery and the $MODE-mode adapters" info "keycloak, both databases and the edge are left alone" # shellcheck disable=SC2046 - docker compose --profile "$MODE" restart sunbird-registry-service discovery-service $(adapters_for "$MODE") - docker compose --profile "$MODE" ps --format 'table {{.Service}}\t{{.Status}}' + docker compose --profile "$(profile_for "$MODE")" restart sunbird-registry-service discovery-service $(adapters_for "$MODE") + docker compose --profile "$(profile_for "$MODE")" ps --format 'table {{.Service}}\t{{.Status}}' } # ----------------------------------------------------------------- misc @@ -397,7 +401,7 @@ smoke() { step 1 2 "mocks (mock-imd, mock-agmarknet), and wait for the adapter" docker compose up -d mock-imd mock-agmarknet >/dev/null for s in $(seq 1 30); do curl -sf "$base_app/health" >/dev/null 2>&1 && break; sleep 1; done - curl -sf "$base_app/health" >/dev/null 2>&1 || die "$base_app/health does not answer after 30s: make logs SVC=${svc%% *}" + curl -sf "$base_app/health" >/dev/null 2>&1 || die "$base_app/health does not answer after 30s: docker logs ${svc%% *}" step 2 2 "publish, discover, select through the $mode topology" ROOT="$ROOT" PUB="$base_pub" APP="$base_app" python3 - <<'PY' import json, os, pathlib, sys, urllib.error, urllib.request, uuid diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 09022c17..8f3e8200 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -1,5 +1,5 @@ # Network Adapter: the consumer, network and provider tiers as -# four modules of ONE adapter process (docker-compose.yml, profile "single"). +# four modules of ONE adapter process (docker-compose.yml, profile "network"). # # / the consumer (bap); the app posts here, unsigned # /network/ verifies and forwards to discovery diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 5d22ce43..a6377709 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -11,7 +11,7 @@ # registry sunbird-registry-postgres, sunbird-registry-keycloak, # sunbird-registry-service # discovery discovery-postgres, discovery-service -# adapters network-adapter-layer (profile single, default) | +# adapters network-adapter-layer (profile network, default) | # provider, network, consumer (profile multi) # observability hyperdx profile: observability # edge nginx-proxy-manager profile: reverse-proxy @@ -27,7 +27,7 @@ # everything EXCEPT the adapters # 3. python3 bin/setup.py keys, the three tier identities, and # every adapter config they mount -# 4. docker compose --profile single up -d the Network Adapter +# 4. docker compose --profile network up -d the Network Adapter # (or --profile multi: provider-, network-, consumer-adapter) # 5. docker compose --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack @@ -537,7 +537,7 @@ services: # ========================================================================== # adapters -- TWO topologies, chosen with a compose profile. Same image. # - # profile "single" (the default; `make up`) + # profile "network" (the single topology, the default; `make up`) # network-adapter-layer (container network-adapter): the consumer, network and provider tiers # as four modules of ONE process, one config, one port (127.0.0.1:9200). # Tier-to-tier hops are loopback HTTP inside the container. @@ -616,7 +616,7 @@ services: # unsigned requests: keep :9200 on 127.0.0.1. network-adapter-layer: <<: *adapter - profiles: ["single"] + profiles: ["network"] container_name: network-adapter depends_on: sunbird-registry-service: @@ -1001,7 +1001,7 @@ services: # HyperDX host 502s until it re-resolves. # # required: false -- only one topology's adapters exist at a time - # (profiles "single" / "multi"); NPM starts with whichever is running. + # (profiles "network" / "multi"); NPM starts with whichever is running. network-adapter-layer: condition: service_started required: false From 2f8bc5fbb15ed9157d3fffa04cdb238475855226 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 14:19:56 +0530 Subject: [PATCH 19/43] fix(quick-start): make smoke waits for the registry; document the shared container name make smoke right after make restart failed select with "connection refused": restart also restarts the registry, and smoke only waited for the adapter's /health. It now also waits for sunbird-registry-service to be healthy. The compose header notes the one container_name that differs from its service (network-adapter-layer -> network-adapter). --- quick-start/bin/stack.sh | 7 +++++++ quick-start/docker-compose.yml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index d77e412d..e991ffb0 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -402,6 +402,13 @@ smoke() { docker compose up -d mock-imd mock-agmarknet >/dev/null for s in $(seq 1 30); do curl -sf "$base_app/health" >/dev/null 2>&1 && break; sleep 1; done curl -sf "$base_app/health" >/dev/null 2>&1 || die "$base_app/health does not answer after 30s: docker logs ${svc%% *}" + # The adapter answers /health before its registry is back (e.g. right after + # `make restart`), and select looks the provider up in the registry. + for s in $(seq 1 90); do + [ "$(docker inspect -f '{{.State.Health.Status}}' sunbird-registry-service 2>/dev/null)" = healthy ] && break; sleep 2 + done + [ "$(docker inspect -f '{{.State.Health.Status}}' sunbird-registry-service 2>/dev/null)" = healthy ] \ + || die "sunbird-registry-service is not healthy after 3 minutes: docker logs sunbird-registry-service" step 2 2 "publish, discover, select through the $mode topology" ROOT="$ROOT" PUB="$base_pub" APP="$base_app" python3 - <<'PY' import json, os, pathlib, sys, urllib.error, urllib.request, uuid diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index a6377709..6edc4350 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -3,6 +3,12 @@ # # Every service name is -, and every container_name matches it, so # what `docker ps` shows is what `docker logs` and the in-stack DNS name take. +# One exception: the Network Adapter Layer is service `network-adapter-layer` +# (profile network) with container `network-adapter`, the same container name +# as the multi network tier's service `network-adapter`. Only one topology runs +# at a time, so `docker ps` / `docker logs network-adapter` mean whichever runs; +# compose commands (`docker compose logs`, `make logs SVC=`) take the service +# name, `network-adapter-layer` in the single topology. # The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among # them authenticates nothing but that registry, and shares its postgres. # From cc03a840386265dd69f78d5008a0b91763e658c0 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 14:31:31 +0530 Subject: [PATCH 20/43] docs(adapter-service, quick-start): make the single/multi topologies clear Install shows the default single topology first (namespace network-adapter-layer) and multi with one namespace per tier; the topology table gives release, namespace and full caller addresses; both READMEs say when to use which; NOTES state the installed topology and the full in-cluster address. --- charts/adapter-service/README.md | 70 +++++++++++++++------- charts/adapter-service/templates/NOTES.txt | 9 ++- quick-start/README.md | 12 +++- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index e80e363b..af69030b 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -39,26 +39,45 @@ reads from the registry at request time. ## Install -Once per role. **Keep the release name per-role** — it is what the Service is -called, which is the DNS name the other adapters address. +Pick **one** topology per environment (→ [Choosing a topology](#choosing-a-topology-single-default-or-multi-optional)). +Install the Secrets first: the render fails without them. ```sh helm dependency build charts/adapter-service +``` + +**single (default): one release, the Network Adapter Layer** -helm upgrade --install provider-adapter charts/adapter-service -n oan \ - -f charts/adapter-service/examples/provider.yaml -helm upgrade --install network-adapter charts/adapter-service -n oan \ - -f charts/adapter-service/examples/network.yaml -helm upgrade --install consumer-adapter charts/adapter-service -n oan \ - -f charts/adapter-service/examples/consumer.yaml +```sh +helm upgrade --install network-adapter charts/adapter-service \ + -n network-adapter-layer --create-namespace \ + -f charts/adapter-service/examples/network-adapter.yaml ``` -Each example sets `fullnameOverride` to `-adapter`. Without it the Service -would be named `-adapter-service`, and nothing routing by name would -resolve. Rename a release and you must update whatever routes to it — `consumer` -points at the network adapter, `network` points at discovery. +It needs `consumer-adapter-keys`, `network-adapter-keys`, +`provider-adapter-keys` and `provider-upstream` in `network-adapter-layer` +(openbao-secrets mirrors them there). -The Secret comes first — the render fails without it. +**multi (optional): one release per tier, each in its own namespace.** **Keep +the release and namespace names**: they make the Service DNS name the other +adapters address (the consumer routes to +`network-adapter.network-adapter.svc` and `provider-adapter.provider-adapter.svc`). + +```sh +helm upgrade --install provider-adapter charts/adapter-service \ + -n provider-adapter --create-namespace -f charts/adapter-service/examples/provider.yaml +helm upgrade --install network-adapter charts/adapter-service \ + -n network-adapter --create-namespace -f charts/adapter-service/examples/network.yaml +helm upgrade --install consumer-adapter charts/adapter-service \ + -n consumer-adapter --create-namespace -f charts/adapter-service/examples/consumer.yaml +``` + +Each example sets `fullnameOverride` to `-adapter` (`network-adapter` for the layer). Without it the Service +would be named `-adapter-service`, and nothing routing by name would +resolve. In multi, rename a release and you must update whatever routes to it — +`consumer` points at the network adapter, `network` points at discovery. In +single nothing routes to another release: the tiers call each other inside the +pod. ## What you must decide @@ -134,19 +153,30 @@ healthy. ## Choosing a topology: single (default) or multi (optional) -| Topology | Releases (release = namespace) | Callers use | -|---|---|---| -| **single (default)**: Network Adapter Layer | `network-adapter` (examples/network-adapter.yaml) | `network-adapter:9200/`, `network-adapter:9200/provider/publish` | -| multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` (examples/consumer.yaml, network.yaml, provider.yaml) | `consumer-adapter:9202/`, `provider-adapter:9200/publish` | +| Topology | Releases → namespaces | Values | App calls | Catalogue publishes to | +|---|---|---|---|---| +| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9200/` | `network-adapter.network-adapter-layer.svc:9200/provider/publish` | +| multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` → one namespace each, same name | `examples/{consumer,network,provider}.yaml` | `consumer-adapter.consumer-adapter.svc:9202/` | `provider-adapter.provider-adapter.svc:9200/publish` | + +**When to use which** + +| Use **single** (default) when | Use **multi** when | +|---|---| +| One team runs all three tiers in one cluster | The tiers are run by different parties or in different trust zones | +| You want fewer hops and fewer objects (one Deployment, Service, HPA, PDB) | A tier needs its own scaling or resource profile (e.g. a provider with heavy upstream calls) | +| | You need to roll out or roll back one tier without the others | + +Both sign as the same three identities from the same registry rows, so the +choice changes nothing in the registry. `role: network-adapter-layer` runs all three tiers in one adapter, as four modules of one process. Its config is `config/network-adapter-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change -to a tier file that should apply to both modes goes in both files). Its routing +to a tier file that should apply to both topologies goes in both files). Its routing is **generated** at render time from `config/routing-*.yaml`, so routes stay in one place. Tier hops are loopback HTTP inside the pod (any image). It signs as the same three identities, from the same three Secrets, so the registry is the -same in both modes. An environment's `config:` override for a multi role is NOT +same in both topologies. An environment's `config:` override for a multi role is NOT seen by the Network Adapter Layer; give its release its own `config:` if needed (it replaces `network-adapter-config.yaml` whole). @@ -200,7 +230,7 @@ rows, so nothing in the registry changes. Reverse the steps to switch back. See `values.yaml` — every field is commented with what it does and what breaks without it. `examples/{provider,network,consumer}.yaml` are working dev deployments for the -multi-adapter mode, and `examples/network-adapter.yaml` for the Network Adapter Layer; +multi topology, and `examples/network-adapter.yaml` for the single topology (the Network Adapter Layer, default); `ci/` holds the files lint renders, including `ci/network-adapter-values.yaml`. ## Related diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index babfbf0b..a793450b 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,8 +1,10 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} {{ if include "adapter-service.isNetworkAdapterLayer" . }} + topology single (the default): the Network Adapter Layer role network-adapter-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) hops loopback HTTP between tiers (http://localhost:{{ .Values.service.targetPort }}/) {{- else }} + topology multi (optional): one release per tier; the default is role network-adapter-layer role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) steps {{ include "adapter-service.steps" . | fromYamlArray | join " -> " }} {{- end }} @@ -12,9 +14,14 @@ routes {{ if include "adapter-service.isNetworkAdapterLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} -This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. +This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}.{{ include "common.namespace" . }}.svc:{{ .Values.service.port }}. +{{- if include "adapter-service.isNetworkAdapterLayer" . }} +The app and the catalogue publisher call it there; its tiers call each other +inside the pod, so no other adapter routes to it. +{{- else }} That name comes from the RELEASE name — whatever routes here must use it, so renaming this release means updating the adapter that points at it. +{{- end }} Check it is serving: diff --git a/quick-start/README.md b/quick-start/README.md index 551411c5..8e91cb00 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -41,9 +41,10 @@ capability's provider lives. A `select` answers in the same HTTP round trip — there is no callback. -That is the request order. **Startup order is the reverse**: the consumer -adapter depends on the other two, so Compose brings them up first. The steps -below follow startup order, so they work top to bottom. +That is the request order. **Startup order is the reverse**: in the multi +topology the consumer adapter depends on the other two, so Compose brings them +up first (in single there is one container). The steps below follow startup +order, so they work top to bottom. ## Which path are you on? @@ -249,6 +250,11 @@ image, same keys, registry and data; a compose profile picks the adapters. | Catalogue publishes to | `:9200/provider/publish` | `:9200/publish` | | Tier-to-tier hops | loopback HTTP inside the container (`http://localhost:9200/...`) | HTTP between containers | +**When to use which.** Single is the everyday choice: one container, fewer +hops, the same requests. Use multi to reproduce a deployment where the tiers +run separately (different teams, separate scaling or rollouts), or to compare +the two. + Only one topology runs at a time (both use 9200). Switching is running the other target: it removes the running topology's adapters first. `make down`, `make ps` and `make logs` cover both. `MODE=multi` also works with `make up`, From b3d5c7e069673b83528663388e56fa3d40e7035a Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 17:11:42 +0530 Subject: [PATCH 21/43] feat(adapter-service): update network-adapter values and README for clarity on single/multi topology --- .../ci/network-adapter-values.yaml | 12 +- quick-start/README.md | 5 + quick-start/bin/setup.py | 159 +++++++----------- 3 files changed, 74 insertions(+), 102 deletions(-) diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index c14581a5..9df5de9c 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -1,8 +1,12 @@ # charts/adapter-service/ci/network-adapter-values.yaml -# role network-adapter-layer: the Network Adapter Layer (consumer, network and -# provider tiers as modules of one process). Renders -# config/network-adapter-config.yaml, the loopback routing and the three -# tier identities. +# CI only (scripts/lint-charts.sh lints and renders every ci/*-values.yaml). +# Not for installs: use examples/network-adapter.yaml. +# +# Renders the single topology, role network-adapter-layer (the Network Adapter +# Layer): config/network-adapter-config.yaml, the generated loopback routing, +# the three tier identities, and the HPA / PDB / spread / graceful-shutdown +# settings, so those templates are linted too. Pinned image tag so the render +# is reproducible. role: network-adapter-layer fullnameOverride: network-adapter diff --git a/quick-start/README.md b/quick-start/README.md index 8e91cb00..77ea1848 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -333,6 +333,11 @@ the VM reachable from the internet.** It runs the Network Adapter Layer adapter; `make up-multi` runs the three adapters instead, and decides what the proxy hosts in Appendix B forward to. +Switching topology on a running VM (`make up` ⇄ `make up-multi`) recreates the +adapters and restarts nginx-proxy-manager so it picks up the new addresses: the +public hosts drop connections for a few seconds. Repoint the proxy hosts +(Appendix B) in the same window. + ## Step V4 — Decide what is exposed Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index cf3607e2..77259270 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -556,8 +556,8 @@ def key_osids(identities): ROLES = ("consumer", "network", "provider") -def _shared_substitutions(): - """Placeholders that are the same in every adapter config.""" +def render(identities): + print("configs:") mausamgram_binding = (f"{env('MAUSAMGRAM_PARTICIPANT_ID')}" f"|{env('MAUSAMGRAM_CAPABILITY')}") agmarknet_binding = (f"{env('AGMARKNET_PARTICIPANT_ID')}" @@ -565,102 +565,65 @@ def _shared_substitutions(): vistaar_binding = (f"{env('VISTAAR_PARTICIPANT_ID')}" f"|{env('VISTAAR_CAPABILITY')}") pocra_binding = f"{env('POCRA_PARTICIPANT_ID')}|{env('POCRA_CAPABILITY')}" - return ( - ("__MAUSAMGRAM_BINDING_KEY__", mausamgram_binding), - ("__AGMARKNET_BINDING_KEY__", agmarknet_binding), - ("__VISTAAR_BINDING_KEY__", vistaar_binding), - ("__VISTAAR_TOKEN_URL__", env("VISTAAR_TOKEN_URL")), - # Agmarknet's own token endpoint. A deployment fact, not a - # credential, so it is rendered directly -- the access name and - # password stay as variable NAMES in the config and reach the - # adapter through its environment. - ("__AGMARKNET_TOKEN_URL__", env("AGMARKNET_TOKEN_URL")), - # Auth is per provider, so the participant id is a YAML KEY in - # the adapter config, not only half of a binding key. - ("__POCRA_BINDING_KEY__", pocra_binding), - ("__MAUSAMGRAM_PARTICIPANT_ID__", env("MAUSAMGRAM_PARTICIPANT_ID")), - ("__AGMARKNET_PARTICIPANT_ID__", env("AGMARKNET_PARTICIPANT_ID")), - ("__VISTAAR_PARTICIPANT_ID__", env("VISTAAR_PARTICIPANT_ID")), - ("__POCRA_PARTICIPANT_ID__", env("POCRA_PARTICIPANT_ID")), - # Telemetry. One switch drives all three signals: with every - # one false the plugin builds no exporter and never dials, so - # a stack running without the observability profile stays - # quiet instead of logging a refused connection on a loop. - ("__OTEL_ENABLED__", env("ADAPTER_OTEL_ENABLED", "true")), - ("__OTLP_ENDPOINT__", env("OTLP_ENDPOINT", "hyperdx:4317")), - ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))) - - -def _role_substitutions(role, identity): - """The six per-role placeholders: id, key id, and the four keys.""" - prefix = role.upper() - return ( - (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), - (f"__{prefix}_KEY_ID__", identity["keyOsid"]), - (f"__{prefix}_SIGNING_PRIVATE__", identity["signingPrivate"]), - (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), - (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), - (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"])) - - -def _fill(name, template, substitutions): - """Fill one template's placeholders; refuse if any survive.""" - for placeholder, value in substitutions: - template = template.replace(placeholder, value) - if "__" in template: - sys.exit(f"setup: {name}.yaml still has unrendered placeholders") - return template - - -def _write(rel, text): - """Write config/adapters/ (mode 600: it holds private keys).""" - out = ADAPTERS / rel - # A bare `docker compose up -d` before this script runs starts the - # adapters too, and Docker creates a DIRECTORY at a bind-mount source - # that does not exist. Writing would then fail with a bare - # IsADirectoryError that says nothing about the cause. - if out.is_dir(): - sys.exit( - f"setup: {out} is a directory, not a file.\n" - f" Docker created it, which means the adapters were started before this\n" - f" script ran. Bring them down, remove the empty directories and retry:\n" - f" docker compose down\n" - f" rmdir {out}\n" - f" make up") - out.write_text(text) - out.chmod(0o600) # holds a private key - print(f" config/adapters/{rel}") - - -def _render_one(name, template, substitutions): - """Fill one template and write config/adapters/.yaml (mode 600).""" - _write(f"{name}.yaml", _fill(name, template, substitutions)) - - -# ------------------------------------------------------ Network Adapter Layer -# -# The Network Adapter Layer runs the consumer, network and provider tiers as -# modules of ONE adapter (docker-compose.yml, profile "network"). Its -# template, network-adapter.yaml.tmpl, carries all three tiers' -# placeholders: each module keeps its own tier identity, so both stacks use the -# same keys and registry rows. -def _render_network_adapter_layer(identities, shared): - substitutions = shared - for role in ROLES: - substitutions = _role_substitutions(role, identities[role]) + substitutions - _render_one("network-adapter", - (ADAPTERS / "network-adapter.yaml.tmpl").read_text(), - substitutions) - - -def render(identities): - print("configs:") - shared = _shared_substitutions() - for role in ROLES: - template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() - _render_one(role, template, - _role_substitutions(role, identities[role]) + shared) - _render_network_adapter_layer(identities, shared) + # The three tier configs, then the Network Adapter Layer's + # (network-adapter.yaml.tmpl): its modules carry all three tiers' + # identities, so it gets every tier's keys. + for name in (*ROLES, "network-adapter"): + template = (ADAPTERS / f"{name}.yaml.tmpl").read_text() + for role in (ROLES if name == "network-adapter" else (name,)): + identity = identities[role] + prefix = role.upper() + for placeholder, value in ( + (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), + (f"__{prefix}_KEY_ID__", identity["keyOsid"]), + (f"__{prefix}_SIGNING_PRIVATE__", identity["signingPrivate"]), + (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), + (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), + (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"])): + template = template.replace(placeholder, value) + for placeholder, value in ( + ("__MAUSAMGRAM_BINDING_KEY__", mausamgram_binding), + ("__AGMARKNET_BINDING_KEY__", agmarknet_binding), + ("__VISTAAR_BINDING_KEY__", vistaar_binding), + ("__VISTAAR_TOKEN_URL__", env("VISTAAR_TOKEN_URL")), + # Agmarknet's own token endpoint. A deployment fact, not a + # credential, so it is rendered directly -- the access name and + # password stay as variable NAMES in the config and reach the + # adapter through its environment. + ("__AGMARKNET_TOKEN_URL__", env("AGMARKNET_TOKEN_URL")), + # Auth is per provider, so the participant id is a YAML KEY in + # the adapter config, not only half of a binding key. + ("__POCRA_BINDING_KEY__", pocra_binding), + ("__MAUSAMGRAM_PARTICIPANT_ID__", env("MAUSAMGRAM_PARTICIPANT_ID")), + ("__AGMARKNET_PARTICIPANT_ID__", env("AGMARKNET_PARTICIPANT_ID")), + ("__VISTAAR_PARTICIPANT_ID__", env("VISTAAR_PARTICIPANT_ID")), + ("__POCRA_PARTICIPANT_ID__", env("POCRA_PARTICIPANT_ID")), + # Telemetry. One switch drives all three signals: with every + # one false the plugin builds no exporter and never dials, so + # a stack running without the observability profile stays + # quiet instead of logging a refused connection on a loop. + ("__OTEL_ENABLED__", env("ADAPTER_OTEL_ENABLED", "true")), + ("__OTLP_ENDPOINT__", env("OTLP_ENDPOINT", "hyperdx:4317")), + ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))): + template = template.replace(placeholder, value) + if "__" in template: + sys.exit(f"setup: {name}.yaml still has unrendered placeholders") + out = ADAPTERS / f"{name}.yaml" + # A bare `docker compose up -d` before this script runs starts the + # adapters too, and Docker creates a DIRECTORY at a bind-mount source + # that does not exist. Writing would then fail with a bare + # IsADirectoryError that says nothing about the cause. + if out.is_dir(): + sys.exit( + f"setup: {out} is a directory, not a file.\n" + f" Docker created it, which means the adapters were started before this\n" + f" script ran. Bring them down, remove the empty directories and retry:\n" + f" docker compose down\n" + f" rmdir config/adapters/*.yaml\n" + f" make up") + out.write_text(template) + out.chmod(0o600) # holds a private key + print(f" config/adapters/{name}.yaml") if __name__ == "__main__": From 125dc97d84ead0fc6ff094bc90da7170ba4e5a99 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Mon, 12 Oct 2026 17:27:36 +0530 Subject: [PATCH 22/43] fix(adapter-service): clarify routing and tier hop descriptions in documentation --- charts/adapter-service/CHANGELOG.md | 3 +-- charts/adapter-service/Chart.yaml | 2 +- charts/adapter-service/README.md | 4 ++-- charts/adapter-service/ci/network-adapter-values.yaml | 2 +- charts/adapter-service/config/network-adapter-config.yaml | 5 ++--- charts/adapter-service/examples/network-adapter.yaml | 8 ++++---- charts/adapter-service/templates/NOTES.txt | 2 +- charts/adapter-service/templates/_network-adapter.tpl | 6 +++--- charts/adapter-service/values.yaml | 2 +- quick-start/README.md | 6 +++--- quick-start/config/adapters/network-adapter.yaml.tmpl | 4 ++-- .../config/adapters/routing-network-adapter-consumer.yaml | 4 ++-- quick-start/docker-compose.yml | 2 +- 13 files changed, 24 insertions(+), 26 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 213e7867..3a4269b3 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -23,8 +23,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h a tier file that should apply to both topologies is made in both. Its routing is generated at render time from `config/routing-*.yaml` (`routing-network-adapter-{consumer,publish}.yaml`), with tier hosts rewritten - to `http://localhost:`. Tier hops are loopback HTTP inside the - pod. The render fails if `http.port` differs from `service.targetPort`. + to `http://localhost:`, so tier hops stay inside the pod. The render fails if `http.port` differs from `service.targetPort`. - Three identities: each module signs as its own tier from `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` (`networkAdapterLayer.extraSubstitutions` for extra placeholders). A literal diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index b8e2f239..e5dd4bde 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -4,7 +4,7 @@ description: >- One chart for the OAN Beckn adapters, in two topologies. single (the default): the Network Adapter Layer (role network-adapter-layer) runs the consumer, network and provider tiers as four modules of one process; - tier-to-tier calls are loopback HTTP and each module keeps its own tier + tier-to-tier calls stay inside the pod and each module keeps its own tier identity. multi (optional): provider, network and consumer, one release per tier. Same image and config format; role selects the handler role, the step list and the routing target. Stateless: an adapter's diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index af69030b..2b329440 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -27,7 +27,7 @@ and where requests are routed. | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | | `network` | `bpp` | `validateSign → addRoute → sign` | discovery | | `consumer` | `bap` | `addRoute → sign` | network | -| `network-adapter-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops are loopback HTTP inside the pod | +| `network-adapter-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops stay inside the pod | `consumer` is the only one that accepts **unsigned** requests: the consumer app is inside the trust boundary, so there is no network signature to check. @@ -174,7 +174,7 @@ modules of one process. Its config is `config/network-adapter-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change to a tier file that should apply to both topologies goes in both files). Its routing is **generated** at render time from `config/routing-*.yaml`, so routes stay in -one place. Tier hops are loopback HTTP inside the pod (any image). It signs as +one place. Tier hops go to the pod's own listener, `localhost:` (any image). It signs as the same three identities, from the same three Secrets, so the registry is the same in both topologies. An environment's `config:` override for a multi role is NOT seen by the Network Adapter Layer; give its release its own `config:` if diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 9df5de9c..ca3e97e4 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -3,7 +3,7 @@ # Not for installs: use examples/network-adapter.yaml. # # Renders the single topology, role network-adapter-layer (the Network Adapter -# Layer): config/network-adapter-config.yaml, the generated loopback routing, +# Layer): config/network-adapter-config.yaml, the generated routing, # the three tier identities, and the HPA / PDB / spread / graceful-shutdown # settings, so those templates are linted too. Pinned image tag so the render # is reproducible. diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index 7137905c..cf527869 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -11,8 +11,7 @@ # apply here must be made in this file too. # # Differences from the three tier files: -# - one listener (9200) and one appName; tier hops are loopback HTTP to this -# same listener (see the routing-network-adapter-*.yaml files, which the chart +# - one listener (9200) and one appName; tier hops call this same listener (see the routing-network-adapter-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; @@ -32,7 +31,7 @@ log: contextKeys: [transaction_id, message_id, subscriber_id, module_id] http: - # Also the port the generated loopback routing uses: keep it equal to the + # Also the port the generated routing uses: keep it equal to the # chart's service.targetPort. port: 9200 timeout: diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 49c11fbe..2555e6de 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -18,8 +18,8 @@ # -f examples/network-adapter.yaml # # The config is config/network-adapter-config.yaml; the routing is -# generated from config/routing-*.yaml. Tier hops are loopback HTTP inside the -# pod, so any adapter image works. +# generated from config/routing-*.yaml. Tier hops stay inside the pod, so any +# adapter image works. # =========================================================================== role: network-adapter-layer @@ -36,8 +36,8 @@ image: registry: url: http://registry:8081/api/v1 -# The same port the quick-start publishes. targetPort is also the loopback port -# the generated routing uses, and must equal http.port in +# The same port the quick-start publishes. targetPort is also the port the +# generated routing uses, and must equal http.port in # config/network-adapter-config.yaml (the render checks). service: port: 9200 diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index a793450b..3391ee2c 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -2,7 +2,7 @@ {{ if include "adapter-service.isNetworkAdapterLayer" . }} topology single (the default): the Network Adapter Layer role network-adapter-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) - hops loopback HTTP between tiers (http://localhost:{{ .Values.service.targetPort }}/) + hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/) {{- else }} topology multi (optional): one release per tier; the default is role network-adapter-layer role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) diff --git a/charts/adapter-service/templates/_network-adapter.tpl b/charts/adapter-service/templates/_network-adapter.tpl index 20b8cd10..97b242f7 100644 --- a/charts/adapter-service/templates/_network-adapter.tpl +++ b/charts/adapter-service/templates/_network-adapter.tpl @@ -40,7 +40,7 @@ Routing for the Network Adapter Layer, as YAML: {: }. routing-network.yaml is the multi-adapter file, unchanged (discover and publish to discovery). The other two are generated from routing-consumer.yaml and routing-provider.yaml by pointing every tier address at THIS adapter's own -listener over loopback HTTP: http://-adapter[....][:port] becomes +listener: http://-adapter[....][:port] becomes http://localhost:/, and any path after it is kept (.../publish becomes /network/publish). A target that is not a tier adapter is kept as it is, and a rule with no target is left alone (the same nil guard as @@ -49,7 +49,7 @@ modules across pods again changes only the hosts. */}} {{- define "adapter-service.networkAdapterLayer.routing" -}} {{- $ctx := . -}} -{{- $loopback := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} +{{- $ownListener := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} {{- /* Whole URL, anchored both ends: only a tier host (optionally with a namespace suffix and port) followed by nothing or a path is rewritten. */ -}} {{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} @@ -63,7 +63,7 @@ modules across pods again changes only the hosts. {{- range $i, $rule := $doc.routingRules -}} {{- if $rule.target -}} {{- $url := $rule.target.url | default "" -}} -{{- $url = regexReplaceAll $tierHost $url $loopback -}} +{{- $url = regexReplaceAll $tierHost $url $ownListener -}} {{- if hasSuffix "/" $url -}} {{- fail (printf "%s: network-adapter-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} {{- end -}} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index e0d62eab..83c98f67 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -28,7 +28,7 @@ # other adapters address. In multi, rename a release and you must update # whatever routes to it -- consumer routes to the network adapter, provider is # routed to by network. The single topology has no such addresses: its tiers -# call each other over loopback inside the pod. +# call each other inside the pod. # # Two things every role needs that this chart cannot give it: # diff --git a/quick-start/README.md b/quick-start/README.md index 77ea1848..85705343 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -33,8 +33,8 @@ where the reason lives. adapter, which files the catalogue in discovery. In the default single topology each box above is a module of the one -`network-adapter`, and the arrows between them are loopback calls -inside that container. +`network-adapter`, and the arrows between them are calls inside that +container. All three adapters read the registry — who signed this request, and where this capability's provider lives. @@ -282,7 +282,7 @@ Each module keeps its own tier identity. The same layout as the Helm chart's | `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | | `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/network/publish` | -The hops between modules are loopback HTTP calls to the same container. In the +The hops between modules are HTTP calls to the same container. In the multi-adapter stack only the hosts differ. 9200 is the container's own listener and must match `http.port` in `network-adapter.yaml.tmpl`. diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 8f3e8200..5324db14 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -12,8 +12,8 @@ # charts/adapter-service/config/network-adapter-config.yaml. # # Differences from the three tier templates: -# - one listener (9200) and one appName; tier hops are loopback HTTP to this -# same listener (routing-network-adapter-*.yaml beside this file); +# - one listener (9200) and one appName; tier hops call this same listener +# (routing-network-adapter-*.yaml beside this file); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; # - log level info. diff --git a/quick-start/config/adapters/routing-network-adapter-consumer.yaml b/quick-start/config/adapters/routing-network-adapter-consumer.yaml index b07d7119..44b659f1 100644 --- a/quick-start/config/adapters/routing-network-adapter-consumer.yaml +++ b/quick-start/config/adapters/routing-network-adapter-consumer.yaml @@ -2,8 +2,8 @@ # (consumer, network and provider as modules of one adapter). # # Both targets are THIS SAME adapter on another path. In the multi-adapter -# stack they are two other containers; here the hop is a loopback call from one -# module to another. It stays a real HTTP hop rather than an in-process call +# stack they are two other containers; here the hop is a call from one module +# to another inside the same container. It stays a real HTTP hop rather than an in-process call # because that keeps the two shapes the same config: split these across # containers and only the host in these URLs changes. # diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 6edc4350..07194889 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -546,7 +546,7 @@ services: # profile "network" (the single topology, the default; `make up`) # network-adapter-layer (container network-adapter): the consumer, network and provider tiers # as four modules of ONE process, one config, one port (127.0.0.1:9200). - # Tier-to-tier hops are loopback HTTP inside the container. + # Tier-to-tier hops stay inside the container. # # profile "multi" (`make up-multi`) # provider-adapter, network-adapter, consumer-adapter: one container per From 8637a7307b0360e4f01c59bac3352347037c342a Mon Sep 17 00:00:00 2001 From: Kelvin Date: Tue, 13 Oct 2026 11:25:29 +0530 Subject: [PATCH 23/43] feat(quick-start): add setup instructions and changelog for single/multi adapter modes --- quick-start/CHANGELOG.md | 61 ++++++ quick-start/README.md | 436 ++++++++------------------------------- quick-start/SETUP.md | 318 ++++++++++++++++++++++++++++ 3 files changed, 470 insertions(+), 345 deletions(-) create mode 100644 quick-start/CHANGELOG.md create mode 100644 quick-start/SETUP.md diff --git a/quick-start/CHANGELOG.md b/quick-start/CHANGELOG.md new file mode 100644 index 00000000..1094bfde --- /dev/null +++ b/quick-start/CHANGELOG.md @@ -0,0 +1,61 @@ +# Changelog + +All notable changes to the quick-start stack are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [0.2.0] - 2026-10-13 + +### Added +- **Single network adapter, now the default mode.** One adapter process runs + the consumer, network and provider tiers as modules on one port: + - service `network-adapter-layer`, container `network-adapter`, compose + profile `network`; + - default port `127.0.0.1:9200`, set with `NETWORK_ADAPTER_LAYER_PORT`; + - paths `/` (consumer), `/network/*`, `/provider/*`, + `/provider/publish`; + - config `config/adapters/network-adapter.yaml`, rendered by + `bin/setup.py` from `network-adapter.yaml.tmpl`; it uses the same keys + and registry rows as the three tier adapters. +- **Multi mode stays available as an option.** The three tier adapters + (`provider-adapter` :9200, `network-adapter` :9201, `consumer-adapter` + :9202) are now under compose profile `multi`. +- Make targets: `make up-multi` and `make up-core-multi` start multi mode; + `make smoke` runs publish, discover and select through the running mode. +- `bin/stack.sh` takes `MODE=single|multi` (default `single`). +- `.env.example` gains `NETWORK_ADAPTER_LAYER_PORT=9200` and + `COMPOSE_PROFILES=network`, so a plain `docker compose up -d` starts single + mode. +- API collection for the single adapter: + `api-collection/OpenAgriNet.network-adapter.api-collection.json`. + +- `SETUP.md`: install and run steps, local and VM, for both modes. The + README now covers the modes, how to switch, and the reference appendices. + +### Changed +- `make up` and `make up-core` start single mode instead of the three tier + adapters. +- Only one mode runs at a time, since both use port 9200. Switching mode + removes the other mode's adapter containers. +- `make restart` restarts the adapters of whichever mode is running. + +### Upgrade notes +- Re-run `python3 bin/setup.py` once to render `network-adapter.yaml`. +- To keep the old three-adapter layout, use `make up-multi`, or set + `COMPOSE_PROFILES=multi` for plain compose commands. + +## [0.1.0] - 2026-09-07 + +### Added +- Initial quick-start stack with Docker Compose: + - registry (Sunbird RC) with its Postgres and Keycloak; + - discovery service with its Postgres; + - three adapters, one per tier: provider (port 9200), network (9201) and + consumer (9202); + - two mock upstreams standing in for Mausamgram and Agmarknet; + - optional profiles `reverse-proxy` (nginx-proxy-manager) and + `observability` (hyperdx). +- `bin/setup.py`: generates keys, seeds the registry and renders the adapter + configs. +- `make up` starts the whole stack in order; `make up-core` starts it without + the reverse proxy and observability. diff --git a/quick-start/README.md b/quick-start/README.md index 85705343..0c529d98 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -1,397 +1,142 @@ # OAN quick-start The whole OAN stack in Docker Compose: a registry, a discovery service, the -adapter tiers, and two mock upstreams standing in for real provider APIs. +network adapter, and two mock upstreams standing in for real provider APIs. -The three adapter tiers (consumer, network, provider) run as **one** container -by default: the Network Adapter Layer (container `network-adapter`), the tiers -as modules of one process on port 9200. **Single is the default topology; -multi is optional:** `make up-core-multi` runs the tiers as three containers -instead. Same compose file, same keys, same registry → -[Two topologies](#two-topologies). +- **Install and run it:** [SETUP.md](SETUP.md) +- **What changed, by version:** [CHANGELOG.md](CHANGELOG.md) +- **TLS certificates for the VM edge:** [CERTIFICATES.md](CERTIFICATES.md) -Nothing is built here — the images are pulled. Locally that is about ten -minutes, most of it waiting for Keycloak. Discovery is the one exception, and -only if you ask for it: see [Appendix H](#appendix-h--telemetry), which is also -where the reason lives. - -## The stack in one picture - -``` - consumer - │ - ▼ - consumer adapter the only one a consumer calls - │ - ├── discover ──► network adapter ──► discovery service - │ - └── select ──► provider adapter ──► mock-imd - └─► mock-agmarknet -``` - -`publish` runs the other way: the provider adapter sends it to the network -adapter, which files the catalogue in discovery. - -In the default single topology each box above is a module of the one -`network-adapter`, and the arrows between them are calls inside that -container. - -All three adapters read the registry — who signed this request, and where this -capability's provider lives. - -A `select` answers in the same HTTP round trip — there is no callback. - -That is the request order. **Startup order is the reverse**: in the multi -topology the consumer adapter depends on the other two, so Compose brings them -up first (in single there is one container). The steps below follow startup -order, so they work top to bottom. - -## Which path are you on? - -| | Local | VM | -|----------------------|--------------------------|-----------------------------| -| Reached over | `localhost` | a public hostname, TLS | -| Public edge (Nginx) | not started | started, on 80 and 443 | -| Observability | not started | optional, wants 2–4 GB | -| Credentials | shipped defaults are ok | **must all be changed** | -| Command | `make up-core` | `make up` | -| Three adapters instead | `make up-core-multi` | `make up-multi` | - -**Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. - ---- - -# Part 1 — Run it locally - -## Step 1 — Prerequisites - -- `git` -- Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` -- `python3` with `cryptography` — `pip install cryptography` -- `curl` - -## Step 2 — Get the repo - -```sh -git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git -cd helmcharts/quick-start -``` - -**Every command below runs from `quick-start/`** — it is where the compose file -and the Makefile live. - -## Step 3 — Configure +## Quick start ```sh cp .env.example .env +make up-core # single mode (default): registry, discovery, setup, the network adapter on 9200 +make smoke # publish, discover, select through it; all three should be 200 ``` -Locally, nothing in it needs changing. It publishes these on `localhost`: - -``` -8081 registry 9200 Network Adapter Layer (default) 9100 mock-imd -8080 keycloak or provider adapter (multi) 9101 mock-agmarknet -9990 keycloak admin 9201 network adapter (multi only) -8090 discovery 9202 consumer adapter (multi only) -``` - -If one is already taken, change it here — that is the only edit a local run -needs. The single adapter's host port is `NETWORK_ADAPTER_LAYER_PORT`; in -multi mode 9200 is `PROVIDER_ADAPTER_PORT`. Adapters reach each other by -Compose service name, so it only changes what you type into Postman. - -## Step 4 — Start it - -```sh -make up-core -``` - -Three tiers, in the only order that works: registry and discovery, then -`bin/setup.py`, then the Network Adapter Layer (container `network-adapter`) on 9200. -Allow up to five minutes the first time — Keycloak on a cold volume. - -Three adapters instead: `make up-core-multi`. Either target removes the other -topology's adapters first (both use 9200, and both have a container called -`network-adapter`: the Network Adapter Layer in one, the network tier in the -other), so switching is just running the other one. - -`setup.py` generates a keypair per tier, registers seven participants and four -capability bindings, and renders all four adapter configs (the three tier -configs and the Network Adapter Layer's). Nothing needs creating by hand. → Appendix C for -why the order matters, Appendix D for what it wrote. - -```sh -make ps -make smoke # publish, discover, select through whatever is running -``` - -## Step 5 — Provider layer - -> Steps 5–7 describe the three tiers. In the default single topology they are -> modules of one container, so read `network-adapter` wherever a -> command names `provider-adapter`, `network-adapter` or `consumer-adapter`, -> and use `docker logs `: the container is `network-adapter` in both -> topologies, but the compose service is `network-adapter-layer` in single and -> `network-adapter` in multi (so `make logs SVC=network-adapter-layer` in -> single). The `.env` keys are the same in both. +Prerequisites, the VM path and the checks are in [SETUP.md](SETUP.md). -Answers `select`, and the only layer that calls an upstream. Serves all four -capabilities from one adapter. +## Modes -`.env` keys: `PROVIDER_SUBSCRIBER_ID`, and the four pairs that become binding -keys — `MAUSAMGRAM_PARTICIPANT_ID` + `MAUSAMGRAM_CAPABILITY`, -`AGMARKNET_PARTICIPANT_ID` + `AGMARKNET_CAPABILITY`, `VISTAAR_PARTICIPANT_ID` + -`VISTAAR_CAPABILITY`, `POCRA_PARTICIPANT_ID` + `POCRA_CAPABILITY`. +The adapter has three tiers: **consumer** (the caller's entry), **network** +(fronts discovery) and **provider** (calls the upstream). The quick-start runs +them in one of two modes. Same `docker-compose.yml`, same image, same keys, +same registry and data; a compose profile picks the adapters. -Plus the credentials each upstream needs: `AGMARKNET_ACCESS_NAME` + -`AGMARKNET_PASSWORD`, and `VISTAAR_CLIENT_ID` + `VISTAAR_CLIENT_SECRET`. POCRA -needs none at all. +**Single mode is the default. Multi mode is optional.** -**Only mausamgram is a mock now.** Mandi, knowledge and POCRA are real -external hosts, so each needs its own base URL and path in `.env` — and the -stack needs egress to all three. Mandi and knowledge each also need a token -endpoint, since both issue short-lived tokens rather than taking a static -credential. - -```sh -docker compose logs provider-adapter | grep 'Processor steps initialized' -``` - -All four capability steps should be listed by name. - -## Step 6 — Network layer - -Fronts discovery: verifies the caller, passes `discover` and `publish` on, and -re-signs as itself. - -`.env` keys: `NETWORK_SUBSCRIBER_ID`. `BECKN_NETWORK_ID` belongs to the discovery -service behind it — a `discover` naming a different network finds nothing. - -```sh -docker logs network-adapter 2>&1 | grep 'Server listening' # either topology -``` - -## Step 7 — Consumer layer - -The consumer's edge. Sends `discover` to the network layer and `select` -straight to the provider layer, per `config/adapters/routing-consumer.yaml`. - -`.env` keys: `CONSUMER_SUBSCRIBER_ID`. - -```sh -docker compose logs consumer-adapter | grep 'Server listening' -``` - -`setup.py` renders all four configs from these keys. **Do not edit the -rendered `config/adapters/*.yaml`** — they are regenerated and hold private -keys. Change `.env`, re-run `make up`. - -## Step 8 — Verify end to end - -Quickest, in either topology: - -```sh -make smoke -``` - -It starts the mocks, sends publish, discover and select to whichever adapters -are running, and prints each HTTP status and the tier hops from the adapter -log. All three should be 200. - -Two copies of the full collection, same requests, in `../api-collection/`: - -| Topology | Collection | Environment | +| | single (default) | multi (optional) | |---|---|---| -| single (`make up-core`, default) | `OpenAgriNet.network-adapter.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` | -| multi (`make up-core-multi`) | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | - -Import both files from `../api-collection/` into Postman — -`OpenAgriNet.api-collection.json` and `OpenAgriNet.local-environment.json`. -Select the environment, or every `{{...-host}}` resolves to nothing. Or: - -```sh -newman run ../api-collection/OpenAgriNet.api-collection.json \ - -e ../api-collection/OpenAgriNet.local-environment.json -``` - -**18 requests, 32 assertions**, one folder per capability plus a health folder. -Each capability folder is publish, then discover, then select, in that order — -so run publish first. Every request's `transactionId`, `messageId` and -`timestamp` are filled fresh by the collection's pre-request script; the rest -of each body comes from `capability-examples/`, unedited. - -Green means the registry is seeded, signatures verify both ways, all four -mappings work and discovery is indexing. - -POCRA is the one folder whose empty answer is LOGGED rather than failed: it -returns 200 with no providers both when nothing is nearby and when it is -rate-limiting, and the two are indistinguishable. Knowledge is also the one folder that -can fail on a missing credential rather than on wiring — it authenticates with -OAuth2 client credentials that only the adapter reads, at call time. - -To point it at another deployment, edit the **environment** file, not the -collection. → Appendix N. - ---- - -## Two topologies - -The three adapter tiers can run two ways. Same `docker-compose.yml`, same -image, same keys, registry and data; a compose profile picks the adapters. -**single is the default; multi is optional.** - -| | single (default): Network Adapter Layer | multi (optional) | -|---|---|---| -| What runs | one container, `network-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | +| What runs | one container, `network-adapter`, with all three tiers as modules | three containers: `consumer-adapter`, `network-adapter`, `provider-adapter` | | Compose profile | `network` | `multi` | -| Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | +| Start, local | `make up-core` | `make up-core-multi` | +| Start, VM (with edge and observability) | `make up` | `make up-multi` | | Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | | App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | | Catalogue publishes to | `:9200/provider/publish` | `:9200/publish` | -| Tier-to-tier hops | loopback HTTP inside the container (`http://localhost:9200/...`) | HTTP between containers | +| Tier-to-tier hops | loopback HTTP inside the container | HTTP between containers | +| Port setting in `.env` | `NETWORK_ADAPTER_LAYER_PORT` | `PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT`, `CONSUMER_ADAPTER_PORT` | **When to use which.** Single is the everyday choice: one container, fewer hops, the same requests. Use multi to reproduce a deployment where the tiers run separately (different teams, separate scaling or rollouts), or to compare the two. -Only one topology runs at a time (both use 9200). Switching is running the -other target: it removes the running topology's adapters first. `make down`, -`make ps` and `make logs` cover both. `MODE=multi` also works with `make up`, -`make up-core` and `make restart`. - -Plain compose, without make: `docker compose --profile network up -d` (or -`--profile multi`). A bare `docker compose up -d` follows `COMPOSE_PROFILES` -in `.env` (`network` in `.env.example`); without it no adapter starts. - -Signing is off in every quick-start template, in both topologies, so keep the -adapter ports on `127.0.0.1` (the compose file does). - -**Config.** `config/adapters/network-adapter.yaml.tmpl`, beside the three -tier templates. `bin/setup.py` fills it into `config/adapters/network-adapter.yaml` -like the others. Each module is its tier's module from the tier template (a -change to a tier template that should apply here goes in this file too), plus -extended schema validation on every module, an outbound connection pool and log level info. -Each module keeps its own tier identity. The same layout as the Helm chart's -`config/network-adapter-config.yaml`. - -| Module | Path | From | Routing file | Routes | -|---|---|---|---|---| -| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | -| `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (multi file, reused) | discover, publish to discovery | -| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | -| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/network/publish` | - -The hops between modules are HTTP calls to the same container. In the -multi-adapter stack only the hosts differ. 9200 is the container's own listener -and must match `http.port` in `network-adapter.yaml.tmpl`. - -Check it: `make smoke`. - -# Part 2 — Run it on a VM +Signing is off in every quick-start adapter config, in both modes, so the +adapter ports stay on `127.0.0.1` (the compose file binds them there). -Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. +## How to switch -## Step V1 — Prepare the VM - -Nothing is cloned yet, so fetch the script rather than running it from the repo: +Only one mode runs at a time: both use port 9200, and both have a container +called `network-adapter`. Switching is running the other mode's target; it +removes the running mode's adapters first, and starts the other's. ```sh -curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash +make up-core-multi # single -> multi +make up-core # multi -> single ``` -Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt -repo, then adds you to the `docker` group — **log out and back in** for that to -take effect. Idempotent, and it deliberately does not clone, write `.env` or -start anything. - -**8 GB** runs the stack; **16 GB** for the observability tier. +On a VM use `make up-multi` / `make up`. If nginx-proxy-manager is running it +is restarted so it re-resolves the adapter addresses: the public hosts drop +connections for a few seconds. Repoint the proxy hosts (Appendix B) in the +same window. -Then Part 1 Steps 2 and 3 as written. +Other ways to pick the mode: -## Step V2 — Change every credential +| How | Single | Multi | +|---|---|---| +| make target | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | +| `MODE` variable (works with `up`, `up-core`, `restart`) | `MODE=single make up-core` (the default) | `MODE=multi make up-core` | +| plain compose | `docker compose --profile network up -d` | `docker compose --profile multi up -d` | +| bare `docker compose up -d` | `COMPOSE_PROFILES=network` in `.env` (as in `.env.example`) | `COMPOSE_PROFILES=multi` in `.env` | -`.env.example` ships working defaults, which means they are public: +Without `COMPOSE_PROFILES` a bare `docker compose up -d` starts no adapter. +`bin/stack.sh` (behind every make target) passes `--profile` itself and +ignores `COMPOSE_PROFILES`. -``` -POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_ADMIN_CLIENT_SECRET -KEYCLOAK_DEFAULT_USER_PASSWORD -``` +These follow whichever mode is running, with nothing to set: `make down`, +`make ps`, `make logs`, `make restart`, `make smoke`. -Change all four before the VM is reachable by anyone but you. Adapter keypairs -are the exception — `setup.py` generates those and never writes them to `.env`. - -## Step V3 — Start it +Which mode is running: ```sh -make up +docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' network-adapter +# network-adapter-layer = single, network-adapter = multi ``` -`make up`, not `make up-core`: two more tiers on top of Part 1's three — -nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes -the VM reachable from the internet.** It runs the Network Adapter Layer -adapter; `make up-multi` runs the three adapters instead, and decides what the -proxy hosts in Appendix B forward to. - -Switching topology on a running VM (`make up` ⇄ `make up-multi`) recreates the -adapters and restarts nginx-proxy-manager so it picks up the new addresses: the -public hosts drop connections for a few seconds. Repoint the proxy hosts -(Appendix B) in the same window. - -## Step V4 — Decide what is exposed - -Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written -literally in `docker-compose.yml` rather than taken from a variable. So the -registry, Keycloak and the databases are not publicly reachable — deliberately. +Use the API collection for the running mode (SETUP.md Step 6): +`OpenAgriNet.network-adapter.*` for single, `OpenAgriNet.*` for multi. -Proxy hosts, certificates, and the `/publish` deny every host gets → -**Appendix B**. Read it before pointing DNS at the box. - -## Step V5 — Reach the loopback ports +## The stack in one picture -```sh -ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ - -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm +``` + consumer + │ + ▼ + consumer adapter the only one a consumer calls + │ + ├── discover ──► network adapter ──► discovery service + │ + └── select ──► provider adapter ──► mock-imd + └─► mock-agmarknet ``` -The collection's defaults then work unchanged, since they already point at -loopback. With the default single topology only 9200 is needed for the -adapter (`-L 9200:127.0.0.1:9200`); 9202 is for `make up-multi`. - -## Step V6 — Observability (optional) +`publish` runs the other way: the provider adapter sends it to the network +adapter, which files the catalogue in discovery. -```sh -make observability -``` +In single mode each adapter box above is a module of the one +`network-adapter` container, and the arrows between them are calls inside +that container. In multi mode each box is its own container. -HyperDX on `127.0.0.1:8085`, OTLP on 4317/4318 — and, once you add its proxy -host, on a public hostname behind an NPM Access List, which is how you read it -without a tunnel. Route + credential → **Appendix B**. → Appendix H, which is -honest about how much actually arrives. +All three tiers read the registry — who signed this request, and where this +capability's provider lives. -Every container's stdout lands there too, next to the traces — so this is also -what replaces keeping `docker compose logs -f` open. How that is wired, and the -one gotcha, are in Appendix H. +A `select` answers in the same HTTP round trip — there is no callback. -`docker compose restart nginx-proxy-manager` after this command, the first -time: NPM only resolves `hyperdx` on reload. +That is the request order. **Startup order is the reverse**: registry and +discovery first, then `setup.py`, then the adapters. → Appendix C. ---- +## How single mode is wired -## If something is wrong +`config/adapters/network-adapter.yaml.tmpl`, beside the three tier templates. +`bin/setup.py` renders it into `config/adapters/network-adapter.yaml` like the +others. Each module is its tier's module from the tier template, so a change +to a tier template that should apply in both modes goes in this file too. On +top: extended schema validation on every module, an outbound connection pool +and log level info. Each module keeps its own tier identity. It is the same +layout as the Helm chart's `config/network-adapter-config.yaml`. -- **`unrecognized step: `** — `ADAPTER_IMAGE` predates the config. - Appendix G. -- **404 `NET_ENTITY_NOT_FOUND`** — no provider step claimed the payload; a - binding key disagrees with `.env`. Appendix F. -- **404 naming a binding with no active record** — the step matched but the - registry has no `ProviderSchema` row for it. Appendix F. -- **502 from a `select`** — the upstream answered non-2xx. Appendix F. -- **NPM's default page, or a 502 that worked yesterday** — Appendix B. -- **An adapter config is a directory** — something started before - `setup.py`. Appendix F. +| Module | Path | From | Routing file | Routes | +|---|---|---|---|---| +| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | +| `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (the multi file, reused) | discover, publish to discovery | +| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | +| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/network/publish` | -Full set, with what to run for each → **Appendix F**. +The hops between modules are HTTP calls to the same container; in multi mode +only the hosts differ. 9200 is the container's own listener and must match +`http.port` in `network-adapter.yaml.tmpl`. --- @@ -419,11 +164,12 @@ Reference, read on demand. Nothing here is a step. ## Appendix A — What is here, and what is not Running: **registry** (SunbirdRC + Postgres + Keycloak), **discovery** -(catalogue search + Postgres), **three adapters** (same image, three configs), +(catalogue search + Postgres), **the adapters** (one Network Adapter by +default, or three tier adapters with `make up-multi`; same image), **two mock upstreams** standing in for Mausamgram and Agmarknet. Behind profiles: **nginx-proxy-manager** (`reverse-proxy`, the only container on a routable interface) and **hyperdx** (`observability`, ClickStack — the -heaviest thing here). +heaviest thing here). What changed, and when: [CHANGELOG.md](CHANGELOG.md). Deliberately absent: @@ -468,7 +214,7 @@ Three hosts rather than one with path prefixes, so a rate limit or a block attaches to a hostname instead of a regex in a textarea, and each gets its own certificate. An unknown `Host` gets NPM's default page, not an adapter. -**Single topology (`make up`, the default).** The table above is for +**Single mode (`make up`, the default).** The table above is for `make up-multi`. With the single adapter there is no `consumer-adapter`, `network-adapter` or `provider-adapter` to forward to: point the consumer host at `network-adapter`, port 9200, with the same Advanced paste. The diff --git a/quick-start/SETUP.md b/quick-start/SETUP.md new file mode 100644 index 00000000..d067405b --- /dev/null +++ b/quick-start/SETUP.md @@ -0,0 +1,318 @@ +# Quick-start setup + +Install and run the OAN stack with Docker Compose. What the stack is and the +two adapter modes are in [README.md](README.md); this file is the steps. + +Nothing is built here — the images are pulled. Locally that is about ten +minutes, most of it waiting for Keycloak. + +| | Local | VM | +|----------------------|--------------------------|-----------------------------| +| Reached over | `localhost` | a public hostname, TLS | +| Public edge (Nginx) | not started | started, on 80 and 443 | +| Observability | not started | optional, wants 2–4 GB | +| Credentials | shipped defaults are ok | **must all be changed** | +| Start, single mode (default) | `make up-core` | `make up` | +| Start, multi mode | `make up-core-multi` | `make up-multi` | + +**Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. + +--- + +# Part 1 — Run it locally + +## Step 1 — Prerequisites + +- `git` +- Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` +- `python3` with `cryptography` — `pip install cryptography` +- `curl` + +## Step 2 — Get the repo + +```sh +git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git +cd helmcharts/quick-start +``` + +**Every command below runs from `quick-start/`** — it is where the compose file +and the Makefile live. + +## Step 3 — Configure + +```sh +cp .env.example .env +``` + +Locally, nothing in it needs changing. It publishes these on `localhost`: + +``` +8081 registry 9100 mock-imd +8080 keycloak 9101 mock-agmarknet +9990 keycloak admin +8090 discovery + +single mode (default) 9200 network adapter (all three tiers) +multi mode 9200 provider adapter + 9201 network adapter + 9202 consumer adapter +``` + +If a port is already taken, change it in `.env` — that is the only edit a +local run needs. The single adapter's host port is +`NETWORK_ADAPTER_LAYER_PORT`; in multi mode the three are +`PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT` and `CONSUMER_ADAPTER_PORT`. +Adapters reach each other by Compose service name, so a port change only +changes what you type into Postman. + +## Step 4 — Start it + +Single mode (the default): + +```sh +make up-core +``` + +Or multi mode, three adapters: + +```sh +make up-core-multi +``` + +Either one runs the steps in the only order that works: registry and +discovery, then `bin/setup.py`, then the adapters. Allow up to five minutes +the first time — Keycloak on a cold volume. + +`setup.py` generates a keypair per tier, registers seven participants and four +capability bindings, and renders all four adapter configs (the three tier +configs and the single adapter's). Nothing needs creating by hand. → +README Appendix C for why the order matters, Appendix D for what it wrote. + +Switching mode later is running the other target; see +[README — How to switch](README.md#how-to-switch). + +```sh +make ps +``` + +## Step 5 — The three tiers and their `.env` keys + +Both modes run the same three tiers with the same keys; single mode runs them +as modules of one container. `setup.py` renders every adapter config from +these keys. **Do not edit the rendered `config/adapters/*.yaml`** — they are +regenerated and hold private keys. Change `.env`, re-run `make up-core` (or +`make up-core-multi`). + +**Provider** — answers `select`, the only tier that calls an upstream. Serves +all four capabilities. + +- `PROVIDER_SUBSCRIBER_ID`, and the four pairs that become binding keys: + `MAUSAMGRAM_PARTICIPANT_ID` + `MAUSAMGRAM_CAPABILITY`, + `AGMARKNET_PARTICIPANT_ID` + `AGMARKNET_CAPABILITY`, + `VISTAAR_PARTICIPANT_ID` + `VISTAAR_CAPABILITY`, + `POCRA_PARTICIPANT_ID` + `POCRA_CAPABILITY`. +- Upstream credentials: `AGMARKNET_ACCESS_NAME` + `AGMARKNET_PASSWORD`, and + `VISTAAR_CLIENT_ID` + `VISTAAR_CLIENT_SECRET`. POCRA needs none. + +**Only mausamgram is a mock now.** Mandi, knowledge and POCRA are real +external hosts, so each needs its own base URL and path in `.env` — and the +stack needs egress to all three. Mandi and knowledge each also need a token +endpoint, since both issue short-lived tokens rather than taking a static +credential. + +**Network** — fronts discovery: verifies the caller, passes `discover` and +`publish` on, and re-signs as itself. + +- `NETWORK_SUBSCRIBER_ID`. `BECKN_NETWORK_ID` belongs to the discovery service + behind it — a `discover` naming a different network finds nothing. + +**Consumer** — the caller's entry. Sends `discover` to the network tier and +`select` straight to the provider tier. + +- `CONSUMER_SUBSCRIBER_ID`. + +Check the adapters came up: + +```sh +# single mode: one container +docker logs network-adapter 2>&1 | grep -E 'Processor steps initialized|Server listening' + +# multi mode: one per tier +docker logs provider-adapter 2>&1 | grep 'Processor steps initialized' +docker logs network-adapter 2>&1 | grep 'Server listening' +docker logs consumer-adapter 2>&1 | grep 'Server listening' +``` + +All four capability steps should be listed by name. + +The container is called `network-adapter` in both modes, so `docker logs +network-adapter` works in either. Compose commands take the service name, +which differs: `make logs SVC=network-adapter-layer` in single mode, +`make logs SVC=network-adapter` in multi mode. + +## Step 6 — Verify end to end + +Quickest, in either mode: + +```sh +make smoke +``` + +It starts the mocks, sends publish, discover and select to whichever adapters +are running, and prints each HTTP status and the tier hops from the adapter +log. All three should be 200. + +The full collection, in `../api-collection/`, one copy per mode: + +| Mode | Collection | Environment | +|---|---|---| +| single (default) | `OpenAgriNet.network-adapter.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` | +| multi | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | + +Import the pair for your mode into Postman and select the environment, or +every `{{...-host}}` resolves to nothing. Or run it with newman: + +```sh +# single mode +newman run ../api-collection/OpenAgriNet.network-adapter.api-collection.json \ + -e ../api-collection/OpenAgriNet.network-adapter.local-environment.json + +# multi mode +newman run ../api-collection/OpenAgriNet.api-collection.json \ + -e ../api-collection/OpenAgriNet.local-environment.json +``` + +One folder per capability plus a health folder. Each capability folder is +publish, then discover, then select, in that order — so run publish first. +Every request's `transactionId`, `messageId` and `timestamp` are filled fresh +by the collection's pre-request script; the rest of each body comes from +`capability-examples/`, unedited. + +Green means the registry is seeded, signatures verify both ways, all four +mappings work and discovery is indexing. + +POCRA is the one folder whose empty answer is LOGGED rather than failed: it +returns 200 with no providers both when nothing is nearby and when it is +rate-limiting, and the two are indistinguishable. Knowledge is also the one +folder that can fail on a missing credential rather than on wiring — it +authenticates with OAuth2 client credentials that only the adapter reads, at +call time. + +To point it at another deployment, edit the **environment** file, not the +collection. → README Appendix N. + +--- + +# Part 2 — Run it on a VM + +Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. + +## Step V1 — Prepare the VM + +Nothing is cloned yet, so fetch the script rather than running it from the repo: + +```sh +curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash +``` + +Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt +repo, then adds you to the `docker` group — **log out and back in** for that to +take effect. Idempotent, and it deliberately does not clone, write `.env` or +start anything. + +**8 GB** runs the stack; **16 GB** for the observability tier. + +Then Part 1 Steps 2 and 3 as written. + +## Step V2 — Change every credential + +`.env.example` ships working defaults, which means they are public: + +``` +POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_ADMIN_CLIENT_SECRET +KEYCLOAK_DEFAULT_USER_PASSWORD +``` + +Change all four before the VM is reachable by anyone but you. Adapter keypairs +are the exception — `setup.py` generates those and never writes them to `.env`. + +## Step V3 — Start it + +Single mode (the default): + +```sh +make up +``` + +Or multi mode: `make up-multi`. + +`make up`, not `make up-core`: two more tiers on top of Part 1's — +nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes +the VM reachable from the internet.** The mode decides what the proxy hosts in +README Appendix B forward to. + +Switching mode on a running VM (`make up` ⇄ `make up-multi`) recreates the +adapters and restarts nginx-proxy-manager so it picks up the new addresses: +the public hosts drop connections for a few seconds. Repoint the proxy hosts +(Appendix B) in the same window. + +## Step V4 — Decide what is exposed + +Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written +literally in `docker-compose.yml` rather than taken from a variable. So the +registry, Keycloak and the databases are not publicly reachable — deliberately. +Signing is off in every quick-start adapter config, so the adapter ports must +stay on `127.0.0.1` too. + +Proxy hosts, certificates, and the `/publish` deny every host gets → +**README Appendix B**. Read it before pointing DNS at the box. + +## Step V5 — Reach the ports bound to 127.0.0.1 + +Single mode needs only the adapter's 9200, plus the registry and Keycloak: + +```sh +ssh -L 9200:127.0.0.1:9200 -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm +``` + +Multi mode adds the consumer's 9202 (`-L 9202:127.0.0.1:9202`). The +collections' defaults then work unchanged, since they already point at +`127.0.0.1`. + +## Step V6 — Observability (optional) + +```sh +make observability +``` + +HyperDX on `127.0.0.1:8085`, OTLP on 4317/4318 — and, once you add its proxy +host, on a public hostname behind an NPM Access List, which is how you read it +without a tunnel. Route + credential → **README Appendix B**. → Appendix H, +which is honest about how much actually arrives. + +Every container's stdout lands there too, next to the traces — so this is also +what replaces keeping `docker compose logs -f` open. How that is wired, and the +one gotcha, are in Appendix H. + +`docker compose restart nginx-proxy-manager` after this command, the first +time: NPM only resolves `hyperdx` on reload. + +--- + +## If something is wrong + +- **`unrecognized step: `** — `ADAPTER_IMAGE` predates the config. + README Appendix G. +- **404 `NET_ENTITY_NOT_FOUND`** — no provider step claimed the payload; a + binding key disagrees with `.env`. Appendix F. +- **404 naming a binding with no active record** — the step matched but the + registry has no `ProviderSchema` row for it. Appendix F. +- **502 from a `select`** — the upstream answered non-2xx. Appendix F. +- **NPM's default page, or a 502 that worked yesterday** — Appendix B. +- **An adapter config is a directory** — something started before + `setup.py`. Appendix F. +- **`make smoke` says no adapter is running** — start one mode first: + `make up-core` or `make up-core-multi`. + +Full set, with what to run for each → +[README Appendix F](README.md#appendix-f--when-it-does-not-work). From b0b6aa31e76c5bb4f0195eae7006e5863b2f34f7 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 13:12:04 +0530 Subject: [PATCH 24/43] feat(adapter-service): update default replica count and disable HPA for clarity in configuration --- charts/adapter-service/README.md | 2 +- charts/adapter-service/ci/network-adapter-values.yaml | 6 +++++- charts/adapter-service/examples/network-adapter.yaml | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index 2b329440..ceb9d5d8 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -163,7 +163,7 @@ healthy. | Use **single** (default) when | Use **multi** when | |---|---| | One team runs all three tiers in one cluster | The tiers are run by different parties or in different trust zones | -| You want fewer hops and fewer objects (one Deployment, Service, HPA, PDB) | A tier needs its own scaling or resource profile (e.g. a provider with heavy upstream calls) | +| You want fewer hops and fewer objects (one Deployment, Service, PDB, and an opt-in HPA) | A tier needs its own scaling or resource profile (e.g. a provider with heavy upstream calls) | | | You need to roll out or roll back one tier without the others | Both sign as the same three identities from the same registry rows, so the diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index ca3e97e4..2567de28 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -43,8 +43,12 @@ resources: cpu: 1500m memory: 1Gi goMemLimit: 900MiB +# Fixed replica count by default; the HPA is opt-in. Turning it on needs +# metrics-server in the cluster -- without it the HPA reports +# CPU and never scales. With it off, replicaCount applies. +replicaCount: 3 autoscaling: - enabled: true + enabled: false minReplicas: 3 maxReplicas: 10 targetCPUUtilizationPercentage: 70 diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 2555e6de..3bc3b806 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -72,8 +72,12 @@ resources: memory: 1Gi goMemLimit: 900MiB +# Fixed replica count by default; the HPA is opt-in. Turning it on needs +# metrics-server in the cluster -- without it the HPA reports +# CPU and never scales. With it off, replicaCount applies. +replicaCount: 3 autoscaling: - enabled: true + enabled: false minReplicas: 3 # one per node with the spread constraint below maxReplicas: 10 targetCPUUtilizationPercentage: 70 From 98b6e90fd9e79da6910efdb2fdba37333820ceea Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 17:25:26 +0530 Subject: [PATCH 25/43] fix(adapter-service): flatten public paths and address PR #75 review (#66) Flat public surface for the network-adapter-layer role, matching multi mode: /discover, /publish, / and /health. The network and provider tiers move to /_internal/network/ and /_internal/provider/ (loopback-only), and provider-publish moves from /provider/publish to /publish. - ingress: replace the per-path guard with one rule (single-segment /; /, /publish and /_internal* refused); default dict on both annotation merges; add ci/network-adapter-ingress-values.yaml so CI renders the guard - server_proxy.conf: enable the /publish deny, add /_internal deny - probes: enable startupProbe (180s first-boot budget) so the new liveness probe cannot kill a slow cold start - bump chart to 0.6.0: probes.* keys removed is a breaking change - NOTES/examples/ci: stop reporting and setting registry.url and otel.*, which this role never reads; drop five unused helpers - autoscaling off by default (replicaCount 3), opt-in HPA - quick-start: move smoke out of the stack, fix SETUP clone branch and Compose >= 2.20, trim .gitignore entries for local-only files - api-collection: publish requests go to /publish --- .gitignore | 7 -- ...griNet.network-adapter.api-collection.json | 14 ++-- charts/adapter-service/CHANGELOG.md | 37 ++++++--- charts/adapter-service/Chart.yaml | 2 +- charts/adapter-service/README.md | 23 ++++-- .../ci/network-adapter-ingress-values.yaml | 44 +++++++++++ .../ci/network-adapter-values.yaml | 6 +- .../config/network-adapter-config.yaml | 35 ++++++--- .../examples/network-adapter.yaml | 31 +++----- charts/adapter-service/templates/NOTES.txt | 20 +++-- charts/adapter-service/templates/_helpers.tpl | 53 ------------- .../templates/_network-adapter.tpl | 7 +- charts/adapter-service/templates/ingress.yaml | 38 +++++----- charts/adapter-service/values.yaml | 15 +++- quick-start/.gitignore | 10 --- quick-start/CHANGELOG.md | 10 ++- quick-start/Makefile | 4 +- quick-start/README.md | 22 +++--- quick-start/SETUP.md | 20 +---- quick-start/bin/stack.sh | 76 +------------------ .../config/adapters/network-adapter.yaml.tmpl | 30 ++++---- .../routing-network-adapter-consumer.yaml | 4 +- .../routing-network-adapter-publish.yaml | 4 +- .../npm-custom/server_proxy.conf | 19 +++-- quick-start/docker-compose.yml | 10 +-- 25 files changed, 245 insertions(+), 296 deletions(-) create mode 100644 charts/adapter-service/ci/network-adapter-ingress-values.yaml diff --git a/.gitignore b/.gitignore index ade35b3b..2df82949 100644 --- a/.gitignore +++ b/.gitignore @@ -69,13 +69,6 @@ rendered/ secrets.json *-secrets.json -# Generated local Postman exports for the unified network layer; keep them out of source -# control even though they are tracked alongside the repo's API collection files. - -/dev_docs # Python bytecode __pycache__/ *.pyc -# Local demo and POC Postman collections (the committed ones are -# OpenAgriNet.api-collection.json and OpenAgriNet.network-adapter.api-collection.json) -/api-collection/OpenAgriNet.network-adapter-demo.* diff --git a/api-collection/OpenAgriNet.network-adapter.api-collection.json b/api-collection/OpenAgriNet.network-adapter.api-collection.json index 433ae866..3515f33f 100644 --- a/api-collection/OpenAgriNet.network-adapter.api-collection.json +++ b/api-collection/OpenAgriNet.network-adapter.api-collection.json @@ -3,7 +3,7 @@ "name": "OpenAgriNet - Network Adapter Layer", "_postman_id": "0a4e1c00-0000-4000-8000-0000000snl01", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.network-adapter.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, network-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /network/ network module (internal)\n /provider/ provider module\n /provider/publish catalogue publishing\n\nPublish enters /provider/publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." + "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.network-adapter.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, network-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /publish catalogue publishing\n /_internal/network/ network module (internal)\n /_internal/provider/ provider module (internal)\n\nPublish enters /publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." }, "event": [ { @@ -58,12 +58,11 @@ } ], "url": { - "raw": "{{network-adapter-host}}/provider/publish", + "raw": "{{network-adapter-host}}/publish", "host": [ "{{network-adapter-host}}" ], "path": [ - "provider", "publish" ] }, @@ -215,12 +214,11 @@ } ], "url": { - "raw": "{{network-adapter-host}}/provider/publish", + "raw": "{{network-adapter-host}}/publish", "host": [ "{{network-adapter-host}}" ], "path": [ - "provider", "publish" ] }, @@ -372,12 +370,11 @@ } ], "url": { - "raw": "{{network-adapter-host}}/provider/publish", + "raw": "{{network-adapter-host}}/publish", "host": [ "{{network-adapter-host}}" ], "path": [ - "provider", "publish" ] }, @@ -529,12 +526,11 @@ } ], "url": { - "raw": "{{network-adapter-host}}/provider/publish", + "raw": "{{network-adapter-host}}/publish", "host": [ "{{network-adapter-host}}" ], "path": [ - "provider", "publish" ] }, diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 3a4269b3..db645063 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -5,13 +5,13 @@ All notable changes to this chart are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.5.3] - 2026-10-09 +## [0.6.0] - 2026-10-09 ### Added - **Two topologies, with single as the default.** `role: network-adapter-layer` is the Network Adapter Layer: the consumer, network and provider tiers as four - modules of one process (`/` for the consumer, `/network/`, `/provider/`, - `/provider/publish`), one release called `network-adapter`. It is the default; + modules of one process (`/` for the consumer, `/publish`, + `/_internal/network/`, `/_internal/provider/`), one release called `network-adapter`. It is the default; the existing roles `provider`, `network` and `consumer` (one release per tier) are the optional multi topology and are unchanged. The README's "Choosing a topology" has the helmfile snippet with @@ -28,10 +28,12 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` (`networkAdapterLayer.extraSubstitutions` for extra placeholders). A literal private key in the config, including a `config:` override, fails the render. -- Ingress for the layer may expose only single-segment `/` and - `/provider/`, as literal paths with `pathType` Prefix or Exact and no - `use-regex` / `rewrite-target` annotation; `/`, `/network*` and - `/provider/publish` are refused. `routing.config` fails the render for this +- Flat public surface: `/discover`, `/publish`, `/` and `/health`, the + same URLs as the multi topology. The network and provider tiers are mounted + under `/_internal/`, which only the process itself calls. +- Ingress for the layer may expose only single-segment `/`, as literal + paths with `pathType` Prefix or Exact and no `use-regex` / `rewrite-target` + annotation; `/`, `/publish` and `/_internal*` are refused. `routing.config` fails the render for this role. - `examples/network-adapter.yaml` and `ci/network-adapter-values.yaml` (HPA 3-10, PDB, spread, graceful shutdown). @@ -56,12 +58,27 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h - Liveness and readiness probes now render. Values used `probes.liveness` / `probes.readiness`, which `common.probes` never read, so no adapter pod had a readiness probe. Upgrading rolls every adapter pod once. +- The startup probe bounds first boot at 180s (`periodSeconds: 5` x + `failureThreshold: 36`) and holds off liveness until `/health` answers. + Pods that had no probe before now get a liveness probe, and the adapter + fetches `beckn.yaml` at boot, so without this a slow or proxied egress + would get a cold start killed after about 70s. +- NOTES.txt for role `network-adapter-layer` no longer prints `registry.url` and + `otel.*`: that role mounts `config/network-adapter-config.yaml` as is, so those + values never reached the pod and NOTES reported the opposite of what it ran. + The example and CI values no longer set them. +- The Ingress annotations merge tolerates `ingress.annotations: null`. + `ci/network-adapter-ingress-values.yaml` renders the path guard in CI. ### Removed +- Unused helpers `adapter-service.name`, `.appName`, `.otelServiceName`, + `.registryUrl` and `.otlpEndpoint` (nothing included them). - **Breaking:** the `probes.liveness` and `probes.readiness` values keys, renamed to - `livenessProbe` / `readinessProbe` (and a new `startupProbe`, off). The - timings are unchanged. An environment that overrides `probes.*` must rename - those keys; the old keys never took effect. + `livenessProbe` / `readinessProbe`, plus a new `startupProbe`, on by + default. The liveness and readiness timings are unchanged. An environment + that overrides `probes.*` must rename those keys; the old keys never took + effect and are now silently ignored, which is why this is a minor bump + rather than a patch. ### Unchanged - Roles `provider`, `network`, `consumer`: same config, routing and manifests diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index e5dd4bde..182e8160 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -11,7 +11,7 @@ description: >- identity is a keypair held in a Secret, and everything else it reads from the registry. type: application -version: 0.5.3 +version: 0.6.0 # The adapter binary, not this chart. `/health` reports the build it is # actually running, which is the number to trust over this one. appVersion: "v1.9.0" diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index ceb9d5d8..b0ccec0a 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -17,11 +17,24 @@ and where requests are routed. ``` network-adapter-layer (one adapter, one pod): - / ──lo──▶ /network/ ──▶ discovery - └──────lo────▶ /provider/ ──▶ upstream APIs - /provider/publish ──lo──▶ /network/publish + / ──lo──▶ /_internal/network/ ──▶ discovery + └──────lo────▶ /_internal/provider/ ──▶ upstream APIs + /publish ──lo──▶ /_internal/network/publish ``` +The public surface is flat and the same in both topologies: `/discover`, +`/publish`, `/` and `/health`. The network and provider tiers sit under +`/_internal/`, which only this process calls (over loopback), so the edge +denies the whole prefix with one rule. + +Why flat: OAN is single-sided today -- our app calls out, and our own +discovery and upstreams answer. If OAN later accepts inbound calls from +external Beckn peers, a peer's `select` and our app's `select` are both +`POST /select`, and path alone cannot tell them apart. The distinguisher exists +(peer calls carry an `Authorization` signature, app calls do not), but ONIX +dispatches on path, so that would mean routing on signature presence at the +edge or giving inbound its own hostname. Neither changes these public paths. + | Role | Handler | Steps | Routes to | |---|---|---|---| | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | @@ -155,7 +168,7 @@ healthy. | Topology | Releases → namespaces | Values | App calls | Catalogue publishes to | |---|---|---|---|---| -| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9200/` | `network-adapter.network-adapter-layer.svc:9200/provider/publish` | +| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9200/` | `network-adapter.network-adapter-layer.svc:9200/publish` | | multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` → one namespace each, same name | `examples/{consumer,network,provider}.yaml` | `consumer-adapter.consumer-adapter.svc:9202/` | `provider-adapter.provider-adapter.svc:9200/publish` | **When to use which** @@ -222,7 +235,7 @@ during a switch-over. In-cluster address of the layer: To switch an environment: install the layer (namespace `network-adapter-layer`) next to the multi releases, repoint the app (`…:9200/`) and the -catalogue publisher (`…:9200/provider/publish`), then remove the three multi +catalogue publisher (`…:9200/publish`), then remove the three multi releases. Both topologies sign with the same identities and the same registry rows, so nothing in the registry changes. Reverse the steps to switch back. diff --git a/charts/adapter-service/ci/network-adapter-ingress-values.yaml b/charts/adapter-service/ci/network-adapter-ingress-values.yaml new file mode 100644 index 00000000..61856fee --- /dev/null +++ b/charts/adapter-service/ci/network-adapter-ingress-values.yaml @@ -0,0 +1,44 @@ +# charts/adapter-service/ci/network-adapter-ingress-values.yaml +# CI only (scripts/lint-charts.sh lints and renders every ci/*-values.yaml). +# Not for installs: use examples/network-adapter.yaml. +# +# Renders role network-adapter-layer with the Ingress on, so the path guard in +# templates/ingress.yaml runs in CI: the allowed path shape (one-segment +# /, Exact and Prefix) must render. ingress.annotations +# is null on purpose, to cover the `| default dict` on the annotations merge. +role: network-adapter-layer +fullnameOverride: network-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + +service: + port: 9200 + targetPort: 9200 + +networkAdapterLayer: + keys: + consumer: + existingSecret: consumer-adapter-keys + network: + existingSecret: network-adapter-keys + provider: + existingSecret: provider-adapter-keys + extraSubstitutions: + __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + +ingress: + enabled: true + className: nginx + annotations: null + hosts: + - host: network-adapter.ci.example.com + paths: + - path: /discover + pathType: Exact + - path: /select + pathType: Prefix + - path: /status + pathType: Exact diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 2567de28..54c1ad12 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -15,8 +15,10 @@ image: repository: openagrinet/network-adapter tag: "v1.9.0" -registry: - url: http://registry:8081/api/v1 +# registry.url, otel.*, appName, logLevel, http.timeout.*, routing.* and +# handler.* are not used by this role: the adapter config is mounted from +# config/network-adapter-config.yaml as is (registry URL, OTLP endpoint and +# log level are set there), so they are left out here. Edit that file instead. service: port: 9200 diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index cf527869..a3c1c146 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -1,10 +1,15 @@ # OAN Network Adapter Layer (role network-adapter-layer). The consumer, network # and provider tiers as four modules of ONE adapter process: # -# / the consumer (bap); the app posts here, unsigned -# /network/ verifies and forwards to discovery -# /provider/ answers from the upstream APIs (bpp) -# /provider/publish the catalogue system publishes here +# / the consumer (bap); the app posts here, unsigned +# /publish the catalogue system publishes here +# /_internal/network/ verifies and forwards to discovery +# /_internal/provider/ answers from the upstream APIs (bpp) +# +# The public surface is flat: /discover, /publish, / and /health. The +# tiers are not part of it. /_internal/ is only ever called by this process's +# own modules (http://localhost:9200/_internal/...), so the edge denies the +# whole prefix with one rule. # # Each module is its tier's module from consumer-config.yaml, network-config.yaml # and provider-config.yaml. A change to one of those files that should also @@ -60,7 +65,15 @@ plugins: modules: # ------------------------------------------------------------------ consumer # From consumer-config.yaml. Accepts unsigned requests from the app, signs as - # the consumer, and routes discover to /network and transactions to /provider. + # the consumer, and routes discover to /_internal/network and transactions to + # /_internal/provider. + # + # Mounted at / on purpose: it is the one catch-all mount, and that is + # load-bearing, not accidental. /publish is an exact pattern, so it wins over + # this subtree, and /_internal// are longer subtrees, so they win too. + # Everything else lands here, and the routing config is in effect an action + # allow-list: addRoute runs before sign, so an action with no routing rule is + # an error, never a forwarded request. - name: consumer path: / handler: @@ -131,7 +144,7 @@ modules: # From network-config.yaml. Verifies the caller's signature and forwards # discover and publish to discovery. Internal: never expose it on an Ingress. - name: network - path: /network/ + path: /_internal/network/ handler: type: std role: bpp @@ -198,7 +211,7 @@ modules: # the upstream APIs itself, so it has no router. Upstream credentials come # from the environment (envFromSecrets: provider-upstream). - name: provider - path: /provider/ + path: /_internal/provider/ handler: type: std role: bpp @@ -301,10 +314,12 @@ modules: # ---------------------------------------------------------- provider-publish # From provider-config.yaml, module 2. The catalogue system publishes here; - # signed as the provider and routed to /network/publish. No validateSign: - # never expose it on an Ingress. + # signed as the provider and routed to /_internal/network/publish. No + # validateSign: never expose it on an Ingress. At top-level /publish, the + # same path as the multi provider adapter's publish module, so both + # topologies present the same URL. - name: provider-publish - path: /provider/publish + path: /publish handler: type: std role: bap diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 3bc3b806..f449c5fb 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -5,10 +5,10 @@ # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: # -# / the consumer (bap); the app posts here, unsigned -# /network/ internal: verifies and forwards to discovery -# /provider/ answers from the upstream APIs (bpp) -# /provider/publish the catalogue system publishes here +# / the consumer (bap); the app posts here, unsigned +# /publish the catalogue system publishes here (internal) +# /_internal/network/ internal: verifies and forwards to discovery +# /_internal/provider/ internal: answers from the upstream APIs (bpp) # # The alternative to installing the three multi-adapter releases # (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the @@ -33,8 +33,10 @@ image: repository: openagrinet/network-adapter tag: "latest" -registry: - url: http://registry:8081/api/v1 +# registry.url, otel.*, appName, logLevel, http.timeout.*, routing.* and +# handler.* are not used by this role: the adapter config is mounted from +# config/network-adapter-config.yaml as is (registry URL, OTLP endpoint and +# log level are set there), so they are left out here. Edit that file instead. # The same port the quick-start publishes. targetPort is also the port the # generated routing uses, and must equal http.port in @@ -107,9 +109,8 @@ topologySpreadConstraints: terminationGracePeriodSeconds: 40 preStopSleepSeconds: 10 # needs Kubernetes 1.30+; 0 on older clusters -# Off here. When on, only single-segment / paths (the consumer) and -# /provider/ are accepted: /, /network*, /provider and -# /provider/publish fail the render. +# Off here. When on, only single-segment / paths (the consumer) are +# accepted: /, /publish and /_internal* fail the render. ingress: enabled: false className: "" @@ -126,18 +127,6 @@ ingress: pathType: Exact - path: /status pathType: Exact - - path: /provider/select - pathType: Prefix - - path: /provider/init - pathType: Prefix - - path: /provider/confirm - pathType: Prefix - - path: /provider/status - pathType: Prefix waitFor: enabled: false - -otel: - enabled: false - endpoint: "" diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index 3391ee2c..a0391275 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,7 +1,7 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} {{ if include "adapter-service.isNetworkAdapterLayer" . }} topology single (the default): the Network Adapter Layer - role network-adapter-layer (all three tiers in one adapter: / consumer bap, /network/ bpp, /provider/ bpp, /provider/publish bap) + role network-adapter-layer (all three tiers in one adapter: / consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/) {{- else }} topology multi (optional): one release per tier; the default is role network-adapter-layer @@ -10,9 +10,15 @@ {{- end }} image {{ include "adapter-service.image" . }} service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} +{{- if include "adapter-service.isNetworkAdapterLayer" . }} + registry set in config/network-adapter-config.yaml (registry.url is not used by this role) + routes generated from config/routing-{consumer,network,provider}.yaml + telemetry set in config/network-adapter-config.yaml (otel.* is not used by this role) +{{- else }} registry {{ .Values.registry.url }} - routes {{ if include "adapter-service.isNetworkAdapterLayer" . }}generated from config/routing-{consumer,network,provider}.yaml{{ else }}from config/routing-{{ include "adapter-service.role" . }}.yaml{{ end }} + routes from config/routing-{{ include "adapter-service.role" . }}.yaml telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} +{{- end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}.{{ include "common.namespace" . }}.svc:{{ .Values.service.port }}. {{- if include "adapter-service.isNetworkAdapterLayer" . }} @@ -45,8 +51,8 @@ TWO THINGS THIS CHART CANNOT CHECK FOR YOU Callers address the tiers by path on this one Service: app (discover, select, ...) {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/ - catalogue publisher {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/provider/publish - /network is an internal tier: never route to it from outside. + catalogue publisher {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}/publish + /_internal/ is the network and provider tiers: never route to it from outside. {{- else -}} 1. The Participant row. This adapter signs as {{ .Values.keys.existingSecret.name }}'s subscriberId, and every peer verifies that signature against the public key @@ -61,7 +67,7 @@ Callers address the tiers by path on this one Service: kubectl -n {{ include "common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} {{- end }} -{{- if not .Values.otel.enabled }} +{{- if and (not .Values.otel.enabled) (not (include "adapter-service.isNetworkAdapterLayer" .)) }} Telemetry is off, which is the right setting until a collector exists: with it on and nothing listening the exporter logs a failure every interval. @@ -73,8 +79,8 @@ Ingress is ON. This is the network-adapter-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is inside the trust boundary and has no validateSign step, so an Ingress on it exposes an unauthenticated entry point to the network. Rate-limit this host. -Rule: only single-segment / paths and /provider/ may be exposed (the render refuses other paths), -and never /network or /provider/publish: /network is an internal tier, and /provider/publish +Rule: only single-segment / paths may be exposed (the render refuses other paths), +never /_internal* or /publish: /_internal/ holds the network and provider tiers, and /publish has no validateSign and signs as the provider. {{- else if eq (include "adapter-service.role" .) "consumer" }} This is the consumer role, which accepts UNSIGNED requests — it is inside the trust diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl index e02f0084..041cf86e 100644 --- a/charts/adapter-service/templates/_helpers.tpl +++ b/charts/adapter-service/templates/_helpers.tpl @@ -32,26 +32,6 @@ and then mis-handles every request, at the far end, in a peer's logs. {{- $role -}} {{- end }} -{{/* -The name the adapter calls itself in its config, its logs and its module. -Defaults to -adapter, matching the compose stack's service names. The -Network Adapter Layer (role network-adapter-layer) is the network adapter as -a whole, so it is called network-adapter, not network-adapter-layer-adapter. -*/}} -{{- define "adapter-service.appName" -}} -{{- $role := include "adapter-service.role" . -}} -{{- .Values.appName | default (ternary "network-adapter" (printf "%s-adapter" $role) (eq $role "network-adapter-layer")) -}} -{{- end }} - -{{/* -OTEL service name. Defaults to oan--adapter, matching OTEL_SERVICE_NAME -in the compose stack so traces from either deployment line up. -*/}} -{{- define "adapter-service.otelServiceName" -}} -{{- $role := include "adapter-service.role" . -}} -{{- .Values.otel.serviceName | default (ternary "oan-network-adapter" (printf "oan-%s-adapter" $role) (eq $role "network-adapter-layer")) -}} -{{- end }} - {{/* bap originates a call, bpp receives one. consumer is the caller; network and provider receive. Overridable, because the role is a shorthand for a default @@ -92,11 +72,6 @@ self-describing when you exec into a pod. routing-{{ include "adapter-service.role" . }}.yaml {{- end }} - -{{- define "adapter-service.name" -}} -{{- include "common.name" . -}} -{{- end }} - {{- define "adapter-service.fullname" -}} {{- include "common.fullname" . -}} {{- end }} @@ -134,34 +109,6 @@ why. {{- $name -}} {{- end }} -{{/* -The registry API base. Required for the same reason as the keys: absent, the -adapter builds a plugin pointed at nothing and every signature verification -fails on a lookup rather than on the signature. -*/}} -{{- define "adapter-service.registryUrl" -}} -{{- $url := .Values.registry.url | default "" -}} -{{- if not $url -}} -{{- fail (printf "%s: registry.url is required -- e.g. http://registry:8081/api/v1. The adapter verifies every caller against the key the registry publishes for them, so with no registry it can verify nobody." .Chart.Name) -}} -{{- end -}} -{{- $url -}} -{{- end }} - -{{/* -OTLP endpoint, required only when telemetry is on. An enabled exporter with -no endpoint dials nothing and logs a failure per export interval, which is -the noise the three enable flags exist to avoid. -*/}} -{{- define "adapter-service.otlpEndpoint" -}} -{{- if .Values.otel.enabled -}} -{{- $ep := .Values.otel.endpoint | default "" -}} -{{- if not $ep -}} -{{- fail (printf "%s: otel.endpoint is required when otel.enabled is true -- e.g. otel-collector.observability.svc.cluster.local:4317. An enabled exporter with nowhere to send logs a failure every export interval." .Chart.Name) -}} -{{- end -}} -{{- $ep -}} -{{- end -}} -{{- end }} - {{/* Where the rendered config lands. An emptyDir, because the init container writes it and readOnlyRootFilesystem means there is nowhere else to write. diff --git a/charts/adapter-service/templates/_network-adapter.tpl b/charts/adapter-service/templates/_network-adapter.tpl index 97b242f7..f867de1a 100644 --- a/charts/adapter-service/templates/_network-adapter.tpl +++ b/charts/adapter-service/templates/_network-adapter.tpl @@ -41,15 +41,16 @@ routing-network.yaml is the multi-adapter file, unchanged (discover and publish to discovery). The other two are generated from routing-consumer.yaml and routing-provider.yaml by pointing every tier address at THIS adapter's own listener: http://-adapter[....][:port] becomes -http://localhost:/, and any path after it is kept -(.../publish becomes /network/publish). A target that is not a tier adapter is +http://localhost:/_internal/, and any path after it is kept +(.../publish becomes /_internal/network/publish). Only the network and provider +tiers are ever targets, so only they are mounted under /_internal/. A target that is not a tier adapter is kept as it is, and a rule with no target is left alone (the same nil guard as configmap.yaml). So the two modes are one routing source, and splitting the modules across pods again changes only the hosts. */}} {{- define "adapter-service.networkAdapterLayer.routing" -}} {{- $ctx := . -}} -{{- $ownListener := printf "http://localhost:%v/${1}${4}" .Values.service.targetPort -}} +{{- $ownListener := printf "http://localhost:%v/_internal/${1}${4}" .Values.service.targetPort -}} {{- /* Whole URL, anchored both ends: only a tier host (optionally with a namespace suffix and port) followed by nothing or a path is rewritten. */ -}} {{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index 29f14834..2e5b8032 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,35 +1,31 @@ {{- if .Values.ingress.enabled }} {{- if include "adapter-service.isNetworkAdapterLayer" . }} -{{- /* role network-adapter-layer: only single-segment / paths (the - consumer module is mounted at /) and individual /provider/ paths - may be public. "/" or "/provider" with Prefix would also expose /network - (an internal tier) and /provider/publish, which has no validateSign and - signs as the provider. - The path check holds only for literal prefix matching, so pathType must be - Prefix or Exact (ImplementationSpecific is a plain string prefix on - ingress-nginx: /provider/p would match /provider/publish), the path must - be literal, and regex or rewrite annotations are refused. */}} -{{- $ann := merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict) }} +{{- /* role network-adapter-layer: the public surface is flat -- /discover, + / and /health, all served by the consumer module at /. Two + things must never be public: /_internal/ (the network and provider + tiers, called only by this process) and /publish (no validateSign, + signs as the provider). So a path is accepted only if it is one literal + segment and is neither of those; "/" is refused because a Prefix "/" + covers both. + Prefix and Exact only: ImplementationSpecific is a plain string prefix + on ingress-nginx, and regex or rewrite annotations turn paths into + patterns, so either could reach /_internal or /publish. */}} +{{- $ann := merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict) }} {{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString) "true" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /_internal or /publish." .Chart.Name) }} {{- end }} {{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-adapter-layer role. It turns paths into regexes and can rewrite a request onto /network or /provider/publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-adapter-layer role. It turns paths into regexes and can rewrite a request onto /_internal or /publish." .Chart.Name) }} {{- end }} {{- range .Values.ingress.hosts }} {{- range .paths }} {{- $p := .path | default "" }} {{- $pt := .pathType | default "Prefix" }} {{- if not (has $pt (list "Prefix" "Exact")) }} -{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-adapter-layer role. Use Prefix or Exact: other path types can match /network or /provider/publish." $.Chart.Name $p $pt) }} +{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-adapter-layer role. Use Prefix or Exact." $.Chart.Name $p $pt) }} {{- end }} -{{- if or (not (regexMatch "^/[A-Za-z0-9/_.-]*$" $p)) (contains ".." $p) (contains "//" $p) }} -{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. It must be a literal path (letters, digits, / _ . -), with no regex characters, .. or //." $.Chart.Name $p) }} -{{- end }} -{{- $consumer := and (regexMatch "^/[A-Za-z0-9_.-]+$" $p) (not (has $p (list "/network" "/provider"))) }} -{{- $provider := and (hasPrefix "/provider/" $p) (ne $p "/provider/") (not (hasPrefix "/provider/publish" $p)) }} -{{- if not (or $consumer $provider) }} -{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. Allowed: single-segment / (the consumer, e.g. /discover) and /provider/ (not /provider/publish). /, /provider and /network* would expose internal tiers." $.Chart.Name $p) }} +{{- if or (not (regexMatch "^/[A-Za-z0-9_.-]+$" $p)) (hasPrefix "/_internal" $p) (eq $p "/publish") }} +{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. Allowed: one literal segment, / (e.g. /discover, /select). /, /publish and /_internal* are never public." $.Chart.Name $p) }} {{- end }} {{- end }} {{- end }} @@ -51,7 +47,7 @@ metadata: name: {{ include "adapter-service.fullname" . }} labels: {{- include "adapter-service.labels" . | nindent 4 }} - {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + {{- with (merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict)) }} annotations: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 83c98f67..63a4631b 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -342,8 +342,19 @@ resources: # Both hit /health, which the adapter serves from its own mux and which does # not touch the registry or discovery -- so a readiness failure here means # this process is unwell, not that something downstream is. +# The startup probe holds liveness off until /health first answers. The +# adapter fetches beckn.yaml from raw.githubusercontent.com at boot, so on +# slow or proxied egress first boot can take well past liveness's ~70s budget +# (10s + 3 x 20s). This bounds it at 180s without loosening steady-state +# liveness. startupProbe: - enabled: false + enabled: true + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 36 livenessProbe: enabled: true httpGet: @@ -430,7 +441,7 @@ envFromSecrets: [] # role: network-adapter-layer -- the Network Adapter Layer. # # The consumer, network and provider tiers as four modules of ONE process -# (/ for the consumer, /network/, /provider/, /provider/publish). Its adapter config is +# (/ for the consumer, /publish, /_internal/network/, /_internal/provider/). Its adapter config is # config/network-adapter-config.yaml, mounted verbatim like any role's # file: log level, timeouts, telemetry, extended schema and the outbound # connection pool are set there. The routing is generated at render time from diff --git a/quick-start/.gitignore b/quick-start/.gitignore index 2351f021..26c000c3 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -17,13 +17,6 @@ config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml config/adapters/network-adapter.yaml -bin/demo-network-adapter.sh -bin/bench-network-adapter.sh -poc-flow.py -bin/poc -# Apple-silicon-only override that forces amd64 emulation for images with no -# arm64 build. A local workaround, not part of the stack. -docker-compose.arm64.yml # A local override for the discovery service, if you make one. config/discovery/instance.yaml @@ -41,6 +34,3 @@ config/discovery/instance.yaml # certificates in the npm-data and npm-letsencrypt volumes, so there is # nothing here to ignore -- and nothing here to review either. Back those # volumes up; they are the only copy. - -# bin/bench-network-adapter.sh scratch: generated bodies and results -.bench/ \ No newline at end of file diff --git a/quick-start/CHANGELOG.md b/quick-start/CHANGELOG.md index 1094bfde..211d8c14 100644 --- a/quick-start/CHANGELOG.md +++ b/quick-start/CHANGELOG.md @@ -12,16 +12,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - service `network-adapter-layer`, container `network-adapter`, compose profile `network`; - default port `127.0.0.1:9200`, set with `NETWORK_ADAPTER_LAYER_PORT`; - - paths `/` (consumer), `/network/*`, `/provider/*`, - `/provider/publish`; + - paths `/` (consumer) and `/publish`, the same as multi mode; the + network and provider tiers under `/_internal/`, which only the process + itself calls; + - `server_proxy.conf` now denies `/publish` (it was commented out) and + `/_internal` on every proxy host; - config `config/adapters/network-adapter.yaml`, rendered by `bin/setup.py` from `network-adapter.yaml.tmpl`; it uses the same keys and registry rows as the three tier adapters. - **Multi mode stays available as an option.** The three tier adapters (`provider-adapter` :9200, `network-adapter` :9201, `consumer-adapter` :9202) are now under compose profile `multi`. -- Make targets: `make up-multi` and `make up-core-multi` start multi mode; - `make smoke` runs publish, discover and select through the running mode. +- Make targets: `make up-multi` and `make up-core-multi` start multi mode. - `bin/stack.sh` takes `MODE=single|multi` (default `single`). - `.env.example` gains `NETWORK_ADAPTER_LAYER_PORT=9200` and `COMPOSE_PROFILES=network`, so a plain `docker compose up -d` starts single diff --git a/quick-start/Makefile b/quick-start/Makefile index 026b2d8a..453ea987 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -6,7 +6,6 @@ # make up the whole stack (Network Adapter), in order # make up-core the same minus the reverse proxy and hyperdx # make up-multi the whole stack with three adapters instead of one -# make smoke publish, discover, select through what is running # make down stop everything, keep the data # make help the rest # @@ -15,7 +14,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core up-multi up-core-multi smoke down destroy setup reverse-proxy observability pull restart restart-edge ps logs +.PHONY: help up up-core up-multi up-core-multi down destroy setup reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -29,7 +28,6 @@ up-core: ; @$(STACK) up-core down: ; @$(STACK) down up-multi: ; @$(STACK) up-multi up-core-multi: ; @$(STACK) up-core-multi -smoke: ; @$(STACK) smoke destroy: ; @$(STACK) destroy # --------------------------------------------- the optional tiers, on their own diff --git a/quick-start/README.md b/quick-start/README.md index 0c529d98..c79387a2 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -12,7 +12,6 @@ network adapter, and two mock upstreams standing in for real provider APIs. ```sh cp .env.example .env make up-core # single mode (default): registry, discovery, setup, the network adapter on 9200 -make smoke # publish, discover, select through it; all three should be 200 ``` Prerequisites, the VM path and the checks are in [SETUP.md](SETUP.md). @@ -34,7 +33,7 @@ same registry and data; a compose profile picks the adapters. | Start, VM (with edge and observability) | `make up` | `make up-multi` | | Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | | App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | -| Catalogue publishes to | `:9200/provider/publish` | `:9200/publish` | +| Catalogue publishes to | `:9200/publish` | `:9200/publish` | | Tier-to-tier hops | loopback HTTP inside the container | HTTP between containers | | Port setting in `.env` | `NETWORK_ADAPTER_LAYER_PORT` | `PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT`, `CONSUMER_ADAPTER_PORT` | @@ -76,7 +75,7 @@ Without `COMPOSE_PROFILES` a bare `docker compose up -d` starts no adapter. ignores `COMPOSE_PROFILES`. These follow whichever mode is running, with nothing to set: `make down`, -`make ps`, `make logs`, `make restart`, `make smoke`. +`make ps`, `make logs`, `make restart`. Which mode is running: @@ -129,10 +128,10 @@ layout as the Helm chart's `config/network-adapter-config.yaml`. | Module | Path | From | Routing file | Routes | |---|---|---|---|---| -| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/network`; select, init, confirm, status to `localhost:9200/provider` | -| `network` | `/network/` | `network.yaml.tmpl` | `routing-network.yaml` (the multi file, reused) | discover, publish to discovery | -| `provider` | `/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | -| `provider-publish` | `/provider/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/network/publish` | +| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/_internal/network`; select, init, confirm, status to `localhost:9200/_internal/provider` | +| `network` | `/_internal/network/` | `network.yaml.tmpl` | `routing-network.yaml` (the multi file, reused) | discover, publish to discovery | +| `provider` | `/_internal/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | +| `provider-publish` | `/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/_internal/network/publish` | The hops between modules are HTTP calls to the same container; in multi mode only the hosts differ. 9200 is the container's own listener and must match @@ -218,10 +217,11 @@ certificate. An unknown `Host` gets NPM's default page, not an adapter. `make up-multi`. With the single adapter there is no `consumer-adapter`, `network-adapter` or `provider-adapter` to forward to: point the consumer host at `network-adapter`, port 9200, with the same Advanced paste. The -app's paths (`/discover`, `/select`, …) do not change. Do **not** make -`/network/…` or `/provider/publish` reachable from a public host: `/network/` is -an internal tier and `/provider/publish` publishes as the provider without -checking the caller, and signing is off in quick-start. +app's paths (`/discover`, `/select`, …) do not change, and neither does +`/publish`: both modes present the same URLs. `server_proxy.conf` denies +`/publish` and `/_internal` on every host: `/_internal/` holds the network and +provider tiers, `/publish` publishes as the provider without checking the +caller, and signing is off in quick-start. **A fourth host for HyperDX**, if the `observability` profile is up — this is what replaces `ssh -L 8085` for reading telemetry: diff --git a/quick-start/SETUP.md b/quick-start/SETUP.md index d067405b..509db85e 100644 --- a/quick-start/SETUP.md +++ b/quick-start/SETUP.md @@ -24,14 +24,14 @@ minutes, most of it waiting for Keycloak. ## Step 1 — Prerequisites - `git` -- Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` +- Docker with **Compose v2.20 or later** — `docker compose version` must work, not `docker-compose`; the compose file uses `depends_on: required: false`, which older v2 releases reject - `python3` with `cryptography` — `pip install cryptography` - `curl` ## Step 2 — Get the repo ```sh -git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git +git clone -b development https://github.com/OpenAgriNet/helmcharts.git cd helmcharts/quick-start ``` @@ -152,17 +152,7 @@ which differs: `make logs SVC=network-adapter-layer` in single mode, ## Step 6 — Verify end to end -Quickest, in either mode: - -```sh -make smoke -``` - -It starts the mocks, sends publish, discover and select to whichever adapters -are running, and prints each HTTP status and the tier hops from the adapter -log. All three should be 200. - -The full collection, in `../api-collection/`, one copy per mode: +Run the API collection in `../api-collection/`, one copy per mode: | Mode | Collection | Environment | |---|---|---| @@ -212,7 +202,7 @@ Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. Nothing is cloned yet, so fetch the script rather than running it from the repo: ```sh -curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash +curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/development/quick-start/bin/bootstrap-ubuntu.sh | bash ``` Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt @@ -311,8 +301,6 @@ time: NPM only resolves `hyperdx` on reload. - **NPM's default page, or a 502 that worked yesterday** — Appendix B. - **An adapter config is a directory** — something started before `setup.py`. Appendix F. -- **`make smoke` says no adapter is running** — start one mode first: - `make up-core` or `make up-core-multi`. Full set, with what to run for each → [README Appendix F](README.md#appendix-f--when-it-does-not-work). diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index e991ffb0..5452be8e 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -179,10 +179,9 @@ up_setup() { # Only now do the bind-mounted config files exist. up_adapters() { - # The mocks are NOT started here (`make smoke` starts them for its select - # test). All four capabilities point at real external upstreams, so - # starting them would pull two images and run two containers nothing - # calls. They are still defined in docker-compose.yml, so + # The mocks are NOT started. All four capabilities point at real external + # upstreams, so starting them would pull two images and run two containers + # nothing calls. They are still defined in docker-compose.yml, so # `docker compose up -d mock-imd mock-agmarknet` brings them up for anyone # debugging a mapping without the real credentials -- but that also needs a # new participant id and base URL in .env plus a setup.py re-run, because @@ -381,73 +380,6 @@ restart_edge() { docker compose --profile reverse-proxy restart nginx-proxy-manager } -# ---------------------------------------------------------------- smoke - -# publish -> discover -> select against whichever topology is running. Starts -# the mocks (select's upstream is mock-imd in the shipped .env). Exits non-zero -# on any non-200 and prints the tier hops from the adapter log. -smoke() { - local mode base_pub base_app svc s - mode=$(running_mode) - if [ "$mode" = single ]; then - base_app="http://127.0.0.1:$(envv NETWORK_ADAPTER_LAYER_PORT 9200)"; base_pub="$base_app/provider" - svc=network-adapter - elif [ "$mode" = multi ]; then base_pub="http://127.0.0.1:$(envv PROVIDER_ADAPTER_PORT 9200)" - base_app="http://127.0.0.1:$(envv CONSUMER_ADAPTER_PORT 9202)" - svc="provider-adapter network-adapter consumer-adapter" - else - die "no adapter is running: make up-core (or make up-core-multi) first" - fi - step 1 2 "mocks (mock-imd, mock-agmarknet), and wait for the adapter" - docker compose up -d mock-imd mock-agmarknet >/dev/null - for s in $(seq 1 30); do curl -sf "$base_app/health" >/dev/null 2>&1 && break; sleep 1; done - curl -sf "$base_app/health" >/dev/null 2>&1 || die "$base_app/health does not answer after 30s: docker logs ${svc%% *}" - # The adapter answers /health before its registry is back (e.g. right after - # `make restart`), and select looks the provider up in the registry. - for s in $(seq 1 90); do - [ "$(docker inspect -f '{{.State.Health.Status}}' sunbird-registry-service 2>/dev/null)" = healthy ] && break; sleep 2 - done - [ "$(docker inspect -f '{{.State.Health.Status}}' sunbird-registry-service 2>/dev/null)" = healthy ] \ - || die "sunbird-registry-service is not healthy after 3 minutes: docker logs sunbird-registry-service" - step 2 2 "publish, discover, select through the $mode topology" - ROOT="$ROOT" PUB="$base_pub" APP="$base_app" python3 - <<'PY' -import json, os, pathlib, sys, urllib.error, urllib.request, uuid -root = pathlib.Path(os.environ["ROOT"]) -env = {} -for line in (root / ".env").read_text().splitlines(): - if "=" in line and not line.lstrip().startswith("#"): - k, v = line.split("=", 1); env[k.strip()] = v.strip() -steps = (("publish", os.environ["PUB"] + "/publish"), - ("discover", os.environ["APP"] + "/discover"), - ("select", os.environ["APP"] + "/select")) -rc = 0 -for step, url in steps: - text = next((root / "capability-examples/WeatherObservation" / step).glob("*.json")).read_text() - for old, key in (("provider.oan.dev", "PROVIDER_SUBSCRIBER_ID"), - ("consumer.oan.dev", "CONSUMER_SUBSCRIBER_ID"), - ("network.oan.dev", "NETWORK_SUBSCRIBER_ID")): - text = text.replace(old, env[key]) - text = text.replace('"mausamgram"', json.dumps(env["MAUSAMGRAM_PARTICIPANT_ID"])) - body = json.loads(text) - body["context"]["transactionId"], body["context"]["messageId"] = str(uuid.uuid4()), str(uuid.uuid4()) - req = urllib.request.Request(url, json.dumps(body).encode(), {"Content-Type": "application/json"}) - try: - with urllib.request.urlopen(req, timeout=30) as r: - status, data = r.status, r.read() - except urllib.error.HTTPError as e: - status, data = e.code, e.read() - except (urllib.error.URLError, ConnectionError) as e: - status, data = 0, str(e).encode() - print(f" {step:9} {url:42} HTTP {status}") - if status != 200 or b"NACK" in data: - rc = 1; print(" body:", data[:300].decode(errors="replace")) -sys.exit(rc) -PY - echo; echo " tier hops in the adapter log:" - for s in $svc; do docker logs --since 2m "$s" 2>&1; done \ - | grep -o 'Forwarding request to URL: [^"]*' | sort | uniq -c | sed 's/^/ /' || true -} - # Just step 2. Re-run it after editing a .tmpl, or to re-render configs that # were deleted. It is idempotent, so this is always safe. setup() { @@ -466,7 +398,6 @@ bin/stack.sh up-core steps 1-3 only. Nothing public, no ClickHouse. up-multi as up, with the three adapters (provider, network, consumer) up-core-multi as up-core, with the three adapters - smoke publish, discover, select through whichever topology runs (MODE=multi also works with up, up-core and restart) down stop everything, keep the data destroy stop everything and DELETE every volume @@ -487,7 +418,6 @@ case "${1:-}" in up-core) up_core ;; up-multi) MODE=multi; up ;; up-core-multi) MODE=multi; up_core ;; - smoke) smoke ;; down) down ;; destroy) destroy ;; setup) setup ;; diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 5324db14..ce5a70e2 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -1,10 +1,13 @@ # Network Adapter: the consumer, network and provider tiers as # four modules of ONE adapter process (docker-compose.yml, profile "network"). # -# / the consumer (bap); the app posts here, unsigned -# /network/ verifies and forwards to discovery -# /provider/ answers from the upstream APIs (bpp) -# /provider/publish the catalogue publishes here +# / the consumer (bap); the app posts here, unsigned +# /publish the catalogue publishes here +# /_internal/network/ verifies and forwards to discovery +# /_internal/provider/ answers from the upstream APIs (bpp) +# +# The public paths are the same as the multi adapters'. /_internal/ is only +# called by this process's own modules over loopback. # # Each module is its tier's module from consumer.yaml.tmpl, network.yaml.tmpl # and provider.yaml.tmpl. A change to one of those templates that should also @@ -19,9 +22,9 @@ # - log level info. # # SIGNING IS OFF, as in the three quick-start tier templates: no module runs -# sign or validateSign, so /network/ and /provider/ accept unsigned requests +# sign or validateSign, so /_internal/ and /publish accept unsigned requests # from anyone who can reach :9200. Keep it on 127.0.0.1 (as the compose file -# does) and never put /network/ or /provider/ behind the reverse proxy. The +# does); server_proxy.conf denies /_internal and /publish at the edge. The # Helm chart's network-adapter-config.yaml has signing on. # # bin/setup.py fills the placeholders -- each module's own tier identity @@ -65,8 +68,8 @@ modules: # ------------------------------------------------------------------ consumer # From consumer.yaml.tmpl. Mounted at /, so the app calls /discover, /select, # ... as on the multi-adapter consumer-adapter. Accepts unsigned requests - # under the consumer identity (signing is off here), routes discover to /network and transactions to - # /provider (routing-network-adapter-consumer.yaml). + # under the consumer identity (signing is off here), routes discover to /_internal/network and + # transactions to /_internal/provider (routing-network-adapter-consumer.yaml). - name: consumer path: / handler: @@ -132,7 +135,7 @@ modules: # From network.yaml.tmpl. Forwards discover and # publish to discovery (routing-network.yaml, the multi-adapter file). - name: network - path: /network/ + path: /_internal/network/ handler: type: std role: bpp @@ -195,7 +198,7 @@ modules: # From provider.yaml.tmpl, module 1. Answers from the # upstream APIs itself: no router. - name: provider - path: /provider/ + path: /_internal/provider/ handler: type: std role: bpp @@ -300,10 +303,11 @@ modules: # ---------------------------------------------------------- provider-publish # From provider.yaml.tmpl, module 2. The catalogue publishes here, under the - # provider identity (signing is off here); routed to /network/publish - # (routing-network-adapter-publish.yaml). + # provider identity (signing is off here); routed to /_internal/network/publish + # (routing-network-adapter-publish.yaml). Top-level /publish, the same path + # as the multi provider adapter's. - name: provider-publish - path: /provider/publish + path: /publish handler: type: std role: bap diff --git a/quick-start/config/adapters/routing-network-adapter-consumer.yaml b/quick-start/config/adapters/routing-network-adapter-consumer.yaml index 44b659f1..dee56952 100644 --- a/quick-start/config/adapters/routing-network-adapter-consumer.yaml +++ b/quick-start/config/adapters/routing-network-adapter-consumer.yaml @@ -16,7 +16,7 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/network" + url: "http://localhost:9200/_internal/network" endpoints: - discover @@ -26,7 +26,7 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/provider" + url: "http://localhost:9200/_internal/provider" endpoints: - select - init diff --git a/quick-start/config/adapters/routing-network-adapter-publish.yaml b/quick-start/config/adapters/routing-network-adapter-publish.yaml index 19838113..d43b425d 100644 --- a/quick-start/config/adapters/routing-network-adapter-publish.yaml +++ b/quick-start/config/adapters/routing-network-adapter-publish.yaml @@ -5,14 +5,14 @@ # takes the same path a publish from anywhere else takes -- one place that # decides what reaches discovery, rather than two. # -# The publish module is mounted at the exact path /provider/publish, so the +# The publish module is mounted at the exact path /publish, so the # endpoint left after stripping it is "" -- which is what this rule keys on, # with excludeAction so nothing is appended to the target. routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/network/publish" + url: "http://localhost:9200/_internal/network/publish" excludeAction: true endpoints: - "" diff --git a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf index 35f956d6..4784a494 100644 --- a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf +++ b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf @@ -2,12 +2,12 @@ # /data/nginx/custom/server_proxy.conf. No UI step. # # Because it applies to every host, only rules that are correct everywhere -# belong here. There is exactly one, and it matters more than the rest of this +# belong here. There are two, and they matter more than the rest of this # stack's edge configuration combined. # The provider adapter mounts TWO modules. / is its Beckn surface and -# verifies the sender's signature against the registry. But oanProviderPublish -# is mounted at `/` with NO validateSign at all, because its intended caller is +# verifies the sender's signature against the registry. But the publish module +# is mounted at `/publish` with NO validateSign at all, because its intended caller is # the provider's own catalogue system, inside the trust boundary -- the same # reason the consumer adapter takes unsigned calls. # @@ -25,6 +25,13 @@ # provider-adapter directly -- an endpoint with no credential to check does not # belong on the public edge, and an address allowlist in front of it is a # statement about the network, which is where it should be made. -# location = /publish { -# deny all; -# } +location = /publish { + deny all; +} + +# The Network Adapter Layer (single mode) mounts its network and provider tiers +# under /_internal/. Only the process itself calls them, over loopback, so the +# whole prefix is denied. ^~ stops regex locations from overriding it. +location ^~ /_internal { + deny all; +} diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 07194889..d7c4b19c 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -611,14 +611,14 @@ services: # config/adapters/network-adapter.yaml.tmpl, with one routing file per # module. # - # / the consumer, the caller's entry (bap), takes unsigned requests - # /network/* hands discover and publish on to discovery - # /provider/* calls the upstream, answers synchronously - # /provider/publish sends the provider's catalogue to /network/publish + # / the consumer, the caller's entry (bap), takes unsigned requests + # /publish sends the provider's catalogue to /_internal/network/publish + # /_internal/network/* hands discover and publish on to discovery + # /_internal/provider/* calls the upstream, answers synchronously # # Each module carries its own tier identity (consumer, network, provider): # the same keys and registry rows as the multi-adapter services. Signing is - # off, as in every quick-start template, so /network/ and /provider/ accept + # off, as in every quick-start template, so /_internal/ and /publish accept # unsigned requests: keep :9200 on 127.0.0.1. network-adapter-layer: <<: *adapter From 3dd6f6e8658dd5f928590d3b1f0c8e15ce3a9ec2 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 20:41:41 +0530 Subject: [PATCH 26/43] feat(env.example): fix - Run a single adapter through the helm charts Fixes #55 add clarification for single mode configuration in environment variables --- quick-start/.env.example | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/quick-start/.env.example b/quick-start/.env.example index 999172a8..236854c7 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -148,6 +148,14 @@ PROVIDER_SUBSCRIBER_ID=provider.oan.dev # The address PEERS are told to POST to, published in each registry row. Nothing # inside this stack resolves it -- routing here is by compose service name -- so # a wrong value costs nothing locally and everything on a shared network. +# +# Single mode (network-adapter) has one public host, so only CONSUMER_BASE_URL +# names a real address there. The network and provider modules sit under +# /_internal/ and the edge denies them; their rows are read for key lookup +# only, and no peer resolves their baseUrl while OAN is single-sided. They are +# still kept per tier, the same as multi mode, on purpose: the registry cannot +# update a row, so a later split to multi mode (or a separate inbound host) +# would otherwise need new participant ids. CONSUMER_BASE_URL=https://consumer.oan.dev NETWORK_BASE_URL=https://network.oan.dev PROVIDER_BASE_URL=https://provider.oan.dev From 5f068598558cf364a1cab562ec26a309e06e1c4a Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 21:00:57 +0530 Subject: [PATCH 27/43] fix(adapter-service, quick-start): align docs with flat paths after PR #75 review (#55, #66) adapter-service: - NOTES/values: tier hops go to localhost:/_internal/ - CHANGELOG: routing entry names /_internal/; drop the removed appName/otelServiceName special case; "Flat paths" wording; HPA noted as opt-in, off by default - values: otel.serviceName comment no longer describes a removed helper - README/config header: define what the ingress may expose (single-segment /; /publish stays in-cluster, /_internal* never routed) - ci/network-adapter-values: header no longer claims the HPA is linted (autoscaling stays off by choice) - ingress: use-regex check is case-insensitive ("True" is refused too) quick-start: - server_proxy.conf, README Appendix B: /publish and /_internal denies on every host, single-mode publish module and host - README: Appendix G pull/log commands per mode; Appendix D keys also rendered into network-adapter.yaml; Appendix K lists the new files; Appendix M removes network-adapter.yaml; restart follows the running mode - SETUP: all four capabilities call real upstreams; single-mode hops are on localhost - stack.sh: drop unused envv(); header and help list the multi targets - Makefile header lists up-core-multi - CHANGELOG: 0.2.0 dated 2026-10-14 Refs #55 Refs #66 --- charts/adapter-service/CHANGELOG.md | 10 +++--- charts/adapter-service/README.md | 9 ++--- .../ci/network-adapter-values.yaml | 5 +-- .../config/network-adapter-config.yaml | 8 ++--- charts/adapter-service/templates/NOTES.txt | 2 +- charts/adapter-service/templates/ingress.yaml | 2 +- charts/adapter-service/values.yaml | 6 ++-- quick-start/CHANGELOG.md | 2 +- quick-start/Makefile | 1 + quick-start/README.md | 34 +++++++++++++------ quick-start/SETUP.md | 11 +++--- quick-start/bin/stack.sh | 9 +++-- .../npm-custom/server_proxy.conf | 8 +++-- 13 files changed, 62 insertions(+), 45 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index db645063..5901ff03 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -23,12 +23,12 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h a tier file that should apply to both topologies is made in both. Its routing is generated at render time from `config/routing-*.yaml` (`routing-network-adapter-{consumer,publish}.yaml`), with tier hosts rewritten - to `http://localhost:`, so tier hops stay inside the pod. The render fails if `http.port` differs from `service.targetPort`. + to `http://localhost:/_internal/`, so tier hops stay inside the pod. The render fails if `http.port` differs from `service.targetPort`. - Three identities: each module signs as its own tier from `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` (`networkAdapterLayer.extraSubstitutions` for extra placeholders). A literal private key in the config, including a `config:` override, fails the render. -- Flat public surface: `/discover`, `/publish`, `/` and `/health`, the +- Flat paths: `/discover`, `/publish`, `/` and `/health`, the same URLs as the multi topology. The network and provider tiers are mounted under `/_internal/`, which only the process itself calls. - Ingress for the layer may expose only single-segment `/`, as literal @@ -36,7 +36,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h annotation; `/`, `/publish` and `/_internal*` are refused. `routing.config` fails the render for this role. - `examples/network-adapter.yaml` and `ci/network-adapter-values.yaml` - (HPA 3-10, PDB, spread, graceful shutdown). + (PDB, spread, graceful shutdown; HPA 3-10 opt-in, off by default). - `autoscaling.behavior`, `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. @@ -48,8 +48,8 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h (telemetry service `oan-network-adapter`); its namespace is `network-adapter-layer`. "layer" appears only in the namespace, the role, the values key and helper names, where it must differ from the multi - `network` tier. The `-adapter` default for app name and telemetry - service name is special-cased for it. The separate namespace keeps the + `network` tier. The app name and telemetry service name are set in + `config/network-adapter-config.yaml`. The separate namespace keeps the consumer and provider keys, which openbao-secrets mirrors to the layer, out of the multi network tier's namespace, and lets both topologies run side by side during a switch-over. diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index b0ccec0a..f8c28d64 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -22,10 +22,11 @@ and where requests are routed. /publish ──lo──▶ /_internal/network/publish ``` -The public surface is flat and the same in both topologies: `/discover`, -`/publish`, `/` and `/health`. The network and provider tiers sit under -`/_internal/`, which only this process calls (over loopback), so the edge -denies the whole prefix with one rule. +The paths are flat and the same in both topologies: `/discover`, `/publish`, +`/` and `/health`. The network and provider tiers sit under +`/_internal/`, which only this process calls (over loopback). The ingress may +expose only single-segment `/` paths: `/publish` stays in-cluster for +the catalogue publisher, and `/_internal*` is never routed. Why flat: OAN is single-sided today -- our app calls out, and our own discovery and upstreams answer. If OAN later accepts inbound calls from diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 54c1ad12..2e7ec458 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -4,8 +4,9 @@ # # Renders the single topology, role network-adapter-layer (the Network Adapter # Layer): config/network-adapter-config.yaml, the generated routing, -# the three tier identities, and the HPA / PDB / spread / graceful-shutdown -# settings, so those templates are linted too. Pinned image tag so the render +# the three tier identities, and the PDB / spread / graceful-shutdown +# settings, so those templates are linted too. The HPA stays off, as in +# examples/network-adapter.yaml: autoscaling is opt-in. Pinned image tag so the render # is reproducible. role: network-adapter-layer fullnameOverride: network-adapter diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index a3c1c146..f45cfabc 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -6,10 +6,10 @@ # /_internal/network/ verifies and forwards to discovery # /_internal/provider/ answers from the upstream APIs (bpp) # -# The public surface is flat: /discover, /publish, / and /health. The -# tiers are not part of it. /_internal/ is only ever called by this process's -# own modules (http://localhost:9200/_internal/...), so the edge denies the -# whole prefix with one rule. +# The paths are flat: /discover, /publish, / and /health. The tiers are +# not part of them. /_internal/ is only ever called by this process's own +# modules (http://localhost:9200/_internal/...); the chart's ingress never +# routes it, and never routes /publish either. # # Each module is its tier's module from consumer-config.yaml, network-config.yaml # and provider-config.yaml. A change to one of those files that should also diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index a0391275..75ddcf6e 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -2,7 +2,7 @@ {{ if include "adapter-service.isNetworkAdapterLayer" . }} topology single (the default): the Network Adapter Layer role network-adapter-layer (all three tiers in one adapter: / consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) - hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/) + hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/_internal/) {{- else }} topology multi (optional): one release per tier; the default is role network-adapter-layer role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index 2e5b8032..a563fb18 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -11,7 +11,7 @@ on ingress-nginx, and regex or rewrite annotations turn paths into patterns, so either could reach /_internal or /publish. */}} {{- $ann := merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict) }} -{{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString) "true" }} +{{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString | trim | lower) "true" }} {{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /_internal or /publish." .Chart.Name) }} {{- end }} {{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 63a4631b..2e60a2c0 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -276,8 +276,8 @@ otel: # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" endpoint: "" environment: dev - # Left empty, defaults to oan--adapter, matching OTEL_SERVICE_NAME in - # the compose stack so traces from either deployment line up. + # Not read by any template: each role's config file sets its telemetry + # service name (config/-config.yaml, telemetry...serviceName). serviceName: "" logLevel: info @@ -446,7 +446,7 @@ envFromSecrets: [] # file: log level, timeouts, telemetry, extended schema and the outbound # connection pool are set there. The routing is generated at render time from # the tier routing files, with tier-to-tier hops pointed at this pod's own -# listener (http://localhost:/): any adapter image +# listener (http://localhost:/_internal/): any adapter image # works. # # Ignored by the provider, network and consumer roles. diff --git a/quick-start/CHANGELOG.md b/quick-start/CHANGELOG.md index 211d8c14..a5b00691 100644 --- a/quick-start/CHANGELOG.md +++ b/quick-start/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the quick-start stack are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## [0.2.0] - 2026-10-13 +## [0.2.0] - 2026-10-14 ### Added - **Single network adapter, now the default mode.** One adapter process runs diff --git a/quick-start/Makefile b/quick-start/Makefile index 453ea987..1d3f3b9a 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -6,6 +6,7 @@ # make up the whole stack (Network Adapter), in order # make up-core the same minus the reverse proxy and hyperdx # make up-multi the whole stack with three adapters instead of one +# make up-core-multi up-core with three adapters instead of one # make down stop everything, keep the data # make help the rest # diff --git a/quick-start/README.md b/quick-start/README.md index c79387a2..8bd65c25 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -66,7 +66,7 @@ Other ways to pick the mode: | How | Single | Multi | |---|---|---| | make target | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | -| `MODE` variable (works with `up`, `up-core`, `restart`) | `MODE=single make up-core` (the default) | `MODE=multi make up-core` | +| `MODE` variable (works with `up` and `up-core`; `restart` follows the running mode) | `MODE=single make up-core` (the default) | `MODE=multi make up-core` | | plain compose | `docker compose --profile network up -d` | `docker compose --profile multi up -d` | | bare `docker compose up -d` | `COMPOSE_PROFILES=network` in `.env` (as in `.env.example`) | `COMPOSE_PROFILES=multi` in `.env` | @@ -301,21 +301,24 @@ request. It is also the only option for a wildcard. Renewal reuses whichever method you chose, so a temporarily-correct DNS record or SG rule fails silently in sixty days. -### `POST /publish` returns 403 on all three hosts +### `POST /publish` and `/_internal` return 403 on every host Not optional hardening. The provider adapter's module at `/` verifies the -sender's signature; `oanProviderPublish`, on the exact path `/publish`, has +sender's signature; the publish module (`oanProviderPublish` in multi mode, +`provider-publish` in single mode), on the exact path `/publish`, has **no signature check at all**, because its intended caller is the provider's own catalogue system inside the trust boundary. A proxy host pointed at -`provider-adapter:9200` therefore exposes `/publish` to anyone. +`provider-adapter:9200` (multi) or `network-adapter:9200` (single) therefore +exposes `/publish` to anyone. In single mode the network and provider +tiers also sit under `/_internal/`, which only the adapter itself calls. NPM's UI cannot route a host while withholding one path, so the block lives in `config/reverse-proxy/npm-custom/server_proxy.conf`, which NPM includes in **every** proxy host's server block — a mounted file, not a click, so not -something to remember on one host out of three. +something to remember on one host. To let a real catalogue system publish, give it a tunnel or put it in the VPC -and let it reach `provider-adapter:9200` directly. Do not turn that `deny` into +and let it reach the adapter on port 9200 directly. Do not turn that `deny` into an `allow`. ### Which nginx config loads itself, and which is a paste job @@ -323,7 +326,7 @@ an `allow`. | File | How it applies | |---|---| | `npm-custom/http_top.conf` | **Automatic**, top of the `http` block. The `consumer` rate-limit zone and `limit_req_status 429`. | -| `npm-custom/server_proxy.conf` | **Automatic**, every server block. The `/publish` deny. | +| `npm-custom/server_proxy.conf` | **Automatic**, every server block. The `/publish` and `/_internal` denies. | | `npm-advanced/consumer.conf` | **Manual** — paste into the consumer host's Advanced tab. `limit_req` for that host only; a 10 r/s ceiling on signed peer traffic would throttle for no gain. | | `npm-advanced/hyperdx.conf` | **Manual** — paste into the HyperDX host's Advanced tab. 300s timeouts for ClickHouse scans, buffering off for live tail. Carries no credential; the Access List does. | @@ -445,7 +448,7 @@ unedited. Those four strings are the hinge. The provider adapter builds the same key from each payload — the provider id and the capability `@type` it carries — and a step answers only when the key matches its own. `setup.py` renders those keys -into `provider.yaml` from the same `.env` it seeds the registry from, which is +into `provider.yaml` and `network-adapter.yaml` from the same `.env` it seeds the registry from, which is what stops the two drifting. **Looking at it** — from the VM or a tunnel. Search takes no token; writes do, @@ -592,7 +595,7 @@ an id is a `.so` basename. If `ADAPTER_IMAGE` names a **new tag**, set it before nothing in `stack.sh` pulls, so the stack quietly comes back on the old image: ```sh -docker compose pull provider-adapter network-adapter consumer-adapter +docker compose pull network-adapter-layer provider-adapter network-adapter consumer-adapter ``` Build from the adapter repo at the commit the config expects, and check the @@ -613,6 +616,10 @@ publish reseeds first; `updateMode: MERGE` updates in place. **Then check, cheapest first:** ```sh +# single mode (the default) +docker logs network-adapter 2>&1 | grep 'Processor steps initialized' +docker logs network-adapter 2>&1 | grep -iE '"level":"(error|fatal)"' +# multi mode docker compose logs provider-adapter | grep 'Processor steps initialized' docker compose logs provider-adapter | grep -iE '"level":"(error|fatal)"' make ps @@ -856,6 +863,8 @@ one minute of adapter cache plus about five of CDN. docker-compose.yml the whole stack, in tiers -- the banner comments are the structure .env.example copy to .env +SETUP.md install and run, local and VM, both modes +CHANGELOG.md what changed, by version Makefile the front door; every target delegates to stack.sh bin/ bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM @@ -865,7 +874,7 @@ config/ reverse-proxy/ npm-custom/ mounted to /data/nginx/custom; NPM includes these http_top.conf on its own -- rate-limit zone, and the /publish - server_proxy.conf deny every proxy host gets + server_proxy.conf and /_internal denies every proxy host gets npm-advanced/ NOT loaded. Paste into a host's Advanced tab; kept consumer.conf here because a textarea in a database is not hyperdx.conf reviewable. consumer = rate limit; hyperdx = ClickHouse @@ -881,6 +890,9 @@ config/ routing-consumer.yaml which action goes where: consumer sends discover routing-network.yaml to the network layer and select to the provider; routing-provider.yaml provider sends publish to the network layer + network-adapter.yaml.tmpl single mode: all three tiers, one file + routing-network-adapter-consumer.yaml single mode: consumer hops to + routing-network-adapter-publish.yaml localhost:9200/_internal/... registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs a restart @@ -922,7 +934,7 @@ that volume — and equally does not survive if you skip it. ```sh docker compose down -v rm -rf keys config/adapters/consumer.yaml config/adapters/network.yaml \ - config/adapters/provider.yaml + config/adapters/provider.yaml config/adapters/network-adapter.yaml ``` New keys mean new identities, so the provider rows must be created again and diff --git a/quick-start/SETUP.md b/quick-start/SETUP.md index 509db85e..f0dd58e8 100644 --- a/quick-start/SETUP.md +++ b/quick-start/SETUP.md @@ -62,8 +62,9 @@ If a port is already taken, change it in `.env` — that is the only edit a local run needs. The single adapter's host port is `NETWORK_ADAPTER_LAYER_PORT`; in multi mode the three are `PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT` and `CONSUMER_ADAPTER_PORT`. -Adapters reach each other by Compose service name, so a port change only -changes what you type into Postman. +Adapters reach each other inside the stack (by Compose service name in multi +mode, on `localhost:9200` inside the one container in single mode), so a port +change only changes what you type into Postman. ## Step 4 — Start it @@ -114,9 +115,9 @@ all four capabilities. - Upstream credentials: `AGMARKNET_ACCESS_NAME` + `AGMARKNET_PASSWORD`, and `VISTAAR_CLIENT_ID` + `VISTAAR_CLIENT_SECRET`. POCRA needs none. -**Only mausamgram is a mock now.** Mandi, knowledge and POCRA are real -external hosts, so each needs its own base URL and path in `.env` — and the -stack needs egress to all three. Mandi and knowledge each also need a token +**All four capabilities call real upstreams.** Mausamgram, mandi, knowledge +and POCRA are external hosts, so each needs its own base URL and path in +`.env` — and the stack needs egress to all four. Mandi and knowledge each also need a token endpoint, since both issue short-lived tokens rather than taking a static credential. diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 5452be8e..fc467770 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -4,6 +4,8 @@ # # bin/stack.sh up the whole stack, in the order it has to start # bin/stack.sh up-core the same minus the gateway and hyperdx +# bin/stack.sh up-multi as up, with the three tier adapters +# bin/stack.sh up-core-multi as up-core, with the three tier adapters # bin/stack.sh down stop everything, keep the data # bin/stack.sh destroy stop everything, DELETE the data # @@ -81,10 +83,6 @@ step() { printf '\n\033[1;36m==> [%s/%s] %s\033[0m\n' "$1" "$2" "$3"; } info() { printf ' %s\n' "$1"; } warn() { printf '\033[1;33mwarning: %s\033[0m\n' "$1" >&2; } die() { printf '\033[1;31mstack: %s\033[0m\n' "$1" >&2; exit 1; } -envv() { # $1 key, $2 default -- value from .env, else the default - local v; v=$(grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-) - echo "${v:-$2}" -} # -------------------------------------------------------------- preflight @@ -398,7 +396,8 @@ bin/stack.sh up-core steps 1-3 only. Nothing public, no ClickHouse. up-multi as up, with the three adapters (provider, network, consumer) up-core-multi as up-core, with the three adapters - (MODE=multi also works with up, up-core and restart) + (MODE=multi also works with up and up-core; restart + follows the running mode) down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only diff --git a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf index 4784a494..fbfaa780 100644 --- a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf +++ b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf @@ -14,11 +14,13 @@ # So a proxy host pointed at provider-adapter:9200 exposes, at /publish, # an unauthenticated "write anything into the catalogue" endpoint. NPM's UI # gives no way to route a host while withholding one path, and this is not a -# thing to leave to remembering an Advanced-tab paste on one host out of three. +# thing to leave to remembering an Advanced-tab paste on one host. # # An exact-match location beats NPM's generated `location /`, so this wins -# without conflicting with it. On the consumer and network hosts it denies a path -# their adapters do not serve, which costs nothing. +# without conflicting with it. In multi mode, on the consumer and network hosts +# it denies a path their adapters do not serve, which costs nothing. In single +# mode the one adapter (network-adapter:9200) serves /publish itself +# (provider-publish module, no validateSign), so the deny matters on its host. # # To open it for a real catalogue system outside this VM, do NOT edit this to # `allow`. Give that caller a tunnel, or put it in the VPC and reach From f62d0574d1a56372a1c26e6622c25a89a5f5ba78 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 21:37:36 +0530 Subject: [PATCH 28/43] refactor(adapter-service, quick-start): tidy network-adapter-layer code (#55, #66) - network-adapter config/tmpl: drop the network module's schemaValidator; validateSchema is not one of its steps, so it was never used - NOTES/ingress: compute the role check and annotation merge once - stack.sh: configs_for() names each mode's rendered configs; drop the cleanup of pre-rename container names that only existed on this branch Refs #55 Refs #66 --- .../config/network-adapter-config.yaml | 13 ++----------- charts/adapter-service/templates/NOTES.txt | 13 +++++++------ charts/adapter-service/templates/ingress.yaml | 4 ++-- quick-start/bin/stack.sh | 12 ++++++++---- .../config/adapters/network-adapter.yaml.tmpl | 13 ++----------- 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index f45cfabc..e8d91dd9 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -176,17 +176,8 @@ modules: signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" encrPublicKey: "__NETWORK_ENCR_PUBLIC__" - schemaValidator: - id: schemav2validator - config: - type: url - location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" - cacheTTL: "3600" - extendedSchema_enabled: "true" - extendedSchema_cacheTTL: "86400" - extendedSchema_maxCacheSize: "100" - extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + # No schemaValidator: validateSchema is not a step here, and a configured + # validator still downloads the spec at boot and refreshes it hourly. signValidator: id: signvalidator signer: diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index 75ddcf6e..d10642f2 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,5 +1,6 @@ +{{- $nal := include "adapter-service.isNetworkAdapterLayer" . -}} {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} -{{ if include "adapter-service.isNetworkAdapterLayer" . }} +{{ if $nal }} topology single (the default): the Network Adapter Layer role network-adapter-layer (all three tiers in one adapter: / consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/_internal/) @@ -10,7 +11,7 @@ {{- end }} image {{ include "adapter-service.image" . }} service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} -{{- if include "adapter-service.isNetworkAdapterLayer" . }} +{{- if $nal }} registry set in config/network-adapter-config.yaml (registry.url is not used by this role) routes generated from config/routing-{consumer,network,provider}.yaml telemetry set in config/network-adapter-config.yaml (otel.* is not used by this role) @@ -21,7 +22,7 @@ {{- end }} This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}.{{ include "common.namespace" . }}.svc:{{ .Values.service.port }}. -{{- if include "adapter-service.isNetworkAdapterLayer" . }} +{{- if $nal }} The app and the catalogue publisher call it there; its tiers call each other inside the pod, so no other adapter routes to it. {{- else }} @@ -39,7 +40,7 @@ chart's appVersion. TWO THINGS THIS CHART CANNOT CHECK FOR YOU -{{ if include "adapter-service.isNetworkAdapterLayer" . -}} +{{ if $nal -}} 1. The Participant rows. This adapter signs as three identities -- consumer, network and provider -- from Secrets {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "consumer") }}, {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "network") }} and {{ include "adapter-service.networkAdapterLayer.tierSecret" (dict "ctx" . "tier" "provider") }}: the same rows the @@ -67,7 +68,7 @@ Callers address the tiers by path on this one Service: kubectl -n {{ include "common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} {{- end }} -{{- if and (not .Values.otel.enabled) (not (include "adapter-service.isNetworkAdapterLayer" .)) }} +{{- if and (not .Values.otel.enabled) (not $nal) }} Telemetry is off, which is the right setting until a collector exists: with it on and nothing listening the exporter logs a failure every interval. @@ -75,7 +76,7 @@ on and nothing listening the exporter logs a failure every interval. {{- if .Values.ingress.enabled }} Ingress is ON. -{{- if include "adapter-service.isNetworkAdapterLayer" . }} +{{- if $nal }} This is the network-adapter-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is inside the trust boundary and has no validateSign step, so an Ingress on it exposes an unauthenticated entry point to the network. Rate-limit this host. diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index a563fb18..ab0044fa 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,4 +1,5 @@ {{- if .Values.ingress.enabled }} +{{- $ann := merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict) }} {{- if include "adapter-service.isNetworkAdapterLayer" . }} {{- /* role network-adapter-layer: the public surface is flat -- /discover, / and /health, all served by the consumer module at /. Two @@ -10,7 +11,6 @@ Prefix and Exact only: ImplementationSpecific is a plain string prefix on ingress-nginx, and regex or rewrite annotations turn paths into patterns, so either could reach /_internal or /publish. */}} -{{- $ann := merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict) }} {{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString | trim | lower) "true" }} {{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /_internal or /publish." .Chart.Name) }} {{- end }} @@ -47,7 +47,7 @@ metadata: name: {{ include "adapter-service.fullname" . }} labels: {{- include "adapter-service.labels" . | nindent 4 }} - {{- with (merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict)) }} + {{- with $ann }} annotations: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index fc467770..db49ed2f 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -69,6 +69,13 @@ adapters_for() { # $1 mode multi) echo provider-adapter network-adapter consumer-adapter ;; esac } +# The rendered config files (config/adapters/.yaml) a topology mounts. +configs_for() { # $1 mode + case "$1" in + single) echo network-adapter ;; + multi) echo provider network consumer ;; + esac +} other_mode() { [ "$1" = single ] && echo multi || echo single; } # The compose profile for a topology: single runs under profile "network", @@ -191,11 +198,8 @@ up_adapters() { # shares the multi network tier's name: remove it (stopped is not enough). # shellcheck disable=SC2046 docker compose --profile "$(profile_for "$other")" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true - # Containers from before the rename (single-network-layer-adapter, and the - # earlier network-layer-adapter) would hold 127.0.0.1:9200. - docker rm -f single-network-layer-adapter network-layer-adapter >/dev/null 2>&1 || true # Never let compose create a directory where a bind-mounted config belongs. - for f in $( [ "$MODE" = single ] && echo network-adapter || echo provider network consumer ); do + for f in $(configs_for "$MODE"); do [ -f "config/adapters/$f.yaml" ] \ || die "config/adapters/$f.yaml was not rendered (a directory there? rm -r it, then re-run)" done diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index ce5a70e2..2a8a7dab 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -176,17 +176,8 @@ modules: config: contextKeys: transaction_id,message_id role: bpp - schemaValidator: - id: schemav2validator - config: - type: url - location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml - cacheTTL: '3600' - extendedSchema_enabled: "true" - extendedSchema_cacheTTL: '86400' - extendedSchema_maxCacheSize: '100' - extendedSchema_downloadTimeout: '30' - extendedSchema_allowedDomains: raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io + # No schemaValidator: validateSchema is not a step here, and a configured + # validator still downloads the spec at boot and refreshes it hourly. router: id: router config: From ba54a2d89a79bad30297938e46c01d71eadd6215 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Wed, 14 Oct 2026 21:42:40 +0530 Subject: [PATCH 29/43] refactor(api-collection, quick-start): one Postman collection for both modes (#55, #66) The paths are flat and the same in single and multi mode, so the single-mode collection was a copy of OpenAgriNet.api-collection.json with different host variables. Drop the copy and keep one collection; the environment picks the mode. - remove OpenAgriNet.network-adapter.api-collection.json - OpenAgriNet.network-adapter.local-environment.json: the multi environment with every adapter host on :9200 (single mode) - SETUP/README/CHANGELOG: one collection, an environment per mode Refs #55 Refs #66 --- ...griNet.network-adapter.api-collection.json | 732 ------------------ ...Net.network-adapter.local-environment.json | 22 +- quick-start/CHANGELOG.md | 6 +- quick-start/README.md | 5 +- quick-start/SETUP.md | 12 +- 5 files changed, 33 insertions(+), 744 deletions(-) delete mode 100644 api-collection/OpenAgriNet.network-adapter.api-collection.json diff --git a/api-collection/OpenAgriNet.network-adapter.api-collection.json b/api-collection/OpenAgriNet.network-adapter.api-collection.json deleted file mode 100644 index 3515f33f..00000000 --- a/api-collection/OpenAgriNet.network-adapter.api-collection.json +++ /dev/null @@ -1,732 +0,0 @@ -{ - "info": { - "name": "OpenAgriNet - Network Adapter Layer", - "_postman_id": "0a4e1c00-0000-4000-8000-0000000snl01", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "description": "The four OAN capabilities, one folder each, requests numbered in the order you run them.\n\nImport api-collection/OpenAgriNet.network-adapter.local-environment.json alongside this and select it, or every {{...-host}} resolves to nothing.\n\nBodies are generated from quick-start/capability-examples/ and are otherwise unedited, so what passes here is what a peer would send. transactionId, messageId and timestamp are filled fresh on every send by the collection pre-request script.\n\nEverything goes to ONE adapter, network-adapter (quick-start default, `make up-core`, http://127.0.0.1:9200), which runs the consumer, network and provider tiers as modules on one port:\n\n / consumer module (discover, select)\n /publish catalogue publishing\n /_internal/network/ network module (internal)\n /_internal/provider/ provider module (internal)\n\nPublish enters /publish; discover and select enter the consumer module at /, which takes unsigned requests because an app sits inside the trust boundary. Tier-to-tier hops stay inside the container (localhost:9200). For the three-adapter stack (`make up-core-multi`) use OpenAgriNet.api-collection.json instead." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// Fresh ids and a fresh timestamp on every send.", - "//", - "// messageId MUST be unique per request: discovery rejects a replay, and PoCRA", - "// caches its answer on this value for ten minutes while ignoring location --", - "// so a reused id silently returns the previous area's facilities.", - "const uuid = () => pm.variables.replaceIn('{{$guid}}');", - "", - "pm.collectionVariables.set('transactionId', uuid());", - "pm.collectionVariables.set('messageId', uuid());", - "pm.collectionVariables.set('timestamp', new Date().toISOString());" - ] - } - } - ], - "variable": [ - { - "key": "transactionId", - "value": "", - "type": "string" - }, - { - "key": "messageId", - "value": "", - "type": "string" - }, - { - "key": "timestamp", - "value": "", - "type": "string" - } - ], - "item": [ - { - "name": "MandiPrice Capability - MH", - "description": "273 Maharashtra markets from Agmarknet. Select fetches live prices, so it needs egress.", - "item": [ - { - "name": "Publish - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/publish", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "publish" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:mandi-price:MH\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MH\",\n \"name\": \"Maharashtra mandi prices\",\n \"shortDesc\": \"On-demand commodity price information for 2 agricultural markets in Maharashtra.\",\n \"longDesc\": \"Each Resource represents one market and advertises the commodities it trades. Consumers discover a market by location or commodity and invoke the Provider to obtain current prices as a Direct MandiPrice Resource.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\",\n \"shortDesc\": \"Provider of live agricultural market price information.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MARKET_1634\",\n \"name\": \"Rahuri APMC mandi prices\",\n \"shortDesc\": \"Current commodity prices for Rahuri APMC, Ahmednagar district, Maharashtra.\",\n \"longDesc\": \"Invoke this Resource with a commodity and applicable date to obtain minimum, maximum, and modal prices for Rahuri APMC. The result uses the same MandiPrice type in Direct information mode.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"agmarknet\",\n \"sourceName\": \"Agmarknet Vistaar\",\n \"sourceUri\": \"https://agmarknet.gov.in/\"\n },\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\",\n \"marketCode\": \"1634\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.66192086448265,\n 19.644527591487904\n ]\n }\n }\n },\n \"supportedCommodities\": [\n {\n \"code\": \"4\",\n \"name\": \"Maize\"\n },\n {\n \"code\": \"13\",\n \"name\": \"Soyabean\"\n },\n {\n \"code\": \"18\",\n \"name\": \"Orange\"\n },\n {\n \"code\": \"20\",\n \"name\": \"Mango\"\n },\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n },\n {\n \"code\": \"24\",\n \"name\": \"Potato\"\n },\n {\n \"code\": \"25\",\n \"name\": \"Garlic\"\n },\n {\n \"code\": \"28\",\n \"name\": \"Bajra(Pearl Millet/Cumbu)\"\n },\n {\n \"code\": \"31\",\n \"name\": \"Cauliflower\"\n },\n {\n \"code\": \"32\",\n \"name\": \"Brinjal\"\n },\n {\n \"code\": \"45\",\n \"name\": \"Arhar(Tur/Red Gram)(Whole)\"\n },\n {\n \"code\": \"46\",\n \"name\": \"Green Peas\"\n },\n {\n \"code\": \"59\",\n \"name\": \"Papaya\"\n },\n {\n \"code\": \"62\",\n \"name\": \"Guar\"\n },\n {\n \"code\": \"64\",\n \"name\": \"Mousambi(Sweet Lime)\"\n },\n {\n \"code\": \"65\",\n \"name\": \"Tomato\"\n },\n {\n \"code\": \"67\",\n \"name\": \"Bitter gourd\"\n },\n {\n \"code\": \"68\",\n \"name\": \"Bottle gourd\"\n },\n {\n \"code\": \"71\",\n \"name\": \"Bhindi(Ladies Finger)\"\n },\n {\n \"code\": \"73\",\n \"name\": \"Green Chilli\"\n },\n {\n \"code\": \"74\",\n \"name\": \"Chilly Capsicum\"\n },\n {\n \"code\": \"87\",\n \"name\": \"Ginger(Green)\"\n },\n {\n \"code\": \"124\",\n \"name\": \"Sweet Potato\"\n },\n {\n \"code\": \"126\",\n \"name\": \"Cabbage\"\n },\n {\n \"code\": \"131\",\n \"name\": \"Cucumbar(Kheera)\"\n },\n {\n \"code\": \"132\",\n \"name\": \"Ridgeguard(Tori)\"\n },\n {\n \"code\": \"140\",\n \"name\": \"Drumstick\"\n },\n {\n \"code\": \"145\",\n \"name\": \"Sweet Pumpkin\"\n },\n {\n \"code\": \"151\",\n \"name\": \"Lime\"\n },\n {\n \"code\": \"156\",\n \"name\": \"Guava\"\n },\n {\n \"code\": \"160\",\n \"name\": \"Pomegranate\"\n },\n {\n \"code\": \"162\",\n \"name\": \"Seetapal\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-16T23:59:59+05:30\"\n },\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.66192086448265,\n 19.644527591487904\n ]\n },\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n }\n ]\n }\n },\n {\n \"id\": \"resource:mandi-price:market:164\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MARKET_164\",\n \"name\": \"Pune APMC mandi prices\",\n \"shortDesc\": \"Current commodity prices for Pune APMC, Pune district, Maharashtra.\",\n \"longDesc\": \"Invoke this Resource with a commodity and applicable date to obtain minimum, maximum, and modal prices for Pune APMC. The result uses the same MandiPrice type in Direct information mode.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"agmarknet\",\n \"sourceName\": \"Agmarknet Vistaar\",\n \"sourceUri\": \"https://agmarknet.gov.in/\"\n },\n \"market\": {\n \"marketName\": \"Pune APMC\",\n \"district\": \"364\",\n \"state\": \"MH\",\n \"marketCode\": \"164\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.86850558,\n 18.49089838\n ]\n }\n }\n },\n \"supportedCommodities\": [\n {\n \"code\": \"3\",\n \"name\": \"Rice\"\n },\n {\n \"code\": \"4\",\n \"name\": \"Maize\"\n },\n {\n \"code\": \"5\",\n \"name\": \"Jowar(Sorghum)\"\n },\n {\n \"code\": \"8\",\n \"name\": \"Black Gram(Urd Beans)(Whole)\"\n },\n {\n \"code\": \"9\",\n \"name\": \"Green Gram(Moong)(Whole)\"\n },\n {\n \"code\": \"18\",\n \"name\": \"Orange\"\n },\n {\n \"code\": \"19\",\n \"name\": \"Banana\"\n },\n {\n \"code\": \"21\",\n \"name\": \"Pineapple\"\n },\n {\n \"code\": \"22\",\n \"name\": \"Grapes\"\n },\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n },\n {\n \"code\": \"24\",\n \"name\": \"Potato\"\n },\n {\n \"code\": \"25\",\n \"name\": \"Garlic\"\n },\n {\n \"code\": \"28\",\n \"name\": \"Bajra(Pearl Millet/Cumbu)\"\n },\n {\n \"code\": \"30\",\n \"name\": \"Ragi(Finger Millet)\"\n },\n {\n \"code\": \"31\",\n \"name\": \"Cauliflower\"\n },\n {\n \"code\": \"32\",\n \"name\": \"Brinjal\"\n },\n {\n \"code\": \"39\",\n \"name\": \"Coriander(Leaves)\"\n },\n {\n \"code\": \"42\",\n \"name\": \"Methi(Leaves)\"\n },\n {\n \"code\": \"46\",\n \"name\": \"Green Peas\"\n },\n {\n \"code\": \"52\",\n \"name\": \"Lentil(Masur)(Whole)\"\n },\n {\n \"code\": \"58\",\n \"name\": \"Chikoos(Sapota)\"\n },\n {\n \"code\": \"59\",\n \"name\": \"Papaya\"\n },\n {\n \"code\": \"60\",\n \"name\": \"Water Melon\"\n },\n {\n \"code\": \"61\",\n \"name\": \"Gur(Jaggery)\"\n },\n {\n \"code\": \"62\",\n \"name\": \"Guar\"\n },\n {\n \"code\": \"64\",\n \"name\": \"Mousambi(Sweet Lime)\"\n },\n {\n \"code\": \"65\",\n \"name\": \"Tomato\"\n },\n {\n \"code\": \"67\",\n \"name\": \"Bitter gourd\"\n },\n {\n \"code\": \"68\",\n \"name\": \"Bottle gourd\"\n },\n {\n \"code\": \"70\",\n \"name\": \"Pumpkin\"\n },\n {\n \"code\": \"71\",\n \"name\": \"Bhindi(Ladies Finger)\"\n },\n {\n \"code\": \"73\",\n \"name\": \"Green Chilli\"\n },\n {\n \"code\": \"74\",\n \"name\": \"Chilly Capsicum\"\n },\n {\n \"code\": \"80\",\n \"name\": \"Beans\"\n },\n {\n \"code\": \"87\",\n \"name\": \"Ginger(Green)\"\n },\n {\n \"code\": \"92\",\n \"name\": \"Corriander seed\"\n },\n {\n \"code\": \"124\",\n \"name\": \"Sweet Potato\"\n },\n {\n \"code\": \"126\",\n \"name\": \"Cabbage\"\n },\n {\n \"code\": \"128\",\n \"name\": \"Snakeguard\"\n },\n {\n \"code\": \"129\",\n \"name\": \"Beetroot\"\n },\n {\n \"code\": \"131\",\n \"name\": \"Cucumbar(Kheera)\"\n },\n {\n \"code\": \"132\",\n \"name\": \"Ridgeguard(Tori)\"\n },\n {\n \"code\": \"133\",\n \"name\": \"Raddish\"\n },\n {\n \"code\": \"140\",\n \"name\": \"Drumstick\"\n },\n {\n \"code\": \"144\",\n \"name\": \"Mango(Raw-Ripe)\"\n },\n {\n \"code\": \"145\",\n \"name\": \"Sweet Pumpkin\"\n },\n {\n \"code\": \"146\",\n \"name\": \"Peas Wet\"\n },\n {\n \"code\": \"151\",\n \"name\": \"Lime\"\n },\n {\n \"code\": \"153\",\n \"name\": \"Jack Fruit\"\n },\n {\n \"code\": \"155\",\n \"name\": \"Jamun(Narale Hannu)\"\n },\n {\n \"code\": \"156\",\n \"name\": \"Guava\"\n },\n {\n \"code\": \"157\",\n \"name\": \"Karbuja(Musk Melon)\"\n },\n {\n \"code\": \"160\",\n \"name\": \"Pomegranate\"\n },\n {\n \"code\": \"161\",\n \"name\": \"Tender Coconut\"\n },\n {\n \"code\": \"162\",\n \"name\": \"Seetapal\"\n },\n {\n \"code\": \"186\",\n \"name\": \"Rose(Loose))\"\n },\n {\n \"code\": \"187\",\n \"name\": \"Jasmine\"\n },\n {\n \"code\": \"188\",\n \"name\": \"Kakada\"\n },\n {\n \"code\": \"189\",\n \"name\": \"Chrysanthemum(Loose)\"\n },\n {\n \"code\": \"190\",\n \"name\": \"Astera\"\n },\n {\n \"code\": \"193\",\n \"name\": \"Marigold(Calcutta)\"\n },\n {\n \"code\": \"205\",\n \"name\": \"Rajgir\"\n },\n {\n \"code\": \"222\",\n \"name\": \"Ground Nut Seed\"\n },\n {\n \"code\": \"247\",\n \"name\": \"Elephant Yam(Suran)\"\n },\n {\n \"code\": \"253\",\n \"name\": \"Little gourd(Kundru)\"\n },\n {\n \"code\": \"254\",\n \"name\": \"Pointed gourd(Parval)\"\n },\n {\n \"code\": \"279\",\n \"name\": \"Plum\"\n },\n {\n \"code\": \"280\",\n \"name\": \"Pear(Marasebu)\"\n },\n {\n \"code\": \"281\",\n \"name\": \"Peach\"\n },\n {\n \"code\": \"303\",\n \"name\": \"Amla(Nelli Kai)\"\n },\n {\n \"code\": \"305\",\n \"name\": \"Onion Green\"\n },\n {\n \"code\": \"307\",\n \"name\": \"Mint(Pudina)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-16T23:59:59+05:30\"\n },\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.86850558,\n 18.49089838\n ]\n },\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:mandi-price:MH\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/MandiPrice/publish/ondemand-mh.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Discover - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/discover", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "discover" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\" && @.resourceAttributes.supportedCommodities[*].code == \\\"23\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/MandiPrice/discover/discover.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Select", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/select", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "select" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\"\n },\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-09T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mandi-price:open-data\",\n \"resourceIds\": [\n \"resource:mandi-price:market:1634\"\n ],\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/MandiPrice/select/select.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "WeatherObservation Capability - MausamGram", - "description": "IMD MausamGram daily forecast. Select needs egress to mausamgram.imd.gov.in.", - "item": [ - { - "name": "Publish - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/publish", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "publish" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:weather-observation:mausamgram\",\n \"descriptor\": {\n \"code\": \"WEATHER_MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram daily weather forecast\",\n \"shortDesc\": \"On-demand five-day daily weather forecast for any point in the IMD numerical weather prediction domain.\",\n \"longDesc\": \"One Resource, not one per place. The Provider forecasts on a 0.25 degree grid and accepts any coordinate inside its domain, snapping it to the nearest grid centre, so there is no enumerable list of locations to publish. Consumers discover this Resource by falling inside its coverage area and then invoke the Provider with the point they care about to obtain a Direct WeatherObservation Resource per forecast day.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\",\n \"shortDesc\": \"Provider of IMD numerical weather prediction point forecasts.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"descriptor\": {\n \"code\": \"WEATHER_MAUSAMGRAM_DAILY_FORECAST\",\n \"name\": \"Five-day daily point weather forecast\",\n \"shortDesc\": \"Daily rainfall, temperature, humidity, wind and warning forecast for a requested coordinate.\",\n \"longDesc\": \"Invoke this Resource with a Point location to obtain one Direct WeatherObservation Resource per forecast day, each carrying that day's parameters. Values are bias-corrected model output, not station observations.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"source\": {\n \"sourceId\": \"mausamgram\",\n \"sourceName\": \"IMD Mausamgram NWP\",\n \"sourceUri\": \"https://mausamgram.imd.gov.in/\"\n },\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"P1D\",\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.5\n ],\n [\n 97.5,\n 6.5\n ],\n [\n 97.5,\n 37.5\n ],\n [\n 68.0,\n 37.5\n ],\n [\n 68.0,\n 6.5\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/WeatherObservation/publish/ondemand-mausamgram.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Discover - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/discover", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "discover" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Weather\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/WeatherObservation/discover/discover.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Select", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/select", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "select" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:weather-observation:mausamgram:open-data\",\n \"resourceIds\": [\n \"resource:weather-observation:mausamgram:daily-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/WeatherObservation/select/select.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "KnowledgeAdvisory Capability - Bharat Vistaar", - "description": "Retrieval over the Bharat Vistaar corpus. A select answer is large -- four passages measured 15.7 KB.", - "item": [ - { - "name": "Publish - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/publish", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "publish" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:knowledge-advisory:knowledge\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_ADVISORY\",\n \"name\": \"OpenAgriNet knowledge advisory retrieval\",\n \"shortDesc\": \"On-demand retrieval of agricultural advisory passages from a curated document corpus.\",\n \"longDesc\": \"One Resource, not one per document. The Provider runs a retrieval query over an ingested corpus of extension literature, scheme documents and package-of-practice material, so there is no enumerable list of advisories to publish. Consumers discover this Resource and then invoke the Provider with the topics they are asking about, receiving one Direct KnowledgeAdvisory Resource per retrieved passage, each carrying the passage text as its recommendation and naming the document it came from as its source.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\",\n \"shortDesc\": \"Provider of retrieval over curated agricultural knowledge documents.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_ADVISORY_RETRIEVAL\",\n \"name\": \"Agricultural advisory retrieval\",\n \"shortDesc\": \"Ask a question about crops, practices, schemes, weather or markets and receive the relevant advisory passages.\",\n \"longDesc\": \"Invoke this Resource with the topics being asked about to obtain one Direct KnowledgeAdvisory Resource per retrieved passage. A request whose subjectCategories name Scheme and nothing else is answered from the narrower schemes corpus; any other combination is answered from the Provider's default corpus, because the request no longer states a question only the schemes corpus can answer.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\",\n \"Livestock\",\n \"Scheme\",\n \"Practice\",\n \"Weather\",\n \"Market\"\n ],\n \"languages\": [\n \"en\",\n \"hi\",\n \"gu\"\n ],\n \"topics\": [\n \"Crop establishment\",\n \"Nutrient management\",\n \"Pest management\",\n \"Government schemes\",\n \"Package of practices\"\n ],\n \"source\": {\n \"sourceId\": \"knowledge-corpus\",\n \"sourceName\": \"OpenAgriNet curated knowledge corpus\",\n \"sourceUri\": \"https://openagrinet.github.io/network-specs/\"\n },\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.5\n ],\n [\n 97.5,\n 6.5\n ],\n [\n 97.5,\n 37.5\n ],\n [\n 68.0,\n 37.5\n ],\n [\n 68.0,\n 6.5\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/KnowledgeAdvisory/publish/ondemand-knowledge.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Discover - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/discover", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "discover" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Crop\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/KnowledgeAdvisory/discover/discover.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Select", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/select", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "select" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"topics\": [\n \"cotton\",\n \"pink bollworm\",\n \"pest management\"\n ]\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:knowledge-advisory:knowledge:open-data\",\n \"resourceIds\": [\n \"resource:knowledge-advisory:knowledge:retrieval\"\n ],\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/KnowledgeAdvisory/select/select.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "AgricultureFacility Capability - PoCRA", - "description": "PoCRA facility locator, the one capability needing no credential. It takes ONE facility type per call and caches on messageId for ten minutes while ignoring location.", - "item": [ - { - "name": "Publish - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/publish", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "publish" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:agriculture-facility:pocra\",\n \"descriptor\": {\n \"code\": \"AGRI_FACILITY_POCRA\",\n \"name\": \"PoCRA agriculture facility locator\",\n \"shortDesc\": \"On-demand search for custom hiring centres, Krishi Vigyan Kendras, warehouses and soil testing laboratories across Maharashtra.\",\n \"longDesc\": \"One Resource, not one per facility. PoCRA aggregates facility registries from several Maharashtra departments and answers a search around a point, ranking results by distance, so there is no enumerable list of facilities to publish. Consumers discover this Resource by falling inside its coverage area and then invoke the Provider with a search origin and the facility types they want, receiving one Direct AgricultureFacility Resource per facility found.\"\n },\n \"isActive\": true,\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\",\n \"shortDesc\": \"Maharashtra state aggregator for agricultural facility registries.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"descriptor\": {\n \"code\": \"AGRI_FACILITY_POCRA_SEARCH\",\n \"name\": \"Nearby agriculture facility search\",\n \"shortDesc\": \"Find custom hiring centres, Krishi Vigyan Kendras, warehouses and soil testing laboratories near a point.\",\n \"longDesc\": \"Invoke this Resource with a fulfillment stop carrying a Point location and the facility types wanted to obtain one Direct AgricultureFacility Resource per facility, each carrying its address, contact and, for warehouses, its reported storage capacity. Results are ranked by the Provider's own distance from the search origin; that distance is not carried on the Resource, because it is a property of the query rather than of the facility.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"languages\": [\n \"en\",\n \"mr\"\n ],\n \"source\": {\n \"sourceId\": \"pocra\",\n \"sourceName\": \"PoCRA, Government of Maharashtra\",\n \"sourceUri\": \"https://mahapocra.gov.in/\"\n },\n \"supportedFacilityTypes\": [\n \"CustomHiringCentre\",\n \"KrishiVigyanKendra\",\n \"Warehouse\",\n \"SoilTestingFacility\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-2\",\n \"areaCode\": \"IN-MH\",\n \"areaLevel\": \"State\",\n \"areaName\": \"Maharashtra\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 72.6,\n 15.6\n ],\n [\n 80.9,\n 15.6\n ],\n [\n 80.9,\n 22.1\n ],\n [\n 72.6,\n 22.1\n ],\n [\n 72.6,\n 15.6\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/AgricultureFacility/publish/ondemand-pocra.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Discover - On-demand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/discover", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "discover" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"discover\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Facility\\\" && @.resourceAttributes.informationMode == \\\"OnDemand\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_INTERSECTS\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/AgricultureFacility/discover/discover.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - }, - { - "name": "Select", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{network-adapter-host}}/select", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "select" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"supportedFacilityTypes\": [\n \"Warehouse\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agriculture-facility:pocra:open-data\",\n \"resourceIds\": [\n \"resource:agriculture-facility:pocra:facility-search\"\n ],\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "Body is capability-examples/AgricultureFacility/select/select.json, unedited apart from the context variables." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));", - "", - "// A fault comes back inside a 200 as often as it comes back as a 4xx, so the", - "// status code alone does not tell you the call worked.", - "pm.test('not a NACK', () => {", - " const b = pm.response.json();", - " const err = (b.error) || (b.message && b.message.error);", - " if (err) throw new Error(`${err.code}: ${err.message}`);", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "Stack Health", - "description": "Run these first. Every one is a plain GET and none of them signs anything.", - "item": [ - { - "name": "Network Adapter", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{network-adapter-host}}/health", - "host": [ - "{{network-adapter-host}}" - ], - "path": [ - "health" - ] - }, - "description": "One adapter process serves all three tiers, so there is one adapter health check. /health is registered by the adapter itself, outside the modules." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));" - ] - } - } - ] - }, - { - "name": "discovery", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{discovery-host}}/healthz", - "host": [ - "{{discovery-host}}" - ], - "path": [ - "healthz" - ] - } - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('HTTP 200', () => pm.response.to.have.status(200));" - ] - } - } - ] - } - ] - } - ] -} diff --git a/api-collection/OpenAgriNet.network-adapter.local-environment.json b/api-collection/OpenAgriNet.network-adapter.local-environment.json index bc72f824..365004ed 100644 --- a/api-collection/OpenAgriNet.network-adapter.local-environment.json +++ b/api-collection/OpenAgriNet.network-adapter.local-environment.json @@ -1,11 +1,27 @@ { - "id": "0a4e1c00-0000-4000-8000-0000000snle1", - "name": "OpenAgriNet local (Network Adapter Layer)", + "id": "oan-local-environment-single", + "name": "OpenAgriNet local (single mode)", "values": [ + { + "key": "consumer-adapter-host", + "value": "http://127.0.0.1:9200", + "enabled": true, + "type": "default", + "description": "Where discover and select enter. Single mode: the one adapter, so all three adapter hosts are the same URL. NETWORK_ADAPTER_LAYER_PORT in .env." + }, + { + "key": "provider-adapter-host", + "value": "http://127.0.0.1:9200", + "enabled": true, + "type": "default", + "description": "Where publish enters. Single mode: the same adapter as consumer-adapter-host. NETWORK_ADAPTER_LAYER_PORT in .env." + }, { "key": "network-adapter-host", "value": "http://127.0.0.1:9200", - "enabled": true + "enabled": true, + "type": "default", + "description": "Used only by the health check. Single mode: the same adapter; its network tier sits under /_internal/ and is not called from Postman. NETWORK_ADAPTER_LAYER_PORT in .env." }, { "key": "discovery-host", diff --git a/quick-start/CHANGELOG.md b/quick-start/CHANGELOG.md index a5b00691..b8d823ee 100644 --- a/quick-start/CHANGELOG.md +++ b/quick-start/CHANGELOG.md @@ -28,8 +28,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - `.env.example` gains `NETWORK_ADAPTER_LAYER_PORT=9200` and `COMPOSE_PROFILES=network`, so a plain `docker compose up -d` starts single mode. -- API collection for the single adapter: - `api-collection/OpenAgriNet.network-adapter.api-collection.json`. +- Postman environment for single mode: + `api-collection/OpenAgriNet.network-adapter.local-environment.json`. It + runs the same `OpenAgriNet.api-collection.json` as multi mode, with every + adapter host on `:9200`. - `SETUP.md`: install and run steps, local and VM, for both modes. The README now covers the modes, how to switch, and the reference appendices. diff --git a/quick-start/README.md b/quick-start/README.md index 8bd65c25..a3480f93 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -84,8 +84,9 @@ docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' networ # network-adapter-layer = single, network-adapter = multi ``` -Use the API collection for the running mode (SETUP.md Step 6): -`OpenAgriNet.network-adapter.*` for single, `OpenAgriNet.*` for multi. +One API collection serves both modes; pick the environment for the running +mode (SETUP.md Step 6): `OpenAgriNet.network-adapter.local-environment.json` +for single, `OpenAgriNet.local-environment.json` for multi. ## The stack in one picture diff --git a/quick-start/SETUP.md b/quick-start/SETUP.md index f0dd58e8..d13d67e1 100644 --- a/quick-start/SETUP.md +++ b/quick-start/SETUP.md @@ -153,19 +153,21 @@ which differs: `make logs SVC=network-adapter-layer` in single mode, ## Step 6 — Verify end to end -Run the API collection in `../api-collection/`, one copy per mode: +Run the API collection in `../api-collection/`. One collection serves both +modes, since the paths are the same; the environment picks the mode: | Mode | Collection | Environment | |---|---|---| -| single (default) | `OpenAgriNet.network-adapter.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` | +| single (default) | `OpenAgriNet.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` (every adapter host is `:9200`) | | multi | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | -Import the pair for your mode into Postman and select the environment, or -every `{{...-host}}` resolves to nothing. Or run it with newman: +Import the collection and the environment for your mode into Postman and +select the environment, or every `{{...-host}}` resolves to nothing. Or run it +with newman: ```sh # single mode -newman run ../api-collection/OpenAgriNet.network-adapter.api-collection.json \ +newman run ../api-collection/OpenAgriNet.api-collection.json \ -e ../api-collection/OpenAgriNet.network-adapter.local-environment.json # multi mode From b4651ab6353f7ab6ca227b8d8e77c0bb1250f9c8 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 10:07:20 +0530 Subject: [PATCH 30/43] chore: remove unused CI values file and update HPA configuration #55 #66 --- .../ci/network-adapter-ingress-values.yaml | 44 --- charts/adapter-service/templates/hpa.yaml | 4 - quick-start/CHANGELOG.md | 65 ---- quick-start/SETUP.md | 309 ------------------ 4 files changed, 422 deletions(-) delete mode 100644 charts/adapter-service/ci/network-adapter-ingress-values.yaml delete mode 100644 quick-start/CHANGELOG.md delete mode 100644 quick-start/SETUP.md diff --git a/charts/adapter-service/ci/network-adapter-ingress-values.yaml b/charts/adapter-service/ci/network-adapter-ingress-values.yaml deleted file mode 100644 index 61856fee..00000000 --- a/charts/adapter-service/ci/network-adapter-ingress-values.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# charts/adapter-service/ci/network-adapter-ingress-values.yaml -# CI only (scripts/lint-charts.sh lints and renders every ci/*-values.yaml). -# Not for installs: use examples/network-adapter.yaml. -# -# Renders role network-adapter-layer with the Ingress on, so the path guard in -# templates/ingress.yaml runs in CI: the allowed path shape (one-segment -# /, Exact and Prefix) must render. ingress.annotations -# is null on purpose, to cover the `| default dict` on the annotations merge. -role: network-adapter-layer -fullnameOverride: network-adapter - -image: - registry: ghcr.io - repository: openagrinet/network-adapter - tag: "v1.9.0" - -service: - port: 9200 - targetPort: 9200 - -networkAdapterLayer: - keys: - consumer: - existingSecret: consumer-adapter-keys - network: - existingSecret: network-adapter-keys - provider: - existingSecret: provider-adapter-keys - extraSubstitutions: - __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl - -ingress: - enabled: true - className: nginx - annotations: null - hosts: - - host: network-adapter.ci.example.com - paths: - - path: /discover - pathType: Exact - - path: /select - pathType: Prefix - - path: /status - pathType: Exact diff --git a/charts/adapter-service/templates/hpa.yaml b/charts/adapter-service/templates/hpa.yaml index 0720c2a3..6f696997 100644 --- a/charts/adapter-service/templates/hpa.yaml +++ b/charts/adapter-service/templates/hpa.yaml @@ -29,8 +29,4 @@ spec: type: Utilization averageUtilization: {{ . }} {{- end }} - {{- with .Values.autoscaling.behavior }} - behavior: - {{- toYaml . | nindent 4 }} - {{- end }} {{- end }} diff --git a/quick-start/CHANGELOG.md b/quick-start/CHANGELOG.md deleted file mode 100644 index b8d823ee..00000000 --- a/quick-start/CHANGELOG.md +++ /dev/null @@ -1,65 +0,0 @@ -# Changelog - -All notable changes to the quick-start stack are documented here. - -The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - -## [0.2.0] - 2026-10-14 - -### Added -- **Single network adapter, now the default mode.** One adapter process runs - the consumer, network and provider tiers as modules on one port: - - service `network-adapter-layer`, container `network-adapter`, compose - profile `network`; - - default port `127.0.0.1:9200`, set with `NETWORK_ADAPTER_LAYER_PORT`; - - paths `/` (consumer) and `/publish`, the same as multi mode; the - network and provider tiers under `/_internal/`, which only the process - itself calls; - - `server_proxy.conf` now denies `/publish` (it was commented out) and - `/_internal` on every proxy host; - - config `config/adapters/network-adapter.yaml`, rendered by - `bin/setup.py` from `network-adapter.yaml.tmpl`; it uses the same keys - and registry rows as the three tier adapters. -- **Multi mode stays available as an option.** The three tier adapters - (`provider-adapter` :9200, `network-adapter` :9201, `consumer-adapter` - :9202) are now under compose profile `multi`. -- Make targets: `make up-multi` and `make up-core-multi` start multi mode. -- `bin/stack.sh` takes `MODE=single|multi` (default `single`). -- `.env.example` gains `NETWORK_ADAPTER_LAYER_PORT=9200` and - `COMPOSE_PROFILES=network`, so a plain `docker compose up -d` starts single - mode. -- Postman environment for single mode: - `api-collection/OpenAgriNet.network-adapter.local-environment.json`. It - runs the same `OpenAgriNet.api-collection.json` as multi mode, with every - adapter host on `:9200`. - -- `SETUP.md`: install and run steps, local and VM, for both modes. The - README now covers the modes, how to switch, and the reference appendices. - -### Changed -- `make up` and `make up-core` start single mode instead of the three tier - adapters. -- Only one mode runs at a time, since both use port 9200. Switching mode - removes the other mode's adapter containers. -- `make restart` restarts the adapters of whichever mode is running. - -### Upgrade notes -- Re-run `python3 bin/setup.py` once to render `network-adapter.yaml`. -- To keep the old three-adapter layout, use `make up-multi`, or set - `COMPOSE_PROFILES=multi` for plain compose commands. - -## [0.1.0] - 2026-09-07 - -### Added -- Initial quick-start stack with Docker Compose: - - registry (Sunbird RC) with its Postgres and Keycloak; - - discovery service with its Postgres; - - three adapters, one per tier: provider (port 9200), network (9201) and - consumer (9202); - - two mock upstreams standing in for Mausamgram and Agmarknet; - - optional profiles `reverse-proxy` (nginx-proxy-manager) and - `observability` (hyperdx). -- `bin/setup.py`: generates keys, seeds the registry and renders the adapter - configs. -- `make up` starts the whole stack in order; `make up-core` starts it without - the reverse proxy and observability. diff --git a/quick-start/SETUP.md b/quick-start/SETUP.md deleted file mode 100644 index d13d67e1..00000000 --- a/quick-start/SETUP.md +++ /dev/null @@ -1,309 +0,0 @@ -# Quick-start setup - -Install and run the OAN stack with Docker Compose. What the stack is and the -two adapter modes are in [README.md](README.md); this file is the steps. - -Nothing is built here — the images are pulled. Locally that is about ten -minutes, most of it waiting for Keycloak. - -| | Local | VM | -|----------------------|--------------------------|-----------------------------| -| Reached over | `localhost` | a public hostname, TLS | -| Public edge (Nginx) | not started | started, on 80 and 443 | -| Observability | not started | optional, wants 2–4 GB | -| Credentials | shipped defaults are ok | **must all be changed** | -| Start, single mode (default) | `make up-core` | `make up` | -| Start, multi mode | `make up-core-multi` | `make up-multi` | - -**Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. - ---- - -# Part 1 — Run it locally - -## Step 1 — Prerequisites - -- `git` -- Docker with **Compose v2.20 or later** — `docker compose version` must work, not `docker-compose`; the compose file uses `depends_on: required: false`, which older v2 releases reject -- `python3` with `cryptography` — `pip install cryptography` -- `curl` - -## Step 2 — Get the repo - -```sh -git clone -b development https://github.com/OpenAgriNet/helmcharts.git -cd helmcharts/quick-start -``` - -**Every command below runs from `quick-start/`** — it is where the compose file -and the Makefile live. - -## Step 3 — Configure - -```sh -cp .env.example .env -``` - -Locally, nothing in it needs changing. It publishes these on `localhost`: - -``` -8081 registry 9100 mock-imd -8080 keycloak 9101 mock-agmarknet -9990 keycloak admin -8090 discovery - -single mode (default) 9200 network adapter (all three tiers) -multi mode 9200 provider adapter - 9201 network adapter - 9202 consumer adapter -``` - -If a port is already taken, change it in `.env` — that is the only edit a -local run needs. The single adapter's host port is -`NETWORK_ADAPTER_LAYER_PORT`; in multi mode the three are -`PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT` and `CONSUMER_ADAPTER_PORT`. -Adapters reach each other inside the stack (by Compose service name in multi -mode, on `localhost:9200` inside the one container in single mode), so a port -change only changes what you type into Postman. - -## Step 4 — Start it - -Single mode (the default): - -```sh -make up-core -``` - -Or multi mode, three adapters: - -```sh -make up-core-multi -``` - -Either one runs the steps in the only order that works: registry and -discovery, then `bin/setup.py`, then the adapters. Allow up to five minutes -the first time — Keycloak on a cold volume. - -`setup.py` generates a keypair per tier, registers seven participants and four -capability bindings, and renders all four adapter configs (the three tier -configs and the single adapter's). Nothing needs creating by hand. → -README Appendix C for why the order matters, Appendix D for what it wrote. - -Switching mode later is running the other target; see -[README — How to switch](README.md#how-to-switch). - -```sh -make ps -``` - -## Step 5 — The three tiers and their `.env` keys - -Both modes run the same three tiers with the same keys; single mode runs them -as modules of one container. `setup.py` renders every adapter config from -these keys. **Do not edit the rendered `config/adapters/*.yaml`** — they are -regenerated and hold private keys. Change `.env`, re-run `make up-core` (or -`make up-core-multi`). - -**Provider** — answers `select`, the only tier that calls an upstream. Serves -all four capabilities. - -- `PROVIDER_SUBSCRIBER_ID`, and the four pairs that become binding keys: - `MAUSAMGRAM_PARTICIPANT_ID` + `MAUSAMGRAM_CAPABILITY`, - `AGMARKNET_PARTICIPANT_ID` + `AGMARKNET_CAPABILITY`, - `VISTAAR_PARTICIPANT_ID` + `VISTAAR_CAPABILITY`, - `POCRA_PARTICIPANT_ID` + `POCRA_CAPABILITY`. -- Upstream credentials: `AGMARKNET_ACCESS_NAME` + `AGMARKNET_PASSWORD`, and - `VISTAAR_CLIENT_ID` + `VISTAAR_CLIENT_SECRET`. POCRA needs none. - -**All four capabilities call real upstreams.** Mausamgram, mandi, knowledge -and POCRA are external hosts, so each needs its own base URL and path in -`.env` — and the stack needs egress to all four. Mandi and knowledge each also need a token -endpoint, since both issue short-lived tokens rather than taking a static -credential. - -**Network** — fronts discovery: verifies the caller, passes `discover` and -`publish` on, and re-signs as itself. - -- `NETWORK_SUBSCRIBER_ID`. `BECKN_NETWORK_ID` belongs to the discovery service - behind it — a `discover` naming a different network finds nothing. - -**Consumer** — the caller's entry. Sends `discover` to the network tier and -`select` straight to the provider tier. - -- `CONSUMER_SUBSCRIBER_ID`. - -Check the adapters came up: - -```sh -# single mode: one container -docker logs network-adapter 2>&1 | grep -E 'Processor steps initialized|Server listening' - -# multi mode: one per tier -docker logs provider-adapter 2>&1 | grep 'Processor steps initialized' -docker logs network-adapter 2>&1 | grep 'Server listening' -docker logs consumer-adapter 2>&1 | grep 'Server listening' -``` - -All four capability steps should be listed by name. - -The container is called `network-adapter` in both modes, so `docker logs -network-adapter` works in either. Compose commands take the service name, -which differs: `make logs SVC=network-adapter-layer` in single mode, -`make logs SVC=network-adapter` in multi mode. - -## Step 6 — Verify end to end - -Run the API collection in `../api-collection/`. One collection serves both -modes, since the paths are the same; the environment picks the mode: - -| Mode | Collection | Environment | -|---|---|---| -| single (default) | `OpenAgriNet.api-collection.json` | `OpenAgriNet.network-adapter.local-environment.json` (every adapter host is `:9200`) | -| multi | `OpenAgriNet.api-collection.json` | `OpenAgriNet.local-environment.json` | - -Import the collection and the environment for your mode into Postman and -select the environment, or every `{{...-host}}` resolves to nothing. Or run it -with newman: - -```sh -# single mode -newman run ../api-collection/OpenAgriNet.api-collection.json \ - -e ../api-collection/OpenAgriNet.network-adapter.local-environment.json - -# multi mode -newman run ../api-collection/OpenAgriNet.api-collection.json \ - -e ../api-collection/OpenAgriNet.local-environment.json -``` - -One folder per capability plus a health folder. Each capability folder is -publish, then discover, then select, in that order — so run publish first. -Every request's `transactionId`, `messageId` and `timestamp` are filled fresh -by the collection's pre-request script; the rest of each body comes from -`capability-examples/`, unedited. - -Green means the registry is seeded, signatures verify both ways, all four -mappings work and discovery is indexing. - -POCRA is the one folder whose empty answer is LOGGED rather than failed: it -returns 200 with no providers both when nothing is nearby and when it is -rate-limiting, and the two are indistinguishable. Knowledge is also the one -folder that can fail on a missing credential rather than on wiring — it -authenticates with OAuth2 client credentials that only the adapter reads, at -call time. - -To point it at another deployment, edit the **environment** file, not the -collection. → README Appendix N. - ---- - -# Part 2 — Run it on a VM - -Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. - -## Step V1 — Prepare the VM - -Nothing is cloned yet, so fetch the script rather than running it from the repo: - -```sh -curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/development/quick-start/bin/bootstrap-ubuntu.sh | bash -``` - -Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt -repo, then adds you to the `docker` group — **log out and back in** for that to -take effect. Idempotent, and it deliberately does not clone, write `.env` or -start anything. - -**8 GB** runs the stack; **16 GB** for the observability tier. - -Then Part 1 Steps 2 and 3 as written. - -## Step V2 — Change every credential - -`.env.example` ships working defaults, which means they are public: - -``` -POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_ADMIN_CLIENT_SECRET -KEYCLOAK_DEFAULT_USER_PASSWORD -``` - -Change all four before the VM is reachable by anyone but you. Adapter keypairs -are the exception — `setup.py` generates those and never writes them to `.env`. - -## Step V3 — Start it - -Single mode (the default): - -```sh -make up -``` - -Or multi mode: `make up-multi`. - -`make up`, not `make up-core`: two more tiers on top of Part 1's — -nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes -the VM reachable from the internet.** The mode decides what the proxy hosts in -README Appendix B forward to. - -Switching mode on a running VM (`make up` ⇄ `make up-multi`) recreates the -adapters and restarts nginx-proxy-manager so it picks up the new addresses: -the public hosts drop connections for a few seconds. Repoint the proxy hosts -(Appendix B) in the same window. - -## Step V4 — Decide what is exposed - -Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written -literally in `docker-compose.yml` rather than taken from a variable. So the -registry, Keycloak and the databases are not publicly reachable — deliberately. -Signing is off in every quick-start adapter config, so the adapter ports must -stay on `127.0.0.1` too. - -Proxy hosts, certificates, and the `/publish` deny every host gets → -**README Appendix B**. Read it before pointing DNS at the box. - -## Step V5 — Reach the ports bound to 127.0.0.1 - -Single mode needs only the adapter's 9200, plus the registry and Keycloak: - -```sh -ssh -L 9200:127.0.0.1:9200 -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm -``` - -Multi mode adds the consumer's 9202 (`-L 9202:127.0.0.1:9202`). The -collections' defaults then work unchanged, since they already point at -`127.0.0.1`. - -## Step V6 — Observability (optional) - -```sh -make observability -``` - -HyperDX on `127.0.0.1:8085`, OTLP on 4317/4318 — and, once you add its proxy -host, on a public hostname behind an NPM Access List, which is how you read it -without a tunnel. Route + credential → **README Appendix B**. → Appendix H, -which is honest about how much actually arrives. - -Every container's stdout lands there too, next to the traces — so this is also -what replaces keeping `docker compose logs -f` open. How that is wired, and the -one gotcha, are in Appendix H. - -`docker compose restart nginx-proxy-manager` after this command, the first -time: NPM only resolves `hyperdx` on reload. - ---- - -## If something is wrong - -- **`unrecognized step: `** — `ADAPTER_IMAGE` predates the config. - README Appendix G. -- **404 `NET_ENTITY_NOT_FOUND`** — no provider step claimed the payload; a - binding key disagrees with `.env`. Appendix F. -- **404 naming a binding with no active record** — the step matched but the - registry has no `ProviderSchema` row for it. Appendix F. -- **502 from a `select`** — the upstream answered non-2xx. Appendix F. -- **NPM's default page, or a 502 that worked yesterday** — Appendix B. -- **An adapter config is a directory** — something started before - `setup.py`. Appendix F. - -Full set, with what to run for each → -[README Appendix F](README.md#appendix-f--when-it-does-not-work). From 5172ff7fbc910b3c831be460a5c9a23d039bef6f Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 10:07:32 +0530 Subject: [PATCH 31/43] refactor(adapter-service): update changelog, CI values, and README for unified single network adapter #66 #55 --- charts/adapter-service/CHANGELOG.md | 4 +- .../ci/network-adapter-values.yaml | 32 +- .../examples/network-adapter.yaml | 13 - charts/adapter-service/values.yaml | 2 - quick-start/README.md | 390 +++++++++++++----- 5 files changed, 318 insertions(+), 123 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 5901ff03..cb79252c 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -37,7 +37,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h role. - `examples/network-adapter.yaml` and `ci/network-adapter-values.yaml` (PDB, spread, graceful shutdown; HPA 3-10 opt-in, off by default). -- `autoscaling.behavior`, `topologySpreadConstraints`, +- `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. @@ -68,7 +68,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h values never reached the pod and NOTES reported the opposite of what it ran. The example and CI values no longer set them. - The Ingress annotations merge tolerates `ingress.annotations: null`. - `ci/network-adapter-ingress-values.yaml` renders the path guard in CI. + `ci/network-adapter-values.yaml` turns the Ingress on, so CI renders the path guard. ### Removed - Unused helpers `adapter-service.name`, `.appName`, `.otelServiceName`, diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 2e7ec458..0add389c 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -8,6 +8,11 @@ # settings, so those templates are linted too. The HPA stays off, as in # examples/network-adapter.yaml: autoscaling is opt-in. Pinned image tag so the render # is reproducible. +# +# The Ingress is on so the path guard in templates/ingress.yaml runs in CI: the +# allowed path shape (one-segment /, Exact and Prefix) must render. +# ingress.annotations is null on purpose, to cover the `| default dict` on the +# annotations merge. role: network-adapter-layer fullnameOverride: network-adapter @@ -55,19 +60,6 @@ autoscaling: minReplicas: 3 maxReplicas: 10 targetCPUUtilizationPercentage: 70 - behavior: - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 30 - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Pods - value: 1 - periodSeconds: 60 podDisruptionBudget: enabled: true minAvailable: 1 @@ -77,3 +69,17 @@ topologySpreadConstraints: whenUnsatisfiable: ScheduleAnyway terminationGracePeriodSeconds: 40 preStopSleepSeconds: 10 + +ingress: + enabled: true + className: nginx + annotations: null + hosts: + - host: network-adapter.ci.example.com + paths: + - path: /discover + pathType: Exact + - path: /select + pathType: Prefix + - path: /status + pathType: Exact diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index f449c5fb..2beb5da9 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -83,19 +83,6 @@ autoscaling: minReplicas: 3 # one per node with the spread constraint below maxReplicas: 10 targetCPUUtilizationPercentage: 70 - behavior: - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 30 - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Pods - value: 1 - periodSeconds: 60 podDisruptionBudget: enabled: true diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 2e60a2c0..feec2638 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -380,8 +380,6 @@ autoscaling: maxReplicas: 6 targetCPUUtilizationPercentage: 75 targetMemoryUtilizationPercentage: "" - # Passed through to the HPA as spec.behavior. {} = Kubernetes defaults. - behavior: {} podDisruptionBudget: enabled: false diff --git a/quick-start/README.md b/quick-start/README.md index a3480f93..fccfffbc 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -3,140 +3,346 @@ The whole OAN stack in Docker Compose: a registry, a discovery service, the network adapter, and two mock upstreams standing in for real provider APIs. -- **Install and run it:** [SETUP.md](SETUP.md) -- **What changed, by version:** [CHANGELOG.md](CHANGELOG.md) -- **TLS certificates for the VM edge:** [CERTIFICATES.md](CERTIFICATES.md) +The adapter runs in one of two modes, from the same compose file, keys and +registry. **Single is the default**: one container, `network-adapter`, runs the +consumer, network and provider tiers on port 9200. **Multi is optional**: the +three tiers as three containers. → [Two modes](#two-modes). -## Quick start +Nothing is built here — the images are pulled. Locally that is about ten +minutes, most of it waiting for Keycloak. Discovery is the one exception, and +only if you ask for it: see [Appendix H](#appendix-h--telemetry), which is also +where the reason lives. + +## The stack in one picture + +``` + consumer + │ + ▼ + consumer adapter the only one a consumer calls + │ + ├── discover ──► network adapter ──► discovery service + │ + └── select ──► provider adapter ──► mock-imd + └─► mock-agmarknet +``` + +`publish` runs the other way: the provider adapter sends it to the network +adapter, which files the catalogue in discovery. + +All three adapters read the registry — who signed this request, and where this +capability's provider lives. + +A `select` answers in the same HTTP round trip — there is no callback. + +The picture shows the three tiers. In single mode (the default) they are +modules of one container, and the arrows between them are calls inside it. + +That is the request order. **Startup order is the reverse**: in multi mode the +consumer adapter depends on the other two, so Compose brings them up first. The steps +below follow startup order, so they work top to bottom. + +## Which path are you on? + +| | Local | VM | +|----------------------|--------------------------|-----------------------------| +| Reached over | `localhost` | a public hostname, TLS | +| Public edge (Nginx) | not started | started, on 80 and 443 | +| Observability | not started | optional, wants 2–4 GB | +| Credentials | shipped defaults are ok | **must all be changed** | +| Command | `make up-core` | `make up` | +| Multi mode instead | `make up-core-multi` | `make up-multi` | + +**Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. + +--- + +# Part 1 — Run it locally + +## Step 1 — Prerequisites + +- `git` +- Docker with **Compose v2.20 or later** — `docker compose version` must work, not `docker-compose`; the compose file uses `depends_on: required: false`, which older v2 releases reject +- `python3` with `cryptography` — `pip install cryptography` +- `curl` + +## Step 2 — Get the repo + +```sh +git clone -b development https://github.com/OpenAgriNet/helmcharts.git +cd helmcharts/quick-start +``` + +**Every command below runs from `quick-start/`** — it is where the compose file +and the Makefile live. + +## Step 3 — Configure ```sh cp .env.example .env -make up-core # single mode (default): registry, discovery, setup, the network adapter on 9200 ``` -Prerequisites, the VM path and the checks are in [SETUP.md](SETUP.md). +Locally, nothing in it needs changing. It publishes these on `localhost`: + +``` +8081 registry 9200 network adapter (single) 9100 mock-imd +8080 keycloak or provider adapter (multi) 9101 mock-agmarknet +9990 keycloak admin 9201 network adapter (multi only) +8090 discovery 9202 consumer adapter (multi only) +``` + +If one is already taken, change it here — that is the only edit a local run +needs. The single adapter's port is `NETWORK_ADAPTER_LAYER_PORT`. Adapters reach each other by Compose service name, so it only changes +what you type into Postman. + +## Step 4 — Start it + +```sh +make up-core +``` + +Three tiers, in the only order that works: registry and discovery, then +`bin/setup.py`, then the adapter (`network-adapter` on 9200; `make up-core-multi` +starts the three adapters instead). Allow up to five minutes +the first time — Keycloak on a cold volume. + +`setup.py` generates a keypair per adapter, registers seven participants and four +capability bindings, and renders the adapter configs (the three tier configs and +the single adapter's `network-adapter.yaml`). Nothing needs +creating by hand. → Appendix C for why the order matters, Appendix D for what +it wrote. + +```sh +make ps +``` + +## Step 5 — Provider layer + +> Steps 5–7 describe the three tiers. In single mode they run in one container, +> so check them with `docker logs network-adapter` instead of the +> `docker compose logs -adapter` commands below. + +Answers `select`, and the only layer that calls an upstream. Serves all four +capabilities from one adapter. + +`.env` keys: `PROVIDER_SUBSCRIBER_ID`, and the four pairs that become binding +keys — `MAUSAMGRAM_PARTICIPANT_ID` + `MAUSAMGRAM_CAPABILITY`, +`AGMARKNET_PARTICIPANT_ID` + `AGMARKNET_CAPABILITY`, `VISTAAR_PARTICIPANT_ID` + +`VISTAAR_CAPABILITY`, `POCRA_PARTICIPANT_ID` + `POCRA_CAPABILITY`. + +Plus the credentials each upstream needs: `AGMARKNET_ACCESS_NAME` + +`AGMARKNET_PASSWORD`, and `VISTAAR_CLIENT_ID` + `VISTAAR_CLIENT_SECRET`. POCRA +needs none at all. + +**Only mausamgram is a mock now.** Mandi, knowledge and POCRA are real +external hosts, so each needs its own base URL and path in `.env` — and the +stack needs egress to all three. Mandi and knowledge each also need a token +endpoint, since both issue short-lived tokens rather than taking a static +credential. + +```sh +docker compose logs provider-adapter | grep 'Processor steps initialized' +``` + +All four capability steps should be listed by name. + +## Step 6 — Network layer + +Fronts discovery: verifies the caller, passes `discover` and `publish` on, and +re-signs as itself. + +`.env` keys: `NETWORK_SUBSCRIBER_ID`. `BECKN_NETWORK_ID` belongs to the discovery +service behind it — a `discover` naming a different network finds nothing. + +```sh +docker compose logs network-adapter | grep 'Server listening' +``` + +## Step 7 — Consumer layer + +The consumer's edge. Sends `discover` to the network layer and `select` +straight to the provider layer, per `config/adapters/routing-consumer.yaml`. + +`.env` keys: `CONSUMER_SUBSCRIBER_ID`. + +```sh +docker compose logs consumer-adapter | grep 'Server listening' +``` + +`setup.py` renders all the adapter configs from these keys. **Do not edit the +rendered `config/adapters/*.yaml`** — they are regenerated and hold private +keys. Change `.env`, re-run `make up`. + +## Step 8 — Verify end to end + +Import both files from `../api-collection/` into Postman — +`OpenAgriNet.api-collection.json` and the environment for your mode: +`OpenAgriNet.network-adapter.local-environment.json` (single, every adapter +host on 9200) or `OpenAgriNet.local-environment.json` (multi). Select the +environment, or every `{{...-host}}` resolves to nothing. Or: + +```sh +newman run ../api-collection/OpenAgriNet.api-collection.json \ + -e ../api-collection/OpenAgriNet.network-adapter.local-environment.json # multi: OpenAgriNet.local-environment.json +``` + +**18 requests, 32 assertions**, one folder per capability plus a health folder. +Each capability folder is publish, then discover, then select, in that order — +so run publish first. Every request's `transactionId`, `messageId` and +`timestamp` are filled fresh by the collection's pre-request script; the rest +of each body comes from `capability-examples/`, unedited. + +Green means the registry is seeded, signatures verify both ways, all four +mappings work and discovery is indexing. -## Modes +POCRA is the one folder whose empty answer is LOGGED rather than failed: it +returns 200 with no providers both when nothing is nearby and when it is +rate-limiting, and the two are indistinguishable. Knowledge is also the one folder that +can fail on a missing credential rather than on wiring — it authenticates with +OAuth2 client credentials that only the adapter reads, at call time. -The adapter has three tiers: **consumer** (the caller's entry), **network** -(fronts discovery) and **provider** (calls the upstream). The quick-start runs -them in one of two modes. Same `docker-compose.yml`, same image, same keys, -same registry and data; a compose profile picks the adapters. +To point it at another deployment, edit the **environment** file, not the +collection. → Appendix N. -**Single mode is the default. Multi mode is optional.** +## Two modes -| | single (default) | multi (optional) | +| | single (default) | multi (optional) | |---|---|---| -| What runs | one container, `network-adapter`, with all three tiers as modules | three containers: `consumer-adapter`, `network-adapter`, `provider-adapter` | +| What runs | one container, `network-adapter` (compose service `network-adapter-layer`) | `consumer-adapter`, `network-adapter`, `provider-adapter` | | Compose profile | `network` | `multi` | -| Start, local | `make up-core` | `make up-core-multi` | -| Start, VM (with edge and observability) | `make up` | `make up-multi` | +| Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | | Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | | App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | | Catalogue publishes to | `:9200/publish` | `:9200/publish` | -| Tier-to-tier hops | loopback HTTP inside the container | HTTP between containers | -| Port setting in `.env` | `NETWORK_ADAPTER_LAYER_PORT` | `PROVIDER_ADAPTER_PORT`, `NETWORK_ADAPTER_PORT`, `CONSUMER_ADAPTER_PORT` | -**When to use which.** Single is the everyday choice: one container, fewer -hops, the same requests. Use multi to reproduce a deployment where the tiers -run separately (different teams, separate scaling or rollouts), or to compare -the two. +Single is the everyday choice: one container, the same requests. Use multi to +reproduce a deployment where the tiers run separately. + +**Switching** is running the other target. It removes the running mode's +adapters first (both use 9200, and both have a container named +`network-adapter`), then starts the other's. `MODE=multi` also works with +`make up` and `make up-core`; `make restart`, `make down`, `make ps` and +`make logs` follow whichever mode is running. Plain compose: +`docker compose --profile network up -d` (or `--profile multi`); a bare +`docker compose up -d` follows `COMPOSE_PROFILES` in `.env` (`network` in +`.env.example`). + +**How single mode is wired.** `config/adapters/network-adapter.yaml.tmpl`, +rendered by `setup.py`, holds four modules, each its tier's module from the +tier templates, each with its own tier identity: + +| Module | Path | Routes to | +|---|---|---| +| `consumer` | `/` | discover to `localhost:9200/_internal/network`, select/init/confirm/status to `localhost:9200/_internal/provider` (`routing-network-adapter-consumer.yaml`) | +| `network` | `/_internal/network/` | discovery (`routing-network.yaml`, reused) | +| `provider` | `/_internal/provider/` | answers itself | +| `provider-publish` | `/publish` | `localhost:9200/_internal/network/publish` (`routing-network-adapter-publish.yaml`) | + +`/_internal/` is called only by the adapter itself; `server_proxy.conf` denies +it, and `/publish`, on every proxy host. Signing is off in every quick-start +template, in both modes, so keep the adapter ports on `127.0.0.1`. + +--- -Signing is off in every quick-start adapter config, in both modes, so the -adapter ports stay on `127.0.0.1` (the compose file binds them there). +# Part 2 — Run it on a VM -## How to switch +Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. -Only one mode runs at a time: both use port 9200, and both have a container -called `network-adapter`. Switching is running the other mode's target; it -removes the running mode's adapters first, and starts the other's. +## Step V1 — Prepare the VM + +Nothing is cloned yet, so fetch the script rather than running it from the repo: ```sh -make up-core-multi # single -> multi -make up-core # multi -> single +curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash ``` -On a VM use `make up-multi` / `make up`. If nginx-proxy-manager is running it -is restarted so it re-resolves the adapter addresses: the public hosts drop -connections for a few seconds. Repoint the proxy hosts (Appendix B) in the -same window. +Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt +repo, then adds you to the `docker` group — **log out and back in** for that to +take effect. Idempotent, and it deliberately does not clone, write `.env` or +start anything. -Other ways to pick the mode: +**8 GB** runs the stack; **16 GB** for the observability tier. -| How | Single | Multi | -|---|---|---| -| make target | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | -| `MODE` variable (works with `up` and `up-core`; `restart` follows the running mode) | `MODE=single make up-core` (the default) | `MODE=multi make up-core` | -| plain compose | `docker compose --profile network up -d` | `docker compose --profile multi up -d` | -| bare `docker compose up -d` | `COMPOSE_PROFILES=network` in `.env` (as in `.env.example`) | `COMPOSE_PROFILES=multi` in `.env` | +Then Part 1 Steps 2 and 3 as written. + +## Step V2 — Change every credential + +`.env.example` ships working defaults, which means they are public: -Without `COMPOSE_PROFILES` a bare `docker compose up -d` starts no adapter. -`bin/stack.sh` (behind every make target) passes `--profile` itself and -ignores `COMPOSE_PROFILES`. +``` +POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_ADMIN_CLIENT_SECRET +KEYCLOAK_DEFAULT_USER_PASSWORD +``` -These follow whichever mode is running, with nothing to set: `make down`, -`make ps`, `make logs`, `make restart`. +Change all four before the VM is reachable by anyone but you. Adapter keypairs +are the exception — `setup.py` generates those and never writes them to `.env`. -Which mode is running: +## Step V3 — Start it ```sh -docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' network-adapter -# network-adapter-layer = single, network-adapter = multi +make up ``` -One API collection serves both modes; pick the environment for the running -mode (SETUP.md Step 6): `OpenAgriNet.network-adapter.local-environment.json` -for single, `OpenAgriNet.local-environment.json` for multi. +`make up`, not `make up-core`: two more tiers on top of Part 1's three — +nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes +the VM reachable from the internet.** `make up-multi` runs the three adapters +instead; the mode decides what the proxy hosts in Appendix B forward to. +Switching mode on a running VM restarts nginx-proxy-manager, so the public +hosts drop connections for a few seconds. -## The stack in one picture +## Step V4 — Decide what is exposed -``` - consumer - │ - ▼ - consumer adapter the only one a consumer calls - │ - ├── discover ──► network adapter ──► discovery service - │ - └── select ──► provider adapter ──► mock-imd - └─► mock-agmarknet +Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written +literally in `docker-compose.yml` rather than taken from a variable. So the +registry, Keycloak and the databases are not publicly reachable — deliberately. + +Proxy hosts, certificates, and the `/publish` deny every host gets → +**Appendix B**. Read it before pointing DNS at the box. + +## Step V5 — Reach the loopback ports + +```sh +ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ + -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm ``` -`publish` runs the other way: the provider adapter sends it to the network -adapter, which files the catalogue in discovery. +The collection's defaults then work unchanged, since they already point at +loopback. Single mode needs only 9200 for the adapter; 9202 is for multi. -In single mode each adapter box above is a module of the one -`network-adapter` container, and the arrows between them are calls inside -that container. In multi mode each box is its own container. +## Step V6 — Observability (optional) -All three tiers read the registry — who signed this request, and where this -capability's provider lives. +```sh +make observability +``` -A `select` answers in the same HTTP round trip — there is no callback. +HyperDX on `127.0.0.1:8085`, OTLP on 4317/4318 — and, once you add its proxy +host, on a public hostname behind an NPM Access List, which is how you read it +without a tunnel. Route + credential → **Appendix B**. → Appendix H, which is +honest about how much actually arrives. + +Every container's stdout lands there too, next to the traces — so this is also +what replaces keeping `docker compose logs -f` open. How that is wired, and the +one gotcha, are in Appendix H. -That is the request order. **Startup order is the reverse**: registry and -discovery first, then `setup.py`, then the adapters. → Appendix C. +`docker compose restart nginx-proxy-manager` after this command, the first +time: NPM only resolves `hyperdx` on reload. -## How single mode is wired +--- -`config/adapters/network-adapter.yaml.tmpl`, beside the three tier templates. -`bin/setup.py` renders it into `config/adapters/network-adapter.yaml` like the -others. Each module is its tier's module from the tier template, so a change -to a tier template that should apply in both modes goes in this file too. On -top: extended schema validation on every module, an outbound connection pool -and log level info. Each module keeps its own tier identity. It is the same -layout as the Helm chart's `config/network-adapter-config.yaml`. +## If something is wrong -| Module | Path | From | Routing file | Routes | -|---|---|---|---|---| -| `consumer` | `/` | `consumer.yaml.tmpl` | `routing-network-adapter-consumer.yaml` | discover to `localhost:9200/_internal/network`; select, init, confirm, status to `localhost:9200/_internal/provider` | -| `network` | `/_internal/network/` | `network.yaml.tmpl` | `routing-network.yaml` (the multi file, reused) | discover, publish to discovery | -| `provider` | `/_internal/provider/` | `provider.yaml.tmpl` (module 1) | none: answers itself | n/a | -| `provider-publish` | `/publish` | `provider.yaml.tmpl` (module 2) | `routing-network-adapter-publish.yaml` | to `localhost:9200/_internal/network/publish` | +- **`unrecognized step: `** — `ADAPTER_IMAGE` predates the config. + Appendix G. +- **404 `NET_ENTITY_NOT_FOUND`** — no provider step claimed the payload; a + binding key disagrees with `.env`. Appendix F. +- **404 naming a binding with no active record** — the step matched but the + registry has no `ProviderSchema` row for it. Appendix F. +- **502 from a `select`** — the upstream answered non-2xx. Appendix F. +- **NPM's default page, or a 502 that worked yesterday** — Appendix B. +- **An adapter config is a directory** — something started before + `setup.py`. Appendix F. -The hops between modules are HTTP calls to the same container; in multi mode -only the hosts differ. 9200 is the container's own listener and must match -`http.port` in `network-adapter.yaml.tmpl`. +Full set, with what to run for each → **Appendix F**. --- @@ -169,7 +375,7 @@ default, or three tier adapters with `make up-multi`; same image), **two mock upstreams** standing in for Mausamgram and Agmarknet. Behind profiles: **nginx-proxy-manager** (`reverse-proxy`, the only container on a routable interface) and **hyperdx** (`observability`, ClickStack — the -heaviest thing here). What changed, and when: [CHANGELOG.md](CHANGELOG.md). +heaviest thing here). Deliberately absent: @@ -864,8 +1070,6 @@ one minute of adapter cache plus about five of CDN. docker-compose.yml the whole stack, in tiers -- the banner comments are the structure .env.example copy to .env -SETUP.md install and run, local and VM, both modes -CHANGELOG.md what changed, by version Makefile the front door; every target delegates to stack.sh bin/ bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM From e0673c18ac29684eeca8396d42383aac2aeb2cb7 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 10:51:46 +0530 Subject: [PATCH 32/43] refactor(adapter-service, quick-start): call single/multi a deployment mode (#55, #66) - helmfile value networkAdapterTopology -> networkAdapterDeploymentMode (README snippet: $mode, guard message) - "topology" -> "deployment mode" in the text this PR added: chart README (section "Choosing a deployment mode", anchor link), CHANGELOGs, Chart.yaml, values, example and CI headers, NOTES ("mode" line), openbao-secrets values, .env.example, stack.sh, docker-compose.yml - topologySpreadConstraints/topologyKey and the existing "routing is topology" notes are unchanged Refs #55 Refs #66 --- charts/adapter-service/CHANGELOG.md | 16 ++++----- charts/adapter-service/Chart.yaml | 2 +- charts/adapter-service/README.md | 36 +++++++++---------- .../ci/network-adapter-values.yaml | 2 +- .../config/network-adapter-config.yaml | 2 +- .../examples/network-adapter.yaml | 2 +- charts/adapter-service/templates/NOTES.txt | 4 +-- charts/adapter-service/values.yaml | 8 ++--- charts/openbao-secrets/CHANGELOG.md | 2 +- charts/openbao-secrets/values.yaml | 2 +- quick-start/.env.example | 6 ++-- quick-start/bin/stack.sh | 16 ++++----- quick-start/docker-compose.yml | 10 +++--- 13 files changed, 54 insertions(+), 54 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index cb79252c..12fbf5e8 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -8,19 +8,19 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ## [0.6.0] - 2026-10-09 ### Added -- **Two topologies, with single as the default.** `role: network-adapter-layer` +- **Two deployment modes, with single as the default.** `role: network-adapter-layer` is the Network Adapter Layer: the consumer, network and provider tiers as four modules of one process (`/` for the consumer, `/publish`, `/_internal/network/`, `/_internal/provider/`), one release called `network-adapter`. It is the default; the existing roles `provider`, `network` and `consumer` (one release per - tier) are the optional multi topology and are unchanged. The README's - "Choosing a topology" has the helmfile snippet with - `networkAdapterTopology: single | multi` (default `single`; a deploy-layer + tier) are the optional multi deployment mode and are unchanged. The README's + "Choosing a deployment mode" has the helmfile snippet with + `networkAdapterDeploymentMode: single | multi` (default `single`; a deploy-layer value, not a chart value). - `config/network-adapter-config.yaml`: the layer's adapter config, mounted like any role's file (extended schema, an outbound connection pool and log level info on every module). Each module is its tier's module, so a change to - a tier file that should apply to both topologies is made in both. Its routing + a tier file that should apply to both deployment modes is made in both. Its routing is generated at render time from `config/routing-*.yaml` (`routing-network-adapter-{consumer,publish}.yaml`), with tier hosts rewritten to `http://localhost:/_internal/`, so tier hops stay inside the pod. The render fails if `http.port` differs from `service.targetPort`. @@ -29,7 +29,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h (`networkAdapterLayer.extraSubstitutions` for extra placeholders). A literal private key in the config, including a `config:` override, fails the render. - Flat paths: `/discover`, `/publish`, `/` and `/health`, the - same URLs as the multi topology. The network and provider tiers are mounted + same URLs as the multi deployment mode. The network and provider tiers are mounted under `/_internal/`, which only the process itself calls. - Ingress for the layer may expose only single-segment `/`, as literal paths with `pathType` Prefix or Exact and no `use-regex` / `rewrite-target` @@ -43,7 +43,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ### Changed - Docs, `values.yaml` and the example headers say single is the default - topology and multi is optional. + deployment mode and multi is optional. - Naming: the layer's release, Service and app name are `network-adapter` (telemetry service `oan-network-adapter`); its namespace is `network-adapter-layer`. "layer" appears only in the namespace, the role, @@ -51,7 +51,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h `network` tier. The app name and telemetry service name are set in `config/network-adapter-config.yaml`. The separate namespace keeps the consumer and provider keys, which openbao-secrets mirrors to the layer, out - of the multi network tier's namespace, and lets both topologies run side by + of the multi network tier's namespace, and lets both deployment modes run side by side during a switch-over. ### Fixed diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index 182e8160..aab0f630 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: adapter-service description: >- - One chart for the OAN Beckn adapters, in two topologies. single (the + One chart for the OAN Beckn adapters, in two deployment modes. single (the default): the Network Adapter Layer (role network-adapter-layer) runs the consumer, network and provider tiers as four modules of one process; tier-to-tier calls stay inside the pod and each module keeps its own tier diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index f8c28d64..f3aabe33 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -1,6 +1,6 @@ # adapter-service -One chart for the Beckn adapters, in two topologies. **single (the default):** +One chart for the Beckn adapters, in two deployment modes. **single (the default):** the Network Adapter Layer, role `network-adapter-layer`, all three tiers in one adapter. **multi (optional):** three roles, `provider`, `network` and `consumer`, one release per tier. All roles are the same image and the same @@ -22,7 +22,7 @@ and where requests are routed. /publish ──lo──▶ /_internal/network/publish ``` -The paths are flat and the same in both topologies: `/discover`, `/publish`, +The paths are flat and the same in both deployment modes: `/discover`, `/publish`, `/` and `/health`. The network and provider tiers sit under `/_internal/`, which only this process calls (over loopback). The ingress may expose only single-segment `/` paths: `/publish` stays in-cluster for @@ -53,7 +53,7 @@ reads from the registry at request time. ## Install -Pick **one** topology per environment (→ [Choosing a topology](#choosing-a-topology-single-default-or-multi-optional)). +Pick **one** deployment mode per environment (→ [Choosing a deployment mode](#choosing-a-deployment-mode-single-default-or-multi-optional)). Install the Secrets first: the render fails without them. ```sh @@ -165,9 +165,9 @@ the **public** half of the keypair. Missing, or carrying a different key, and every peer rejects what this adapter signs while the pod stays perfectly healthy. -## Choosing a topology: single (default) or multi (optional) +## Choosing a deployment mode: single (default) or multi (optional) -| Topology | Releases → namespaces | Values | App calls | Catalogue publishes to | +| Deployment mode | Releases → namespaces | Values | App calls | Catalogue publishes to | |---|---|---|---|---| | **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9200/` | `network-adapter.network-adapter-layer.svc:9200/publish` | | multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` → one namespace each, same name | `examples/{consumer,network,provider}.yaml` | `consumer-adapter.consumer-adapter.svc:9202/` | `provider-adapter.provider-adapter.svc:9200/publish` | @@ -186,39 +186,39 @@ choice changes nothing in the registry. `role: network-adapter-layer` runs all three tiers in one adapter, as four modules of one process. Its config is `config/network-adapter-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change -to a tier file that should apply to both topologies goes in both files). Its routing +to a tier file that should apply to both deployment modes goes in both files). Its routing is **generated** at render time from `config/routing-*.yaml`, so routes stay in one place. Tier hops go to the pod's own listener, `localhost:` (any image). It signs as the same three identities, from the same three Secrets, so the registry is the -same in both topologies. An environment's `config:` override for a multi role is NOT +same in both deployment modes. An environment's `config:` override for a multi role is NOT seen by the Network Adapter Layer; give its release its own `config:` if needed (it replaces `network-adapter-config.yaml` whole). -Pick the topology per environment with ONE value where releases are chosen, -`networkAdapterTopology: single | multi` (a helmfile environment value; no chart +Pick the deployment mode per environment with ONE value where releases are chosen, +`networkAdapterDeploymentMode: single | multi` (a helmfile environment value; no chart reads it; **default `single`**). `single` installs one release, `network-adapter` in namespace `network-adapter-layer` (the Network Adapter Layer); `multi` installs the three tier releases. For a helmfile (adapt to infra-automation's layout): ```yaml -# environments: values may carry networkAdapterTopology: multi # default: single -{{- $topology := .Values.networkAdapterTopology | default "single" }} -{{- if not (has $topology (list "single" "multi")) }} -{{- fail (printf "networkAdapterTopology must be single or multi, not %q" $topology) }} +# environments: values may carry networkAdapterDeploymentMode: multi # default: single +{{- $mode := .Values.networkAdapterDeploymentMode | default "single" }} +{{- if not (has $mode (list "single" "multi")) }} +{{- fail (printf "networkAdapterDeploymentMode must be single or multi, not %q" $mode) }} {{- end }} releases: {{- range $tier := list "consumer" "network" "provider" }} - name: {{ $tier }}-adapter namespace: {{ $tier }}-adapter chart: ../helmcharts/charts/adapter-service - installed: {{ eq $topology "multi" }} + installed: {{ eq $mode "multi" }} values: [values/{{ $tier }}-adapter.yaml] {{- end }} - name: network-adapter namespace: network-adapter-layer chart: ../helmcharts/charts/adapter-service - installed: {{ eq $topology "single" }} + installed: {{ eq $mode "single" }} values: [values/network-adapter.yaml] ``` @@ -230,21 +230,21 @@ adapter as a whole, so its release, Service and app name are `network-adapter`, the same as the multi network tier's. Its **namespace** is its own, `network-adapter-layer`, for two reasons: openbao-secrets mirrors all three tier identities into it, which must not happen in the multi network tier's -namespace `network-adapter`; and the two topologies can then run side by side +namespace `network-adapter`; and the two deployment modes can then run side by side during a switch-over. In-cluster address of the layer: `network-adapter.network-adapter-layer.svc.cluster.local:9200`. To switch an environment: install the layer (namespace `network-adapter-layer`) next to the multi releases, repoint the app (`…:9200/`) and the catalogue publisher (`…:9200/publish`), then remove the three multi -releases. Both topologies sign with the same identities and the same registry +releases. Both deployment modes sign with the same identities and the same registry rows, so nothing in the registry changes. Reverse the steps to switch back. ## Values See `values.yaml` — every field is commented with what it does and what breaks without it. `examples/{provider,network,consumer}.yaml` are working dev deployments for the -multi topology, and `examples/network-adapter.yaml` for the single topology (the Network Adapter Layer, default); +multi deployment mode, and `examples/network-adapter.yaml` for the single deployment mode (the Network Adapter Layer, default); `ci/` holds the files lint renders, including `ci/network-adapter-values.yaml`. ## Related diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 0add389c..153ec7d9 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -2,7 +2,7 @@ # CI only (scripts/lint-charts.sh lints and renders every ci/*-values.yaml). # Not for installs: use examples/network-adapter.yaml. # -# Renders the single topology, role network-adapter-layer (the Network Adapter +# Renders the single deployment mode, role network-adapter-layer (the Network Adapter # Layer): config/network-adapter-config.yaml, the generated routing, # the three tier identities, and the PDB / spread / graceful-shutdown # settings, so those templates are linted too. The HPA stays off, as in diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index e8d91dd9..16d7502e 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -308,7 +308,7 @@ modules: # signed as the provider and routed to /_internal/network/publish. No # validateSign: never expose it on an Ingress. At top-level /publish, the # same path as the multi provider adapter's publish module, so both - # topologies present the same URL. + # deployment modes present the same URL. - name: provider-publish path: /publish handler: diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 2beb5da9..7a6acd17 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -1,5 +1,5 @@ # =========================================================================== -# network-adapter-layer role -- the Network Adapter Layer: the DEFAULT topology +# network-adapter-layer role -- the Network Adapter Layer: the DEFAULT deployment mode # (single). The optional alternative is multi: examples/{consumer,network,provider}.yaml. # # The consumer, network and provider tiers as four modules of ONE adapter diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index d10642f2..462931b1 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -1,11 +1,11 @@ {{- $nal := include "adapter-service.isNetworkAdapterLayer" . -}} {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} {{ if $nal }} - topology single (the default): the Network Adapter Layer + mode single (the default): the Network Adapter Layer role network-adapter-layer (all three tiers in one adapter: / consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/_internal/) {{- else }} - topology multi (optional): one release per tier; the default is role network-adapter-layer + mode multi (optional): one release per tier; the default is role network-adapter-layer role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) steps {{ include "adapter-service.steps" . | fromYamlArray | join " -> " }} {{- end }} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index feec2638..7190936a 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -1,7 +1,7 @@ # =========================================================================== # adapter-service # -# One chart for the Beckn adapters, in two topologies. Same image and config +# One chart for the Beckn adapters, in two deployment modes. Same image and config # format; `role` below decides the handler role, the step list and where # requests are routed. # @@ -18,16 +18,16 @@ # helm install network-adapter . -f examples/network.yaml # helm install consumer-adapter . -f examples/consumer.yaml # -# Run one topology per environment. The Network Adapter Layer is released as +# Run one deployment mode per environment. The Network Adapter Layer is released as # `network-adapter` but in its own namespace, `network-adapter-layer`, so the # tier identities openbao-secrets mirrors into it never land in the multi -# network tier's namespace, and the two topologies can coexist during a +# network tier's namespace, and the two deployment modes can coexist during a # switch-over. # # The release name is what the Service is called, which is the DNS name the # other adapters address. In multi, rename a release and you must update # whatever routes to it -- consumer routes to the network adapter, provider is -# routed to by network. The single topology has no such addresses: its tiers +# routed to by network. The single deployment mode has no such addresses: its tiers # call each other inside the pod. # # Two things every role needs that this chart cannot give it: diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index 2a89a8ff..8378f538 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `network-adapter-layer` as a target namespace of `consumer-adapter-keys`, `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for `adapter-service` role `network-adapter-layer`: the Network Adapter Layer - (the default topology) runs all three tiers, so it reads all three + (the default deployment mode) runs all three tiers, so it reads all three identities. It has a namespace of its own rather than the multi network tier's `network-adapter`, so multi environments do not get the consumer and provider keys or the upstream credentials copied next to the network tier. diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index f55e05df..c8966a59 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -59,7 +59,7 @@ secrets: # change once per environment after the first bring-up. # # network-adapter-layer is the namespace of the Network Adapter Layer (the - # single topology, role network-adapter-layer): it runs all three tiers, so + # single deployment mode, role network-adapter-layer): it runs all three tiers, so # it reads all three identities. It is a namespace of its own, NOT the multi # network tier's `network-adapter`, so a multi environment never gets the # consumer and provider keys copied next to the network tier. Reflector diff --git a/quick-start/.env.example b/quick-start/.env.example index 236854c7..118538fd 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,7 +39,7 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 -# The Network Adapter (the default topology). Inside the container +# The Network Adapter (the default deployment mode). Inside the container # it always listens on 9200; this only moves the host side. NETWORK_ADAPTER_LAYER_PORT=9200 # Only for looking at a mock directly; the adapter reaches it by service name. @@ -223,7 +223,7 @@ POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in POCRA_PATH=/search POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/0.0.1/quick-start/config/mappings/pocra/agriculture-facility.select.yaml -# Which adapter topology a bare `docker compose up -d` starts: "network" (the -# single topology: one adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes +# Which adapter deployment mode a bare `docker compose up -d` starts: "network" (the +# single deployment mode: one adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes # --profile explicitly and ignores this; it is for plain compose commands. COMPOSE_PROFILES=network diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index db49ed2f..784df0d8 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -34,16 +34,16 @@ cd "$ROOT" # Every optional profile, named explicitly. This matters for `down`: compose # only acts on services whose profile is active, so a plain `docker compose -# down` leaves the gateway, hyperdx and the adapters (both topologies are +# down` leaves the gateway, hyperdx and the adapters (both deployment modes are # profiled) running and then reports success. Naming them on teardown is what # makes "down" mean down. PROFILES=(--profile reverse-proxy --profile observability --profile network --profile multi) -# Adapter topology: "single" (default) runs the Network Adapter Layer (compose +# Adapter deployment mode: "single" (default) runs the Network Adapter Layer (compose # service network-adapter-layer, container network-adapter), one process for # all three tiers; "multi" runs provider-, network- and consumer-adapter. Both # publish 127.0.0.1:9200, and the single container shares the multi network -# tier's name, so only one topology exists at a time: the other is REMOVED +# tier's name, so only one deployment mode exists at a time: the other is REMOVED # (not just stopped) before this one starts. # Validated only by the commands that use it (check_mode), so an unrelated # MODE in the shell never blocks `down`, `destroy` or `help`. @@ -52,8 +52,8 @@ check_mode() { case "$MODE" in single|multi) ;; *) die "MODE must be single or multi, not '$MODE'" ;; esac } -# The topology whose adapters are running now: single, multi, or empty. -# Both topologies have a container called network-adapter, so single is told +# The deployment mode whose adapters are running now: single, multi, or empty. +# Both deployment modes have a container called network-adapter, so single is told # apart by its compose service label. running_mode() { if [ "$(docker inspect -f '{{.State.Running}}' consumer-adapter 2>/dev/null)" = true ]; then echo multi @@ -69,7 +69,7 @@ adapters_for() { # $1 mode multi) echo provider-adapter network-adapter consumer-adapter ;; esac } -# The rendered config files (config/adapters/.yaml) a topology mounts. +# The rendered config files (config/adapters/.yaml) a deployment mode mounts. configs_for() { # $1 mode case "$1" in single) echo network-adapter ;; @@ -78,7 +78,7 @@ configs_for() { # $1 mode } other_mode() { [ "$1" = single ] && echo multi || echo single; } -# The compose profile for a topology: single runs under profile "network", +# The compose profile for a deployment mode: single runs under profile "network", # multi under profile "multi". profile_for() { [ "$1" = single ] && echo network || echo multi; } @@ -194,7 +194,7 @@ up_adapters() { check_mode local other f; other=$(other_mode "$MODE") step 3 "$1" "adapters, mode $MODE ($(adapters_for "$MODE"))" - # The other topology also publishes 127.0.0.1:9200, and the single container + # The other deployment mode also publishes 127.0.0.1:9200, and the single container # shares the multi network tier's name: remove it (stopped is not enough). # shellcheck disable=SC2046 docker compose --profile "$(profile_for "$other")" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index d7c4b19c..6cbe2843 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -5,10 +5,10 @@ # what `docker ps` shows is what `docker logs` and the in-stack DNS name take. # One exception: the Network Adapter Layer is service `network-adapter-layer` # (profile network) with container `network-adapter`, the same container name -# as the multi network tier's service `network-adapter`. Only one topology runs +# as the multi network tier's service `network-adapter`. Only one deployment mode runs # at a time, so `docker ps` / `docker logs network-adapter` mean whichever runs; # compose commands (`docker compose logs`, `make logs SVC=`) take the service -# name, `network-adapter-layer` in the single topology. +# name, `network-adapter-layer` in the single deployment mode. # The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among # them authenticates nothing but that registry, and shares its postgres. # @@ -541,9 +541,9 @@ services: condition: service_healthy # ========================================================================== - # adapters -- TWO topologies, chosen with a compose profile. Same image. + # adapters -- TWO deployment modes, chosen with a compose profile. Same image. # - # profile "network" (the single topology, the default; `make up`) + # profile "network" (the single deployment mode, the default; `make up`) # network-adapter-layer (container network-adapter): the consumer, network and provider tiers # as four modules of ONE process, one config, one port (127.0.0.1:9200). # Tier-to-tier hops stay inside the container. @@ -1006,7 +1006,7 @@ services: # `make observability` -- NPM was started before the name existed, so the # HyperDX host 502s until it re-resolves. # - # required: false -- only one topology's adapters exist at a time + # required: false -- only one deployment mode's adapters exist at a time # (profiles "network" / "multi"); NPM starts with whichever is running. network-adapter-layer: condition: service_started From 7d19cfcc5a06ebac3ea4863cf7831d90860862a7 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 11:13:16 +0530 Subject: [PATCH 33/43] feat(adapter-service): add routing network adapter provider configuration Implementation plan for running one adapter, and review Fixes #65 --- ...adapter-publish.yaml => routing-network-adapter-provider.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename quick-start/config/adapters/{routing-network-adapter-publish.yaml => routing-network-adapter-provider.yaml} (100%) diff --git a/quick-start/config/adapters/routing-network-adapter-publish.yaml b/quick-start/config/adapters/routing-network-adapter-provider.yaml similarity index 100% rename from quick-start/config/adapters/routing-network-adapter-publish.yaml rename to quick-start/config/adapters/routing-network-adapter-provider.yaml From edc86541beb398eddaab2c81918b3e24dc74db56 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 11:13:35 +0530 Subject: [PATCH 34/43] fix(adapter-service): update routing configuration for provider tier in various files #55 End-to-end verification and demonstration of the single adapter Fixes #66 --- charts/adapter-service/CHANGELOG.md | 2 +- charts/adapter-service/config/network-adapter-config.yaml | 2 +- charts/adapter-service/templates/_network-adapter.tpl | 2 +- charts/adapter-service/values.yaml | 2 +- quick-start/README.md | 4 ++-- quick-start/config/adapters/network-adapter.yaml.tmpl | 4 ++-- quick-start/docker-compose.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 12fbf5e8..303297c4 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -22,7 +22,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h level info on every module). Each module is its tier's module, so a change to a tier file that should apply to both deployment modes is made in both. Its routing is generated at render time from `config/routing-*.yaml` - (`routing-network-adapter-{consumer,publish}.yaml`), with tier hosts rewritten + (`routing-network-adapter-{consumer,provider}.yaml`), with tier hosts rewritten to `http://localhost:/_internal/`, so tier hops stay inside the pod. The render fails if `http.port` differs from `service.targetPort`. - Three identities: each module signs as its own tier from `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index 16d7502e..42356db3 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -358,7 +358,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-network-adapter-publish.yaml + routingConfig: /app/config/routing-network-adapter-provider.yaml middleware: - id: reqpreprocessor config: diff --git a/charts/adapter-service/templates/_network-adapter.tpl b/charts/adapter-service/templates/_network-adapter.tpl index f867de1a..79384315 100644 --- a/charts/adapter-service/templates/_network-adapter.tpl +++ b/charts/adapter-service/templates/_network-adapter.tpl @@ -55,7 +55,7 @@ modules across pods again changes only the hosts. namespace suffix and port) followed by nothing or a path is rewritten. */ -}} {{- $tierHost := "^https?://(consumer|network|provider)-adapter(\\.[a-z0-9.-]+)?(:[0-9]+)?(/.*)?$" -}} {{- $out := dict -}} -{{- range $pair := list (list "routing-consumer.yaml" "routing-network-adapter-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-network-adapter-publish.yaml") -}} +{{- range $pair := list (list "routing-consumer.yaml" "routing-network-adapter-consumer.yaml") (list "routing-network.yaml" "routing-network.yaml") (list "routing-provider.yaml" "routing-network-adapter-provider.yaml") -}} {{- $src := index $pair 0 -}} {{- $doc := $ctx.Files.Get (printf "config/%s" $src) | fromYaml -}} {{- if not $doc.routingRules -}} diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 7190936a..c2e27456 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -454,7 +454,7 @@ envFromSecrets: [] # config file). `config` replaces config/network-adapter-config.yaml # WHOLE: the private-key guard still runs on it, and it must point its routers # at /app/config/routing-network-adapter-consumer.yaml, routing-network.yaml and -# routing-network-adapter-publish.yaml. +# routing-network-adapter-provider.yaml. # --------------------------------------------------------------------------- networkAdapterLayer: # Three identities, one per tier: the same Secrets (six keys each, named as diff --git a/quick-start/README.md b/quick-start/README.md index fccfffbc..66317291 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -236,7 +236,7 @@ tier templates, each with its own tier identity: | `consumer` | `/` | discover to `localhost:9200/_internal/network`, select/init/confirm/status to `localhost:9200/_internal/provider` (`routing-network-adapter-consumer.yaml`) | | `network` | `/_internal/network/` | discovery (`routing-network.yaml`, reused) | | `provider` | `/_internal/provider/` | answers itself | -| `provider-publish` | `/publish` | `localhost:9200/_internal/network/publish` (`routing-network-adapter-publish.yaml`) | +| `provider-publish` | `/publish` | `localhost:9200/_internal/network/publish` (`routing-network-adapter-provider.yaml`) | `/_internal/` is called only by the adapter itself; `server_proxy.conf` denies it, and `/publish`, on every proxy host. Signing is off in every quick-start @@ -1097,7 +1097,7 @@ config/ routing-provider.yaml provider sends publish to the network layer network-adapter.yaml.tmpl single mode: all three tiers, one file routing-network-adapter-consumer.yaml single mode: consumer hops to - routing-network-adapter-publish.yaml localhost:9200/_internal/... + routing-network-adapter-provider.yaml localhost:9200/_internal/... registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs a restart diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 2a8a7dab..8780a1b5 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -295,7 +295,7 @@ modules: # ---------------------------------------------------------- provider-publish # From provider.yaml.tmpl, module 2. The catalogue publishes here, under the # provider identity (signing is off here); routed to /_internal/network/publish - # (routing-network-adapter-publish.yaml). Top-level /publish, the same path + # (routing-network-adapter-provider.yaml). Top-level /publish, the same path # as the multi provider adapter's. - name: provider-publish path: /publish @@ -351,7 +351,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-network-adapter-publish.yaml + routingConfig: /app/config/routing-network-adapter-provider.yaml steps: - validateSchema - addRoute diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 6cbe2843..2b8c3157 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -646,7 +646,7 @@ services: # multi-adapter routing-network.yaml unchanged. - ./config/adapters/routing-network-adapter-consumer.yaml:/app/config/routing-network-adapter-consumer.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - - ./config/adapters/routing-network-adapter-publish.yaml:/app/config/routing-network-adapter-publish.yaml:ro + - ./config/adapters/routing-network-adapter-provider.yaml:/app/config/routing-network-adapter-provider.yaml:ro ports: - "127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9200}:9200" From 3ea6492f90ea7ec7b0306ca5d70e3fe6e04666f6 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 12:33:33 +0530 Subject: [PATCH 35/43] refactor(adapter-service): drop autoscaling and token-URL substitution from network-adapter values (#55, #66) - examples/network-adapter.yaml, ci/network-adapter-values.yaml: remove the autoscaling block (fixed replicaCount: 3; the chart default stays off) - networkAdapterLayer.extraSubstitutions back to {}; the example says to set __VISTAAR_TOKEN_URL__ per environment - CHANGELOG and CI header no longer mention the HPA Refs #55 Refs #66 --- charts/adapter-service/CHANGELOG.md | 2 +- .../adapter-service/ci/network-adapter-values.yaml | 14 ++------------ .../adapter-service/examples/network-adapter.yaml | 14 +++----------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 303297c4..281a1db0 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -36,7 +36,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h annotation; `/`, `/publish` and `/_internal*` are refused. `routing.config` fails the render for this role. - `examples/network-adapter.yaml` and `ci/network-adapter-values.yaml` - (PDB, spread, graceful shutdown; HPA 3-10 opt-in, off by default). + (3 replicas, PDB, spread, graceful shutdown). - `topologySpreadConstraints`, `terminationGracePeriodSeconds`, `preStopSleepSeconds`, `goMemLimit`; all off by default. diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 153ec7d9..de5d2957 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -5,8 +5,7 @@ # Renders the single deployment mode, role network-adapter-layer (the Network Adapter # Layer): config/network-adapter-config.yaml, the generated routing, # the three tier identities, and the PDB / spread / graceful-shutdown -# settings, so those templates are linted too. The HPA stays off, as in -# examples/network-adapter.yaml: autoscaling is opt-in. Pinned image tag so the render +# settings, so those templates are linted too. Pinned image tag so the render # is reproducible. # # The Ingress is on so the path guard in templates/ingress.yaml runs in CI: the @@ -38,8 +37,7 @@ networkAdapterLayer: existingSecret: network-adapter-keys provider: existingSecret: provider-adapter-keys - extraSubstitutions: - __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + extraSubstitutions: {} # Scale defaults for the Network Adapter Layer: roughly the three tiers' # requests in one pod, at least one replica per node. @@ -51,15 +49,7 @@ resources: cpu: 1500m memory: 1Gi goMemLimit: 900MiB -# Fixed replica count by default; the HPA is opt-in. Turning it on needs -# metrics-server in the cluster -- without it the HPA reports -# CPU and never scales. With it off, replicaCount applies. replicaCount: 3 -autoscaling: - enabled: false - minReplicas: 3 - maxReplicas: 10 - targetCPUUtilizationPercentage: 70 podDisruptionBudget: enabled: true minAvailable: 1 diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 7a6acd17..3615f509 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -55,10 +55,10 @@ networkAdapterLayer: existingSecret: network-adapter-keys provider: existingSecret: provider-adapter-keys - # The provider tier's token endpoint, from the provider Secret. Without it + # Set per environment: the config's __VISTAAR_TOKEN_URL__ placeholder needs + # an entry here (e.g. __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl), or # the init container stops on the leftover placeholder. - extraSubstitutions: - __VISTAAR_TOKEN_URL__: provider/vistaarTokenUrl + extraSubstitutions: {} # The provider tier reads its upstream credentials (MAUSAMGRAM_AUTH, # AGMARKNET_*, VISTAAR_CLIENT_*) from the environment. @@ -74,15 +74,7 @@ resources: memory: 1Gi goMemLimit: 900MiB -# Fixed replica count by default; the HPA is opt-in. Turning it on needs -# metrics-server in the cluster -- without it the HPA reports -# CPU and never scales. With it off, replicaCount applies. replicaCount: 3 -autoscaling: - enabled: false - minReplicas: 3 # one per node with the spread constraint below - maxReplicas: 10 - targetCPUUtilizationPercentage: 70 podDisruptionBudget: enabled: true From a9facae665bc447f02184ec8d3a754117283be88 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 13:08:53 +0530 Subject: [PATCH 36/43] refactor(adapter-service, quick-start): single mode listens on 9201 (#55, #66) The single adapter takes the multi network tier's port, which is also the chart's default service port, instead of the multi provider's 9200. - adapter-service: config http.port 9201; example and CI service port and targetPort 9201; README single-mode addresses - quick-start: compose 127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9201}:9201, .env.example, template listener, routing files (localhost:9201), server_proxy.conf and stack.sh comments, README tables - api-collection: single-mode environment hosts on 9201 Refs #55 Refs #66 --- ...Net.network-adapter.local-environment.json | 6 ++-- charts/adapter-service/README.md | 8 ++--- .../ci/network-adapter-values.yaml | 4 +-- .../config/network-adapter-config.yaml | 6 ++-- .../examples/network-adapter.yaml | 6 ++-- quick-start/.env.example | 5 +-- quick-start/README.md | 36 +++++++++---------- quick-start/bin/stack.sh | 4 +-- .../config/adapters/network-adapter.yaml.tmpl | 6 ++-- .../routing-network-adapter-consumer.yaml | 6 ++-- .../routing-network-adapter-provider.yaml | 2 +- .../npm-custom/server_proxy.conf | 2 +- quick-start/docker-compose.yml | 8 ++--- 13 files changed, 50 insertions(+), 49 deletions(-) diff --git a/api-collection/OpenAgriNet.network-adapter.local-environment.json b/api-collection/OpenAgriNet.network-adapter.local-environment.json index 365004ed..d158a88f 100644 --- a/api-collection/OpenAgriNet.network-adapter.local-environment.json +++ b/api-collection/OpenAgriNet.network-adapter.local-environment.json @@ -4,21 +4,21 @@ "values": [ { "key": "consumer-adapter-host", - "value": "http://127.0.0.1:9200", + "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", "description": "Where discover and select enter. Single mode: the one adapter, so all three adapter hosts are the same URL. NETWORK_ADAPTER_LAYER_PORT in .env." }, { "key": "provider-adapter-host", - "value": "http://127.0.0.1:9200", + "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", "description": "Where publish enters. Single mode: the same adapter as consumer-adapter-host. NETWORK_ADAPTER_LAYER_PORT in .env." }, { "key": "network-adapter-host", - "value": "http://127.0.0.1:9200", + "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", "description": "Used only by the health check. Single mode: the same adapter; its network tier sits under /_internal/ and is not called from Postman. NETWORK_ADAPTER_LAYER_PORT in .env." diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index f3aabe33..6a23bc56 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -169,7 +169,7 @@ healthy. | Deployment mode | Releases → namespaces | Values | App calls | Catalogue publishes to | |---|---|---|---|---| -| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9200/` | `network-adapter.network-adapter-layer.svc:9200/publish` | +| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9201/` | `network-adapter.network-adapter-layer.svc:9201/publish` | | multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` → one namespace each, same name | `examples/{consumer,network,provider}.yaml` | `consumer-adapter.consumer-adapter.svc:9202/` | `provider-adapter.provider-adapter.svc:9200/publish` | **When to use which** @@ -232,11 +232,11 @@ the same as the multi network tier's. Its **namespace** is its own, tier identities into it, which must not happen in the multi network tier's namespace `network-adapter`; and the two deployment modes can then run side by side during a switch-over. In-cluster address of the layer: -`network-adapter.network-adapter-layer.svc.cluster.local:9200`. +`network-adapter.network-adapter-layer.svc.cluster.local:9201`. To switch an environment: install the layer (namespace `network-adapter-layer`) -next to the multi releases, repoint the app (`…:9200/`) and the -catalogue publisher (`…:9200/publish`), then remove the three multi +next to the multi releases, repoint the app (`…:9201/`) and the +catalogue publisher (`…:9201/publish`), then remove the three multi releases. Both deployment modes sign with the same identities and the same registry rows, so nothing in the registry changes. Reverse the steps to switch back. diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index de5d2957..fa187fa0 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -26,8 +26,8 @@ image: # log level are set there), so they are left out here. Edit that file instead. service: - port: 9200 - targetPort: 9200 + port: 9201 + targetPort: 9201 networkAdapterLayer: keys: diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index 42356db3..b4d54be6 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -8,7 +8,7 @@ # # The paths are flat: /discover, /publish, / and /health. The tiers are # not part of them. /_internal/ is only ever called by this process's own -# modules (http://localhost:9200/_internal/...); the chart's ingress never +# modules (http://localhost:9201/_internal/...); the chart's ingress never # routes it, and never routes /publish either. # # Each module is its tier's module from consumer-config.yaml, network-config.yaml @@ -16,7 +16,7 @@ # apply here must be made in this file too. # # Differences from the three tier files: -# - one listener (9200) and one appName; tier hops call this same listener (see the routing-network-adapter-*.yaml files, which the chart +# - one listener (9201) and one appName; tier hops call this same listener (see the routing-network-adapter-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; @@ -38,7 +38,7 @@ log: http: # Also the port the generated routing uses: keep it equal to the # chart's service.targetPort. - port: 9200 + port: 9201 timeout: read: 30 write: 30 diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 3615f509..2072e84e 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -25,7 +25,7 @@ role: network-adapter-layer # The Service name and DNS name callers use. The namespace is its own, # network-adapter-layer (not the multi network tier's network-adapter): -# network-adapter.network-adapter-layer.svc.cluster.local:9200 +# network-adapter.network-adapter-layer.svc.cluster.local:9201 fullnameOverride: network-adapter image: @@ -42,8 +42,8 @@ image: # generated routing uses, and must equal http.port in # config/network-adapter-config.yaml (the render checks). service: - port: 9200 - targetPort: 9200 + port: 9201 + targetPort: 9201 networkAdapterLayer: # The three tier Secrets the multi-adapter releases use. openbao-secrets diff --git a/quick-start/.env.example b/quick-start/.env.example index 118538fd..96f3e084 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -40,8 +40,9 @@ PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 # The Network Adapter (the default deployment mode). Inside the container -# it always listens on 9200; this only moves the host side. -NETWORK_ADAPTER_LAYER_PORT=9200 +# it always listens on 9201 (the multi network tier's port); this only moves +# the host side. +NETWORK_ADAPTER_LAYER_PORT=9201 # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 diff --git a/quick-start/README.md b/quick-start/README.md index 66317291..c5ce087c 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -5,7 +5,7 @@ network adapter, and two mock upstreams standing in for real provider APIs. The adapter runs in one of two modes, from the same compose file, keys and registry. **Single is the default**: one container, `network-adapter`, runs the -consumer, network and provider tiers on port 9200. **Multi is optional**: the +consumer, network and provider tiers on port 9201. **Multi is optional**: the three tiers as three containers. → [Two modes](#two-modes). Nothing is built here — the images are pulled. Locally that is about ten @@ -85,10 +85,10 @@ cp .env.example .env Locally, nothing in it needs changing. It publishes these on `localhost`: ``` -8081 registry 9200 network adapter (single) 9100 mock-imd -8080 keycloak or provider adapter (multi) 9101 mock-agmarknet -9990 keycloak admin 9201 network adapter (multi only) -8090 discovery 9202 consumer adapter (multi only) +8081 registry 9201 network adapter (single; network tier in multi) 9100 mock-imd +8080 keycloak 9200 provider adapter (multi only) 9101 mock-agmarknet +9990 keycloak admin 9202 consumer adapter (multi only) +8090 discovery ``` If one is already taken, change it here — that is the only edit a local run @@ -102,7 +102,7 @@ make up-core ``` Three tiers, in the only order that works: registry and discovery, then -`bin/setup.py`, then the adapter (`network-adapter` on 9200; `make up-core-multi` +`bin/setup.py`, then the adapter (`network-adapter` on 9201; `make up-core-multi` starts the three adapters instead). Allow up to five minutes the first time — Keycloak on a cold volume. @@ -178,7 +178,7 @@ keys. Change `.env`, re-run `make up`. Import both files from `../api-collection/` into Postman — `OpenAgriNet.api-collection.json` and the environment for your mode: `OpenAgriNet.network-adapter.local-environment.json` (single, every adapter -host on 9200) or `OpenAgriNet.local-environment.json` (multi). Select the +host on 9201) or `OpenAgriNet.local-environment.json` (multi). Select the environment, or every `{{...-host}}` resolves to nothing. Or: ```sh @@ -211,15 +211,15 @@ collection. → Appendix N. | What runs | one container, `network-adapter` (compose service `network-adapter-layer`) | `consumer-adapter`, `network-adapter`, `provider-adapter` | | Compose profile | `network` | `multi` | | Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | -| Ports | `127.0.0.1:9200` | 9200 provider, 9201 network, 9202 consumer | -| App calls | `:9200/discover`, `:9200/select` | `:9202/discover`, `:9202/select` | -| Catalogue publishes to | `:9200/publish` | `:9200/publish` | +| Ports | `127.0.0.1:9201` | 9200 provider, 9201 network, 9202 consumer | +| App calls | `:9201/discover`, `:9201/select` | `:9202/discover`, `:9202/select` | +| Catalogue publishes to | `:9201/publish` | `:9200/publish` | Single is the everyday choice: one container, the same requests. Use multi to reproduce a deployment where the tiers run separately. **Switching** is running the other target. It removes the running mode's -adapters first (both use 9200, and both have a container named +adapters first (both use 9201, and both have a container named `network-adapter`), then starts the other's. `MODE=multi` also works with `make up` and `make up-core`; `make restart`, `make down`, `make ps` and `make logs` follow whichever mode is running. Plain compose: @@ -233,10 +233,10 @@ tier templates, each with its own tier identity: | Module | Path | Routes to | |---|---|---| -| `consumer` | `/` | discover to `localhost:9200/_internal/network`, select/init/confirm/status to `localhost:9200/_internal/provider` (`routing-network-adapter-consumer.yaml`) | +| `consumer` | `/` | discover to `localhost:9201/_internal/network`, select/init/confirm/status to `localhost:9201/_internal/provider` (`routing-network-adapter-consumer.yaml`) | | `network` | `/_internal/network/` | discovery (`routing-network.yaml`, reused) | | `provider` | `/_internal/provider/` | answers itself | -| `provider-publish` | `/publish` | `localhost:9200/_internal/network/publish` (`routing-network-adapter-provider.yaml`) | +| `provider-publish` | `/publish` | `localhost:9201/_internal/network/publish` (`routing-network-adapter-provider.yaml`) | `/_internal/` is called only by the adapter itself; `server_proxy.conf` denies it, and `/publish`, on every proxy host. Signing is off in every quick-start @@ -307,7 +307,7 @@ ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ ``` The collection's defaults then work unchanged, since they already point at -loopback. Single mode needs only 9200 for the adapter; 9202 is for multi. +loopback. Single mode needs only 9201 for the adapter; 9200 and 9202 are for multi. ## Step V6 — Observability (optional) @@ -423,7 +423,7 @@ certificate. An unknown `Host` gets NPM's default page, not an adapter. **Single mode (`make up`, the default).** The table above is for `make up-multi`. With the single adapter there is no `consumer-adapter`, `network-adapter` or `provider-adapter` to forward to: point the consumer host -at `network-adapter`, port 9200, with the same Advanced paste. The +at `network-adapter`, port 9201, with the same Advanced paste. The app's paths (`/discover`, `/select`, …) do not change, and neither does `/publish`: both modes present the same URLs. `server_proxy.conf` denies `/publish` and `/_internal` on every host: `/_internal/` holds the network and @@ -515,7 +515,7 @@ sender's signature; the publish module (`oanProviderPublish` in multi mode, `provider-publish` in single mode), on the exact path `/publish`, has **no signature check at all**, because its intended caller is the provider's own catalogue system inside the trust boundary. A proxy host pointed at -`provider-adapter:9200` (multi) or `network-adapter:9200` (single) therefore +`provider-adapter:9200` (multi) or `network-adapter:9201` (single) therefore exposes `/publish` to anyone. In single mode the network and provider tiers also sit under `/_internal/`, which only the adapter itself calls. @@ -525,7 +525,7 @@ NPM's UI cannot route a host while withholding one path, so the block lives in something to remember on one host. To let a real catalogue system publish, give it a tunnel or put it in the VPC -and let it reach the adapter on port 9200 directly. Do not turn that `deny` into +and let it reach the adapter directly (9200 in multi, 9201 in single). Do not turn that `deny` into an `allow`. ### Which nginx config loads itself, and which is a paste job @@ -1097,7 +1097,7 @@ config/ routing-provider.yaml provider sends publish to the network layer network-adapter.yaml.tmpl single mode: all three tiers, one file routing-network-adapter-consumer.yaml single mode: consumer hops to - routing-network-adapter-provider.yaml localhost:9200/_internal/... + routing-network-adapter-provider.yaml localhost:9201/_internal/... registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs a restart diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 784df0d8..aea28aca 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -42,7 +42,7 @@ PROFILES=(--profile reverse-proxy --profile observability --profile network --pr # Adapter deployment mode: "single" (default) runs the Network Adapter Layer (compose # service network-adapter-layer, container network-adapter), one process for # all three tiers; "multi" runs provider-, network- and consumer-adapter. Both -# publish 127.0.0.1:9200, and the single container shares the multi network +# publish 127.0.0.1:9201, and the single container shares the multi network # tier's name, so only one deployment mode exists at a time: the other is REMOVED # (not just stopped) before this one starts. # Validated only by the commands that use it (check_mode), so an unrelated @@ -194,7 +194,7 @@ up_adapters() { check_mode local other f; other=$(other_mode "$MODE") step 3 "$1" "adapters, mode $MODE ($(adapters_for "$MODE"))" - # The other deployment mode also publishes 127.0.0.1:9200, and the single container + # The other deployment mode also publishes 127.0.0.1:9201, and the single container # shares the multi network tier's name: remove it (stopped is not enough). # shellcheck disable=SC2046 docker compose --profile "$(profile_for "$other")" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 8780a1b5..20d70716 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -15,7 +15,7 @@ # charts/adapter-service/config/network-adapter-config.yaml. # # Differences from the three tier templates: -# - one listener (9200) and one appName; tier hops call this same listener +# - one listener (9201) and one appName; tier hops call this same listener # (routing-network-adapter-*.yaml beside this file); # - extended schema validation on in every module; # - an outbound connection pool (httpClientConfig) on every module; @@ -23,7 +23,7 @@ # # SIGNING IS OFF, as in the three quick-start tier templates: no module runs # sign or validateSign, so /_internal/ and /publish accept unsigned requests -# from anyone who can reach :9200. Keep it on 127.0.0.1 (as the compose file +# from anyone who can reach :9201. Keep it on 127.0.0.1 (as the compose file # does); server_proxy.conf denies /_internal and /publish at the edge. The # Helm chart's network-adapter-config.yaml has signing on. # @@ -44,7 +44,7 @@ log: - module_id http: - port: 9200 + port: 9201 timeout: read: 30 write: 30 diff --git a/quick-start/config/adapters/routing-network-adapter-consumer.yaml b/quick-start/config/adapters/routing-network-adapter-consumer.yaml index dee56952..9f181810 100644 --- a/quick-start/config/adapters/routing-network-adapter-consumer.yaml +++ b/quick-start/config/adapters/routing-network-adapter-consumer.yaml @@ -8,7 +8,7 @@ # containers and only the host in these URLs changes. # # targetType "url" appends the action to the path, so the base here is the -# module mount point and /discover or /select is added to it. localhost:9200 is +# module mount point and /discover or /select is added to it. localhost:9201 is # the container's own listener and must match `http.port` in network-adapter.yaml.tmpl. routingRules: # Discovery is answered from this network's own discovery service, which the @@ -16,7 +16,7 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/_internal/network" + url: "http://localhost:9201/_internal/network" endpoints: - discover @@ -26,7 +26,7 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/_internal/provider" + url: "http://localhost:9201/_internal/provider" endpoints: - select - init diff --git a/quick-start/config/adapters/routing-network-adapter-provider.yaml b/quick-start/config/adapters/routing-network-adapter-provider.yaml index d43b425d..00c210a0 100644 --- a/quick-start/config/adapters/routing-network-adapter-provider.yaml +++ b/quick-start/config/adapters/routing-network-adapter-provider.yaml @@ -12,7 +12,7 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://localhost:9200/_internal/network/publish" + url: "http://localhost:9201/_internal/network/publish" excludeAction: true endpoints: - "" diff --git a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf index fbfaa780..6c740255 100644 --- a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf +++ b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf @@ -19,7 +19,7 @@ # An exact-match location beats NPM's generated `location /`, so this wins # without conflicting with it. In multi mode, on the consumer and network hosts # it denies a path their adapters do not serve, which costs nothing. In single -# mode the one adapter (network-adapter:9200) serves /publish itself +# mode the one adapter (network-adapter:9201) serves /publish itself # (provider-publish module, no validateSign), so the deny matters on its host. # # To open it for a real catalogue system outside this VM, do NOT edit this to diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 2b8c3157..23f7d33e 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -545,14 +545,14 @@ services: # # profile "network" (the single deployment mode, the default; `make up`) # network-adapter-layer (container network-adapter): the consumer, network and provider tiers - # as four modules of ONE process, one config, one port (127.0.0.1:9200). + # as four modules of ONE process, one config, one port (127.0.0.1:9201, the multi network tier's port). # Tier-to-tier hops stay inside the container. # # profile "multi" (`make up-multi`) # provider-adapter, network-adapter, consumer-adapter: one container per # tier, tier-to-tier hops over the compose network. # - # Both publish 127.0.0.1:9200, so run one profile at a time (bin/stack.sh + # Both publish 127.0.0.1:9201, so run one profile at a time (bin/stack.sh # stops the other first). Both use the same keys and registry rows. # # Their configs are bind-mounted FILES that do not exist until bin/setup.py @@ -619,7 +619,7 @@ services: # Each module carries its own tier identity (consumer, network, provider): # the same keys and registry rows as the multi-adapter services. Signing is # off, as in every quick-start template, so /_internal/ and /publish accept - # unsigned requests: keep :9200 on 127.0.0.1. + # unsigned requests: keep :9201 on 127.0.0.1. network-adapter-layer: <<: *adapter profiles: ["network"] @@ -648,7 +648,7 @@ services: - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - ./config/adapters/routing-network-adapter-provider.yaml:/app/config/routing-network-adapter-provider.yaml:ro ports: - - "127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9200}:9200" + - "127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9201}:9201" # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, From a14273bc7e174ecc542015e232a3667553f0dc03 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 14:08:52 +0530 Subject: [PATCH 37/43] feat(adr): add initial architecture decision records and templates Ref #66 #55 --- docs/adr/0000-template.md | 24 ++++++ ...-single-network-adapter-deployment-mode.md | 79 +++++++++++++++++++ docs/adr/README.md | 12 +++ 3 files changed, 115 insertions(+) create mode 100644 docs/adr/0000-template.md create mode 100644 docs/adr/0001-single-network-adapter-deployment-mode.md create mode 100644 docs/adr/README.md diff --git a/docs/adr/0000-template.md b/docs/adr/0000-template.md new file mode 100644 index 00000000..8751ab3f --- /dev/null +++ b/docs/adr/0000-template.md @@ -0,0 +1,24 @@ +# ADR-0000 — Title + +**Status:** Proposed | Accepted | Superseded by ADR-NNNN +**Date:** YYYY-MM-DD +**Issues:** owner/repo#N + +## Context + +What forced a decision. The constraint, not the preamble. + +## Decision + +What was chosen, stated so that a reader can tell whether a given change +complies with it. + +## Alternatives considered + +What was rejected, and the specific property that disqualified it. "It was +worse" is not a reason; "it exposes an unsigned endpoint" is. + +## Consequences + +What this costs, including the cases it makes harder. An ADR that lists only +benefits is advocacy, not a record. diff --git a/docs/adr/0001-single-network-adapter-deployment-mode.md b/docs/adr/0001-single-network-adapter-deployment-mode.md new file mode 100644 index 00000000..50a4c1a4 --- /dev/null +++ b/docs/adr/0001-single-network-adapter-deployment-mode.md @@ -0,0 +1,79 @@ +# ADR-0001 — One Beckn OAN network adapter process for all three tiers, by default + +**Status:** Proposed +**Date:** 2026-10-15 +**Issues:** OpenAgriNet/helmcharts#55, OpenAgriNet/helmcharts#66, OpenAgriNet/network-adapter#55 (parent OpenAgriNet/engineering-tracker#171) + +## Context + +The Beckn OAN network adapter runs in three tiers: consumer (`bap`, 9202), network (`bpp`, 9201) and provider (`bpp`, 9200). Each was its own container in `quick-start/` and its own Helm release in `charts/adapter-service`, installed once per role. The release name is the Service name the other tiers address. + +So every request crossed two containers or pods, with a network hop and a DNS lookup: +- consumer to network for `discover`; +- consumer to provider for `select`; +- provider to network for `publish`. + +Every environment also ran three of every object: Deployment, Service, ConfigMap, and the PDB or HPA where enabled. + +OAN is single-sided today: our app calls out, and our own discovery and upstream APIs answer. The separate scaling and ownership the split allows is not needed. + +Constraints: +- **The registry cannot update a row.** The identities and participant ids already written must keep working. +- **The published Beckn OAN network adapter image must work unchanged.** + +## Decision + +Two deployment modes from the same chart, compose file, keys and registry rows. **`single` is the default; `multi` (the three adapters, unchanged) is optional.** + +In `single`, one Beckn OAN network adapter process listens on **9201** (the multi network tier's port) and mounts four modules. Each module is its tier's module from the tier configs, with the same steps and its own identity and Secret: + +| Module | Path | Steps | Routes to | +|---|---|---|---| +| consumer | `/` (app calls `/`) | validateSchema, addRoute, sign | `localhost:9201/_internal/network` (discover), `localhost:9201/_internal/provider` (select, init, confirm, status) | +| network | `/_internal/network/` | validateSign, addRoute | discovery | +| provider | `/_internal/provider/` | validateSign, validateSchema, capability steps | answers itself | +| provider-publish | `/publish` | validateSchema, addRoute, sign | `localhost:9201/_internal/network/publish` | + +A change complies with this decision if: +- **Paths:** the paths callers see stay `/discover`, `/publish`, `/` and `/health`, the same as multi. Tier-to-tier paths stay under `/_internal/`, called only by the process itself. +- **Routing:** each routing module keeps its own routing file (`routing-network-adapter-consumer.yaml`, `routing-network-adapter-provider.yaml`; network reuses `routing-network.yaml`). A module's routing file is its list of allowed actions: `addRoute` runs before `sign`, so an action with no rule errors instead of being forwarded. +- **Helm:** the role is `network-adapter-layer`, released once as `network-adapter` in its own namespace, `network-adapter-layer`. Routing is generated at render time from the existing `config/routing-*.yaml`, with hosts rewritten to `localhost`. The render fails on: + - a literal private key; + - `http.port` ≠ `service.targetPort`; + - an unset tier Secret name; + - an Ingress path other than single-segment `/`. +- **Secrets:** `openbao-secrets` reflects the three tier key Secrets and `provider-upstream` into `network-adapter-layer` only. The multi network tier's namespace never receives the consumer or provider keys. +- **Choosing the mode:** a helmfile value `networkAdapterDeploymentMode: single | multi` selects which releases are installed. In quick-start, compose profiles `network` (single) and `multi`, via `make up` and `make up-multi`. + +## Alternatives considered + +- **Keep three adapters only.** It keeps three of every object and a cross-pod hop on every request, for a separation a single-sided deployment does not use. +- **In-process transport (`inproc://`) between modules.** It needs an adapter image that includes the transport, so it fails the "image unchanged" constraint. The gain does not pay for that dependency: the `localhost` hop was already sub-millisecond on Docker, the prototype saved about 0.1–0.3 ms per hop, and an end-to-end benchmark showed no measurable gain. +- **Tier prefixes in the public paths** (`/network/`, `/provider/`, `/provider/publish`). This was the first version. It put the tiers back into the URLs callers use, gave single and multi different URLs, and needed a per-path allow-list at the edge instead of one `/_internal` deny. Replaced after review. +- **One Helm role `network` with a mode switch.** With `single` as the default, every existing `role: network` release would become a three-identity adapter on upgrade and fail without the consumer and provider Secrets. It also makes every existing network-tier code path depend on a new setting. +- **One shared routing file for the consumer and publish modules.** The publish module's rule is for the empty action `""`. Shared, it would also apply to the consumer, so `POST /` on the public consumer would publish into the catalogue. +- **Port 9200.** That is the multi provider's port. 9201 is the multi network tier's port and already the chart's default `service.port`. + +## Consequences + +**Gains:** +- One Deployment, Service, ConfigMap and PDB per environment. +- No hop between pods. +- No registry, identity or image change. +- Reversible: install the multi releases next to the single one, repoint the app, remove the single release. + +**Costs:** +- **Scaling is per adapter, not per tier.** An environment that needs a tier scaled on its own uses `multi`. +- **One crash takes all three tiers down.** Replicas (3 in the example), the PDB and the spread constraint limit the blast radius. +- **Tier configs exist twice.** Each module in `network-adapter-config.yaml` mirrors its tier file, so a change meant for both modes is made in both. +- **The catalogue publisher's port differs by mode:** `:9201/publish` in single, `:9200/publish` in multi. + +**Security:** +- `/publish` has no `validateSign`, and `/_internal/` is internal. Their protection is path-level: + - **Helm:** the Ingress guard; + - **quick-start:** the `server_proxy.conf` denies. +- Inside the cluster, anything that reaches port 9201 can call `/publish`. A NetworkPolicy cannot isolate it, because it has no notion of paths. Port-level isolation would need a second listener, which is not built. + +**Outside this repository:** `OpenAgriNet/network-adapter`'s own `install/` and `config/` still describe the per-role layout. Retiring them is the remaining part of network-adapter#55. + +**Verification:** the k3d and quick-start runs (publish, discover and select 200, and an unsigned call to the network tier refused with 401) predate the flat paths and port 9201. They must be repeated before this record is accepted. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..e4ffadcc --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,12 @@ +# Architecture Decision Records + +One file per decision, numbered, never edited after acceptance. A decision +that turns out wrong is **superseded** by a new record, not rewritten. The +point of the log is to answer *why does this look the way it does*, and a +rewritten record cannot answer it. + +Start from [`0000-template.md`](0000-template.md). + +| # | Decision | Issues | +|---|---|---| +| [0001](0001-single-network-adapter-deployment-mode.md) | One Beckn OAN network adapter process for all three tiers, by default (`single`); three adapters stay optional (`multi`) | helmcharts#55, #66 | From 3295088a632235d86df2869197582b8eb592f4f1 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 14:31:50 +0530 Subject: [PATCH 38/43] fix(adapter-service, quick-start): address PR #75 re-review (#55, #66) - network module: restore schemaValidator verbatim from the network tier file (chart config and quick-start template); narrow the header claim to the modules that validate schema - bootstrap-ubuntu.sh and README Step V1: clone/fetch from development - ci/network-adapter-values.yaml: note preStopSleepSeconds needs k8s 1.30+ Refs #55 Refs #66 --- .../ci/network-adapter-values.yaml | 2 +- .../config/network-adapter-config.yaml | 21 ++++++++++++++++--- quick-start/README.md | 2 +- quick-start/bin/bootstrap-ubuntu.sh | 2 +- .../config/adapters/network-adapter.yaml.tmpl | 21 ++++++++++++++++--- 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index fa187fa0..9d87db1b 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -58,7 +58,7 @@ topologySpreadConstraints: topologyKey: kubernetes.io/hostname whenUnsatisfiable: ScheduleAnyway terminationGracePeriodSeconds: 40 -preStopSleepSeconds: 10 +preStopSleepSeconds: 10 # needs Kubernetes 1.30+; 0 on older clusters ingress: enabled: true diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index b4d54be6..c90cafc5 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -18,7 +18,10 @@ # Differences from the three tier files: # - one listener (9201) and one appName; tier hops call this same listener (see the routing-network-adapter-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); -# - extended schema validation on in every module; +# - extended schema validation on in the modules that validate schema +# (consumer, provider, provider-publish); the network module keeps its +# tier's schemaValidator unchanged (extended off, and validateSchema is not +# one of its steps); # - an outbound connection pool (httpClientConfig) on every module; # - log level info. # @@ -176,8 +179,20 @@ modules: signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" encrPublicKey: "__NETWORK_ENCR_PUBLIC__" - # No schemaValidator: validateSchema is not a step here, and a configured - # validator still downloads the spec at boot and refreshes it hourly. + # The extended layer is off: it fetches a resource's own @context per + # payload, and nothing here needs it. The settings below only apply if + # it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" signValidator: id: signvalidator signer: diff --git a/quick-start/README.md b/quick-start/README.md index c5ce087c..f772941e 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -253,7 +253,7 @@ Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. Nothing is cloned yet, so fetch the script rather than running it from the repo: ```sh -curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash +curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/development/quick-start/bin/bootstrap-ubuntu.sh | bash ``` Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt diff --git a/quick-start/bin/bootstrap-ubuntu.sh b/quick-start/bin/bootstrap-ubuntu.sh index 519c6bf0..87140659 100755 --- a/quick-start/bin/bootstrap-ubuntu.sh +++ b/quick-start/bin/bootstrap-ubuntu.sh @@ -110,7 +110,7 @@ cat <<'NEXT' Then: - git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git + git clone -b development https://github.com/OpenAgriNet/helmcharts.git cd helmcharts/quick-start cp .env.example .env && nano .env # change every credential make up diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 20d70716..43e5692c 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -17,7 +17,10 @@ # Differences from the three tier templates: # - one listener (9201) and one appName; tier hops call this same listener # (routing-network-adapter-*.yaml beside this file); -# - extended schema validation on in every module; +# - extended schema validation on in the modules that validate schema +# (consumer, provider, provider-publish); the network module keeps its +# tier's schemaValidator unchanged (extended off, and validateSchema is not +# one of its steps); # - an outbound connection pool (httpClientConfig) on every module; # - log level info. # @@ -176,8 +179,20 @@ modules: config: contextKeys: transaction_id,message_id role: bpp - # No schemaValidator: validateSchema is not a step here, and a configured - # validator still downloads the spec at boot and refreshes it hourly. + # The extended layer is off: it fetches a resource's own @context per + # payload, and nothing here needs it. The settings below only apply if + # it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" router: id: router config: From aeb41fae076981a9af5631f727d5f37eaed3b15e Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 15 Oct 2026 15:16:33 +0530 Subject: [PATCH 39/43] refactor(quick-start): reuse NETWORK_ADAPTER_PORT for the single adapter (#55, #66) Single mode listens on 9201, the multi network tier's port, and the two modes never run together, so NETWORK_ADAPTER_LAYER_PORT duplicated NETWORK_ADAPTER_PORT. Drop it; container ports stay literal. - .env.example: remove NETWORK_ADAPTER_LAYER_PORT - docker-compose.yml: "127.0.0.1:${NETWORK_ADAPTER_PORT}:9201" - README Step 3 and the single-mode Postman environment name NETWORK_ADAPTER_PORT Refs #55 Refs #66 --- .../OpenAgriNet.network-adapter.local-environment.json | 6 +++--- quick-start/.env.example | 4 ---- quick-start/README.md | 2 +- quick-start/docker-compose.yml | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/api-collection/OpenAgriNet.network-adapter.local-environment.json b/api-collection/OpenAgriNet.network-adapter.local-environment.json index d158a88f..273aa5c7 100644 --- a/api-collection/OpenAgriNet.network-adapter.local-environment.json +++ b/api-collection/OpenAgriNet.network-adapter.local-environment.json @@ -7,21 +7,21 @@ "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", - "description": "Where discover and select enter. Single mode: the one adapter, so all three adapter hosts are the same URL. NETWORK_ADAPTER_LAYER_PORT in .env." + "description": "Where discover and select enter. Single mode: the one adapter, so all three adapter hosts are the same URL. NETWORK_ADAPTER_PORT in .env." }, { "key": "provider-adapter-host", "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", - "description": "Where publish enters. Single mode: the same adapter as consumer-adapter-host. NETWORK_ADAPTER_LAYER_PORT in .env." + "description": "Where publish enters. Single mode: the same adapter as consumer-adapter-host. NETWORK_ADAPTER_PORT in .env." }, { "key": "network-adapter-host", "value": "http://127.0.0.1:9201", "enabled": true, "type": "default", - "description": "Used only by the health check. Single mode: the same adapter; its network tier sits under /_internal/ and is not called from Postman. NETWORK_ADAPTER_LAYER_PORT in .env." + "description": "Used only by the health check. Single mode: the same adapter; its network tier sits under /_internal/ and is not called from Postman. NETWORK_ADAPTER_PORT in .env." }, { "key": "discovery-host", diff --git a/quick-start/.env.example b/quick-start/.env.example index 96f3e084..0440faa0 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -39,10 +39,6 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 CONSUMER_ADAPTER_PORT=9202 -# The Network Adapter (the default deployment mode). Inside the container -# it always listens on 9201 (the multi network tier's port); this only moves -# the host side. -NETWORK_ADAPTER_LAYER_PORT=9201 # Only for looking at a mock directly; the adapter reaches it by service name. MOCKIMD_PORT=9100 MOCKAGMARKNET_PORT=9101 diff --git a/quick-start/README.md b/quick-start/README.md index f772941e..d66ed86c 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -92,7 +92,7 @@ Locally, nothing in it needs changing. It publishes these on `localhost`: ``` If one is already taken, change it here — that is the only edit a local run -needs. The single adapter's port is `NETWORK_ADAPTER_LAYER_PORT`. Adapters reach each other by Compose service name, so it only changes +needs. `NETWORK_ADAPTER_PORT` is the network adapter's port in both modes. Adapters reach each other by Compose service name, so it only changes what you type into Postman. ## Step 4 — Start it diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 23f7d33e..8d58c08f 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -648,7 +648,7 @@ services: - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - ./config/adapters/routing-network-adapter-provider.yaml:/app/config/routing-network-adapter-provider.yaml:ro ports: - - "127.0.0.1:${NETWORK_ADAPTER_LAYER_PORT:-9201}:9201" + - "127.0.0.1:${NETWORK_ADAPTER_PORT}:9201" # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, From 372abe2116a853493f12953318a170feda6eee62 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 22 Oct 2026 12:48:09 +0530 Subject: [PATCH 40/43] refactor(adapter-service): update changelog and configuration for single network adapter mode ref : #55 #66 --- charts/adapter-service/CHANGELOG.md | 4 ++-- charts/adapter-service/config/network-adapter-config.yaml | 4 ---- charts/adapter-service/values.yaml | 4 ++-- docs/adr/0001-single-network-adapter-deployment-mode.md | 2 +- quick-start/README.md | 5 +++++ quick-start/bin/setup.py | 4 +--- quick-start/config/adapters/network-adapter.yaml.tmpl | 6 +----- 7 files changed, 12 insertions(+), 17 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 281a1db0..453cd4b6 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -18,8 +18,8 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h `networkAdapterDeploymentMode: single | multi` (default `single`; a deploy-layer value, not a chart value). - `config/network-adapter-config.yaml`: the layer's adapter config, mounted - like any role's file (extended schema, an outbound connection pool and log - level info on every module). Each module is its tier's module, so a change to + like any role's file (an outbound connection pool and log level info on + every module). Each module is its tier's module, so a change to a tier file that should apply to both deployment modes is made in both. Its routing is generated at render time from `config/routing-*.yaml` (`routing-network-adapter-{consumer,provider}.yaml`), with tier hosts rewritten diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index c90cafc5..acac3abf 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -18,10 +18,6 @@ # Differences from the three tier files: # - one listener (9201) and one appName; tier hops call this same listener (see the routing-network-adapter-*.yaml files, which the chart # generates from routing-consumer.yaml and routing-provider.yaml); -# - extended schema validation on in the modules that validate schema -# (consumer, provider, provider-publish); the network module keeps its -# tier's schemaValidator unchanged (extended off, and validateSchema is not -# one of its steps); # - an outbound connection pool (httpClientConfig) on every module; # - log level info. # diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index c2e27456..828a9943 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -276,8 +276,8 @@ otel: # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" endpoint: "" environment: dev - # Not read by any template: each role's config file sets its telemetry - # service name (config/-config.yaml, telemetry...serviceName). + # Not read by any template: the role's config file sets it + # (plugins.otelsetup.config.serviceName). serviceName: "" logLevel: info diff --git a/docs/adr/0001-single-network-adapter-deployment-mode.md b/docs/adr/0001-single-network-adapter-deployment-mode.md index 50a4c1a4..aab879f8 100644 --- a/docs/adr/0001-single-network-adapter-deployment-mode.md +++ b/docs/adr/0001-single-network-adapter-deployment-mode.md @@ -76,4 +76,4 @@ A change complies with this decision if: **Outside this repository:** `OpenAgriNet/network-adapter`'s own `install/` and `config/` still describe the per-role layout. Retiring them is the remaining part of network-adapter#55. -**Verification:** the k3d and quick-start runs (publish, discover and select 200, and an unsigned call to the network tier refused with 401) predate the flat paths and port 9201. They must be repeated before this record is accepted. +**Verification (2026-10-22, flat paths, port 9201):** quick-start single (11/11 assertions) and multi (10/10), and k3d single (12/12), all passing: publish, discover and select 200, tier hops on `localhost:9201/_internal/`, and on k3d an unsigned call to `/_internal/network/discover` refused with 401. diff --git a/quick-start/README.md b/quick-start/README.md index d66ed86c..92d32692 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -227,6 +227,11 @@ adapters first (both use 9201, and both have a container named `docker compose up -d` follows `COMPOSE_PROFILES` in `.env` (`network` in `.env.example`). +**Upgrading a multi stack.** `make up` now starts single mode: it removes the +9200/9202 adapters, so repoint NPM proxy hosts at `network-adapter:9201`, or +use `make up-multi`. An older `.env` without `COMPOSE_PROFILES` makes a bare +`docker compose up -d` start no adapter; add `COMPOSE_PROFILES=network`. + **How single mode is wired.** `config/adapters/network-adapter.yaml.tmpl`, rendered by `setup.py`, holds four modules, each its tier's module from the tier templates, each with its own tier identity: diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 77259270..c7f3f7ad 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -665,9 +665,7 @@ def render(identities): happens from here. Next: `make up` continues to step 3 and starts the adapters. If you ran this -on its own, the adapters need recreating to pick up the rendered configs: - - docker compose up -d --force-recreate provider-adapter network-adapter consumer-adapter +on its own, run `make up` (or `make up-multi`) to pick up the rendered configs. Then import ../api-collection/ -- both files, the collection and the environment -- and run it. Eighteen requests across the four capabilities, diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 43e5692c..5550b914 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -17,10 +17,6 @@ # Differences from the three tier templates: # - one listener (9201) and one appName; tier hops call this same listener # (routing-network-adapter-*.yaml beside this file); -# - extended schema validation on in the modules that validate schema -# (consumer, provider, provider-publish); the network module keeps its -# tier's schemaValidator unchanged (extended off, and validateSchema is not -# one of its steps); # - an outbound connection pool (httpClientConfig) on every module; # - log level info. # @@ -121,7 +117,7 @@ modules: type: url location: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml cacheTTL: '3600' - extendedSchema_enabled: "true" + extendedSchema_enabled: "false" extendedSchema_cacheTTL: '86400' extendedSchema_maxCacheSize: '100' extendedSchema_downloadTimeout: '30' From 54c1a122e5c8dd808591d7905f921141deedc63b Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 22 Oct 2026 16:07:41 +0530 Subject: [PATCH 41/43] refactor(adapter-service, quick-start): select single/multi with networkAdapterDeploymentMode (#55, #66) Drop the network-adapter-layer role. Role network now reads networkAdapterDeploymentMode (single | multi, default single): single runs the consumer, network and provider tiers as modules of one adapter (the former network-adapter-layer behaviour); multi is the network tier alone. provider and consumer ignore the value; any other value fails the render. adapter-service: - Roles are provider | network | consumer again; the single-mode checks (config, generated routing, tier Secrets, ingress path guard, private-key and port guards) key off role network + mode single. - Single installs as release network-adapter in namespace network-adapter, the same as the multi network tier; switching modes upgrades that release in place. - examples/network.yaml and ci/lint-values.yaml set multi; examples/network-adapter.yaml and ci/network-adapter-values.yaml use role network + single. - BREAKING: an existing role network release without networkAdapterDeploymentMode: multi becomes single on upgrade. - README: new "Multi mode" section, including removing the consumer and provider keys from network-adapter. openbao-secrets: - consumer-adapter-keys, provider-adapter-keys and provider-upstream mirror into network-adapter (single mode) instead of network-adapter-layer. ADR-0001 updated for the role, namespace, secrets and mode. --- charts/adapter-service/CHANGELOG.md | 33 +++--- charts/adapter-service/Chart.yaml | 3 +- charts/adapter-service/README.md | 102 ++++++++---------- charts/adapter-service/ci/lint-values.yaml | 1 + .../ci/network-adapter-values.yaml | 5 +- .../config/network-adapter-config.yaml | 2 +- .../examples/network-adapter.yaml | 14 +-- charts/adapter-service/examples/network.yaml | 6 +- charts/adapter-service/templates/NOTES.txt | 6 +- charts/adapter-service/templates/_helpers.tpl | 15 +-- .../templates/_network-adapter.tpl | 27 +++-- .../adapter-service/templates/configmap.yaml | 12 +-- .../adapter-service/templates/deployment.yaml | 2 +- charts/adapter-service/templates/ingress.yaml | 12 +-- charts/adapter-service/values.yaml | 31 +++--- charts/openbao-secrets/CHANGELOG.md | 12 +-- charts/openbao-secrets/values.yaml | 22 ++-- ...-single-network-adapter-deployment-mode.md | 6 +- 18 files changed, 156 insertions(+), 155 deletions(-) diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md index 453cd4b6..5093db13 100644 --- a/charts/adapter-service/CHANGELOG.md +++ b/charts/adapter-service/CHANGELOG.md @@ -8,15 +8,12 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h ## [0.6.0] - 2026-10-09 ### Added -- **Two deployment modes, with single as the default.** `role: network-adapter-layer` - is the Network Adapter Layer: the consumer, network and provider tiers as four +- **Two deployment modes, with single as the default.** + `networkAdapterDeploymentMode: single | multi` (default `single`), read by + role `network` only. In single, role `network` is the Network Adapter Layer: the consumer, network and provider tiers as four modules of one process (`/` for the consumer, `/publish`, - `/_internal/network/`, `/_internal/provider/`), one release called `network-adapter`. It is the default; - the existing roles `provider`, `network` and `consumer` (one release per - tier) are the optional multi deployment mode and are unchanged. The README's - "Choosing a deployment mode" has the helmfile snippet with - `networkAdapterDeploymentMode: single | multi` (default `single`; a deploy-layer - value, not a chart value). + `/_internal/network/`, `/_internal/provider/`), one release called `network-adapter`. In multi, + `provider`, `network` and `consumer` run one release per tier, unchanged. - `config/network-adapter-config.yaml`: the layer's adapter config, mounted like any role's file (an outbound connection pool and log level info on every module). Each module is its tier's module, so a change to @@ -42,17 +39,17 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h by default. ### Changed +- **Breaking:** `role: network` now defaults to the single deployment mode. + An existing network-tier release must set `networkAdapterDeploymentMode: + multi` (as `examples/network.yaml` does), or the upgrade turns it into the + Network Adapter Layer, which needs the three `networkAdapterLayer.keys` + Secrets. - Docs, `values.yaml` and the example headers say single is the default deployment mode and multi is optional. -- Naming: the layer's release, Service and app name are `network-adapter` - (telemetry service `oan-network-adapter`); its namespace is - `network-adapter-layer`. "layer" appears only in the namespace, the role, - the values key and helper names, where it must differ from the multi - `network` tier. The app name and telemetry service name are set in - `config/network-adapter-config.yaml`. The separate namespace keeps the - consumer and provider keys, which openbao-secrets mirrors to the layer, out - of the multi network tier's namespace, and lets both deployment modes run side by - side during a switch-over. +- Naming: in single mode the release, Service, app name and namespace are + `network-adapter`, the same as the multi network tier's (telemetry service + `oan-network-adapter`, set in `config/network-adapter-config.yaml`). + Switching modes upgrades that release in place. ### Fixed - Liveness and readiness probes now render. Values used `probes.liveness` / @@ -63,7 +60,7 @@ and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.h Pods that had no probe before now get a liveness probe, and the adapter fetches `beckn.yaml` at boot, so without this a slow or proxied egress would get a cold start killed after about 70s. -- NOTES.txt for role `network-adapter-layer` no longer prints `registry.url` and +- NOTES.txt in single mode no longer prints `registry.url` and `otel.*`: that role mounts `config/network-adapter-config.yaml` as is, so those values never reached the pod and NOTES reported the opposite of what it ran. The example and CI values no longer set them. diff --git a/charts/adapter-service/Chart.yaml b/charts/adapter-service/Chart.yaml index aab0f630..b1d7bbb0 100644 --- a/charts/adapter-service/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: adapter-service description: >- One chart for the OAN Beckn adapters, in two deployment modes. single (the - default): the Network Adapter Layer (role network-adapter-layer) runs the + default): the Network Adapter Layer (role network, networkAdapterDeploymentMode single) runs the consumer, network and provider tiers as four modules of one process; tier-to-tier calls stay inside the pod and each module keeps its own tier identity. multi (optional): provider, network and consumer, one release per @@ -26,7 +26,6 @@ keywords: - provider - network - consumer - - network-adapter-layer home: https://github.com/OpenAgriNet/helmcharts sources: - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/adapter-service/README.md b/charts/adapter-service/README.md index 6a23bc56..ac000a2a 100644 --- a/charts/adapter-service/README.md +++ b/charts/adapter-service/README.md @@ -1,9 +1,10 @@ # adapter-service One chart for the Beckn adapters, in two deployment modes. **single (the default):** -the Network Adapter Layer, role `network-adapter-layer`, all three tiers in one -adapter. **multi (optional):** three roles, `provider`, `network` and -`consumer`, one release per tier. All roles are the same image and the same +the Network Adapter Layer: role `network` runs all three tiers in one adapter. +**multi (optional):** roles `provider`, `network` and `consumer`, one release +per tier. `networkAdapterDeploymentMode: single | multi` (default `single`) +picks the mode; only role `network` reads it. All roles are the same image and the same config format, so `role` selects the handler role, the step list and where requests are routed. @@ -16,7 +17,7 @@ and where requests are routed. ``` ``` - network-adapter-layer (one adapter, one pod): + role network, single mode (one adapter, one pod): / ──lo──▶ /_internal/network/ ──▶ discovery └──────lo────▶ /_internal/provider/ ──▶ upstream APIs /publish ──lo──▶ /_internal/network/publish @@ -39,9 +40,9 @@ edge or giving inbound its own hostname. Neither changes these public paths. | Role | Handler | Steps | Routes to | |---|---|---|---| | `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | -| `network` | `bpp` | `validateSign → addRoute → sign` | discovery | +| `network` (multi) | `bpp` | `validateSign → addRoute → sign` | discovery | | `consumer` | `bap` | `addRoute → sign` | network | -| `network-adapter-layer` | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops stay inside the pod | +| `network` (single, default) | `bap` + `bpp` (four modules) | each module runs its own tier's steps | discovery and the upstream APIs; tier hops stay inside the pod | `consumer` is the only one that accepts **unsigned** requests: the consumer app is inside the trust boundary, so there is no network signature to check. @@ -64,15 +65,16 @@ helm dependency build charts/adapter-service ```sh helm upgrade --install network-adapter charts/adapter-service \ - -n network-adapter-layer --create-namespace \ + -n network-adapter --create-namespace \ -f charts/adapter-service/examples/network-adapter.yaml ``` It needs `consumer-adapter-keys`, `network-adapter-keys`, -`provider-adapter-keys` and `provider-upstream` in `network-adapter-layer` +`provider-adapter-keys` and `provider-upstream` in `network-adapter` (openbao-secrets mirrors them there). -**multi (optional): one release per tier, each in its own namespace.** **Keep +**multi (optional): one release per tier, each in its own namespace** (see +[Multi mode](#multi-mode)). **Keep the release and namespace names**: they make the Service DNS name the other adapters address (the consumer routes to `network-adapter.network-adapter.svc` and `provider-adapter.provider-adapter.svc`). @@ -102,7 +104,7 @@ render error rather than a default: | Value | Why there is no default | |---|---| | `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | -| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `network-adapter-layer` uses `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | +| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs. Role `network` in single mode uses `networkAdapterLayer.keys.{consumer,network,provider}.existingSecret` instead (three identities) | | `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | | `role` | It decides the handler role, the step list and the routing target. A default would give one adapter another one's behaviour | | `routing.rules` | An adapter with no route accepts requests and has nowhere to send them | @@ -169,7 +171,7 @@ healthy. | Deployment mode | Releases → namespaces | Values | App calls | Catalogue publishes to | |---|---|---|---|---| -| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter-layer` | `examples/network-adapter.yaml` | `network-adapter.network-adapter-layer.svc:9201/` | `network-adapter.network-adapter-layer.svc:9201/publish` | +| **single (default)**: Network Adapter Layer | `network-adapter` → `network-adapter` | `examples/network-adapter.yaml` | `network-adapter.network-adapter.svc:9201/` | `network-adapter.network-adapter.svc:9201/publish` | | multi (optional) | `consumer-adapter`, `network-adapter`, `provider-adapter` → one namespace each, same name | `examples/{consumer,network,provider}.yaml` | `consumer-adapter.consumer-adapter.svc:9202/` | `provider-adapter.provider-adapter.svc:9200/publish` | **When to use which** @@ -183,7 +185,7 @@ healthy. Both sign as the same three identities from the same registry rows, so the choice changes nothing in the registry. -`role: network-adapter-layer` runs all three tiers in one adapter, as four +Role `network` in single mode runs all three tiers in one adapter, as four modules of one process. Its config is `config/network-adapter-config.yaml` (each module is its tier's module from `config/-config.yaml`, so a change to a tier file that should apply to both deployment modes goes in both files). Its routing @@ -194,51 +196,37 @@ same in both deployment modes. An environment's `config:` override for a multi r seen by the Network Adapter Layer; give its release its own `config:` if needed (it replaces `network-adapter-config.yaml` whole). -Pick the deployment mode per environment with ONE value where releases are chosen, -`networkAdapterDeploymentMode: single | multi` (a helmfile environment value; no chart -reads it; **default `single`**). `single` installs one release, `network-adapter` -in namespace `network-adapter-layer` (the Network Adapter Layer); `multi` -installs the three tier releases. For a helmfile (adapt to -infra-automation's layout): - -```yaml -# environments: values may carry networkAdapterDeploymentMode: multi # default: single -{{- $mode := .Values.networkAdapterDeploymentMode | default "single" }} -{{- if not (has $mode (list "single" "multi")) }} -{{- fail (printf "networkAdapterDeploymentMode must be single or multi, not %q" $mode) }} -{{- end }} -releases: -{{- range $tier := list "consumer" "network" "provider" }} - - name: {{ $tier }}-adapter - namespace: {{ $tier }}-adapter - chart: ../helmcharts/charts/adapter-service - installed: {{ eq $mode "multi" }} - values: [values/{{ $tier }}-adapter.yaml] -{{- end }} - - name: network-adapter - namespace: network-adapter-layer - chart: ../helmcharts/charts/adapter-service - installed: {{ eq $mode "single" }} - values: [values/network-adapter.yaml] -``` - -With Argo CD, gate the same releases on the same value (for example an -ApplicationSet list generator, or `{{ if }}` in an app-of-apps chart). - -**Namespace `network-adapter-layer`.** The Network Adapter Layer is the network -adapter as a whole, so its release, Service and app name are `network-adapter`, -the same as the multi network tier's. Its **namespace** is its own, -`network-adapter-layer`, for two reasons: openbao-secrets mirrors all three -tier identities into it, which must not happen in the multi network tier's -namespace `network-adapter`; and the two deployment modes can then run side by side -during a switch-over. In-cluster address of the layer: -`network-adapter.network-adapter-layer.svc.cluster.local:9201`. - -To switch an environment: install the layer (namespace `network-adapter-layer`) -next to the multi releases, repoint the app (`…:9201/`) and the -catalogue publisher (`…:9201/publish`), then remove the three multi -releases. Both deployment modes sign with the same identities and the same registry -rows, so nothing in the registry changes. Reverse the steps to switch back. +Pick the deployment mode with ONE value, `networkAdapterDeploymentMode: single | +multi` (**default `single`**), on the `network-adapter` release. `single`: that +release is the Network Adapter Layer and nothing else is installed. `multi`: it +is the network tier, and `consumer-adapter` and `provider-adapter` are +installed beside it. `provider` and `consumer` ignore the value. Any other +value fails the render. + +Both modes release `network-adapter` into namespace `network-adapter`, so +switching modes is `helm upgrade` of that release with the other example, +plus installing or removing the other two releases. + +### Multi mode + +1. Set `networkAdapterDeploymentMode: multi` on the network release + (`examples/network.yaml` does). **An existing `role: network` release + without it becomes single on upgrade.** +2. Install the three releases as above. +3. In openbao-secrets, remove `network-adapter` from the namespaces of + `consumer-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, so + the network tier holds only its own key. Then delete any copies already in + the namespace: + + ```sh + kubectl -n network-adapter delete secret consumer-adapter-keys \ + provider-adapter-keys provider-upstream --ignore-not-found + ``` +4. Point the app at `consumer-adapter.consumer-adapter.svc:9202/` and + the catalogue publisher at `provider-adapter.provider-adapter.svc:9200/publish`. + +To return to single, reverse the steps: put the namespaces back, upgrade +`network-adapter` with `examples/network-adapter.yaml`, uninstall the other two. ## Values diff --git a/charts/adapter-service/ci/lint-values.yaml b/charts/adapter-service/ci/lint-values.yaml index d01cdfad..e4a060ee 100644 --- a/charts/adapter-service/ci/lint-values.yaml +++ b/charts/adapter-service/ci/lint-values.yaml @@ -4,6 +4,7 @@ # role is first among them: it has no default, because a default would hand one # adapter another one's handler role and step list. role: network +networkAdapterDeploymentMode: multi fullnameOverride: network-adapter image: diff --git a/charts/adapter-service/ci/network-adapter-values.yaml b/charts/adapter-service/ci/network-adapter-values.yaml index 9d87db1b..4efe2e98 100644 --- a/charts/adapter-service/ci/network-adapter-values.yaml +++ b/charts/adapter-service/ci/network-adapter-values.yaml @@ -2,7 +2,7 @@ # CI only (scripts/lint-charts.sh lints and renders every ci/*-values.yaml). # Not for installs: use examples/network-adapter.yaml. # -# Renders the single deployment mode, role network-adapter-layer (the Network Adapter +# Renders the single deployment mode, role network (the Network Adapter # Layer): config/network-adapter-config.yaml, the generated routing, # the three tier identities, and the PDB / spread / graceful-shutdown # settings, so those templates are linted too. Pinned image tag so the render @@ -12,7 +12,8 @@ # allowed path shape (one-segment /, Exact and Prefix) must render. # ingress.annotations is null on purpose, to cover the `| default dict` on the # annotations merge. -role: network-adapter-layer +role: network +networkAdapterDeploymentMode: single fullnameOverride: network-adapter image: diff --git a/charts/adapter-service/config/network-adapter-config.yaml b/charts/adapter-service/config/network-adapter-config.yaml index acac3abf..f0817118 100644 --- a/charts/adapter-service/config/network-adapter-config.yaml +++ b/charts/adapter-service/config/network-adapter-config.yaml @@ -1,4 +1,4 @@ -# OAN Network Adapter Layer (role network-adapter-layer). The consumer, network +# OAN Network Adapter Layer (role network, single deployment mode). The consumer, network # and provider tiers as four modules of ONE adapter process: # # / the consumer (bap); the app posts here, unsigned diff --git a/charts/adapter-service/examples/network-adapter.yaml b/charts/adapter-service/examples/network-adapter.yaml index 2072e84e..63d9416b 100644 --- a/charts/adapter-service/examples/network-adapter.yaml +++ b/charts/adapter-service/examples/network-adapter.yaml @@ -1,6 +1,6 @@ # =========================================================================== -# network-adapter-layer role -- the Network Adapter Layer: the DEFAULT deployment mode -# (single). The optional alternative is multi: examples/{consumer,network,provider}.yaml. +# role network, single deployment mode -- the Network Adapter Layer, the DEFAULT +# deployment mode. The optional alternative is multi: examples/{consumer,network,provider}.yaml. # # The consumer, network and provider tiers as four modules of ONE adapter # process, on one Service: @@ -14,18 +14,18 @@ # (examples/consumer.yaml, network.yaml, provider.yaml). Run one mode or the # other in an environment, not both. # -# helm install network-adapter . -n network-adapter-layer --create-namespace \ +# helm install network-adapter . -n network-adapter --create-namespace \ # -f examples/network-adapter.yaml # # The config is config/network-adapter-config.yaml; the routing is # generated from config/routing-*.yaml. Tier hops stay inside the pod, so any # adapter image works. # =========================================================================== -role: network-adapter-layer +role: network +networkAdapterDeploymentMode: single -# The Service name and DNS name callers use. The namespace is its own, -# network-adapter-layer (not the multi network tier's network-adapter): -# network-adapter.network-adapter-layer.svc.cluster.local:9201 +# The Service name and DNS name callers use: +# network-adapter.network-adapter.svc.cluster.local:9201 fullnameOverride: network-adapter image: diff --git a/charts/adapter-service/examples/network.yaml b/charts/adapter-service/examples/network.yaml index 3b7eb061..5d4c15f9 100644 --- a/charts/adapter-service/examples/network.yaml +++ b/charts/adapter-service/examples/network.yaml @@ -1,15 +1,17 @@ # =========================================================================== -# network role -- the network layer of the Beckn network. +# network role, multi deployment mode -- the network tier alone. # # Verifies that the caller's signature checks out against the key the registry # publishes for them, hands discover and publish to the discovery service, and # signs on the way out. It answers neither action itself. # -# helm install network-adapter . -f examples/network.yaml +# helm install network-adapter . -n network-adapter -f examples/network.yaml # # Keep the release name `network-adapter`: consumer routes to this Service by name. # =========================================================================== role: network +# Required for the network tier: the default, single, runs all three tiers. +networkAdapterDeploymentMode: multi # The Service name, and therefore the in-cluster DNS name other adapters use. # Set explicitly because fullname is otherwise "-" -- release diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt index 462931b1..151c8bbe 100644 --- a/charts/adapter-service/templates/NOTES.txt +++ b/charts/adapter-service/templates/NOTES.txt @@ -2,10 +2,10 @@ {{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} {{ if $nal }} mode single (the default): the Network Adapter Layer - role network-adapter-layer (all three tiers in one adapter: / consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) + role network, all three tiers in one adapter (/ consumer bap, /publish bap, /_internal/network/ bpp, /_internal/provider/ bpp) hops between tiers inside the pod (http://localhost:{{ .Values.service.targetPort }}/_internal/) {{- else }} - mode multi (optional): one release per tier; the default is role network-adapter-layer + mode multi: one release per tier role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) steps {{ include "adapter-service.steps" . | fromYamlArray | join " -> " }} {{- end }} @@ -77,7 +77,7 @@ on and nothing listening the exporter logs a failure every interval. Ingress is ON. {{- if $nal }} -This is the network-adapter-layer role. / (the consumer) accepts UNSIGNED requests: the consumer tier is +This is the single deployment mode. / (the consumer) accepts UNSIGNED requests: the consumer tier is inside the trust boundary and has no validateSign step, so an Ingress on it exposes an unauthenticated entry point to the network. Rate-limit this host. Rule: only single-segment / paths may be exposed (the render refuses other paths), diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl index 041cf86e..2f494c0f 100644 --- a/charts/adapter-service/templates/_helpers.tpl +++ b/charts/adapter-service/templates/_helpers.tpl @@ -5,9 +5,10 @@ # Purpose: chart-local helpers delegating to common, plus the role, # identity, upstream and config-rendering wiring an adapter needs. # -# One chart, four roles. provider, network and consumer (one release per tier) -# and network-adapter-layer (all three tiers in one adapter) are the same image and the -# same config format; the role decides the handler role, the step list and +# One chart, three roles: provider, network and consumer. Role network with +# networkAdapterDeploymentMode single (the default) runs all three tiers in one +# adapter; in multi it is the network tier and the three roles run one release +# each. Same image and config format; the role decides the handler role, the step list and # where requests are routed. Install once per role, and keep the RELEASE name # per-role -- the release name is what becomes the Service DNS name that the # other adapters address. @@ -15,7 +16,7 @@ */}} {{/* -The adapter's role: provider | network | consumer | network-adapter-layer. +The adapter's role: provider | network | consumer. Required, with no default. A default would silently give one role's step list and handler role to a different adapter -- which renders, starts, reports Ready @@ -24,10 +25,10 @@ and then mis-handles every request, at the far end, in a peer's logs. {{- define "adapter-service.role" -}} {{- $role := .Values.role | default "" -}} {{- if not $role -}} -{{- fail (printf "%s: role is required -- one of provider, network, consumer (the three multi-adapter tiers) or network-adapter-layer (all three tiers in one adapter). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} +{{- fail (printf "%s: role is required -- one of provider, network, consumer (role network runs all three tiers when networkAdapterDeploymentMode is single). It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} {{- end -}} -{{- if not (has $role (list "provider" "network" "consumer" "network-adapter-layer")) -}} -{{- fail (printf "%s: role must be one of provider, network, consumer, network-adapter-layer (got %q)." .Chart.Name $role) -}} +{{- if not (has $role (list "provider" "network" "consumer")) -}} +{{- fail (printf "%s: role must be one of provider, network, consumer (got %q)." .Chart.Name $role) -}} {{- end -}} {{- $role -}} {{- end }} diff --git a/charts/adapter-service/templates/_network-adapter.tpl b/charts/adapter-service/templates/_network-adapter.tpl index 79384315..39b6b358 100644 --- a/charts/adapter-service/templates/_network-adapter.tpl +++ b/charts/adapter-service/templates/_network-adapter.tpl @@ -1,6 +1,6 @@ {{/* # ============================================================================ -# role: network-adapter-layer -- the Network Adapter Layer. +# role network, networkAdapterDeploymentMode single -- the Network Adapter Layer. # # The consumer, network and provider tiers as four modules of ONE adapter # process. The adapter config is config/network-adapter-config.yaml @@ -11,14 +11,27 @@ */}} {{/* -"true" when this release runs the Network Adapter Layer, else empty. +networkAdapterDeploymentMode: single (default) | multi. Only role network +reads it; provider and consumer ignore it. +*/}} +{{- define "adapter-service.deploymentMode" -}} +{{- $mode := .Values.networkAdapterDeploymentMode | default "single" -}} +{{- if not (has $mode (list "single" "multi")) -}} +{{- fail (printf "%s: networkAdapterDeploymentMode must be single or multi (got %q)." .Chart.Name $mode) -}} +{{- end -}} +{{- $mode -}} +{{- end }} + +{{/* +"true" when this release runs the Network Adapter Layer (role network, mode +single), else empty. */}} {{- define "adapter-service.isNetworkAdapterLayer" -}} -{{- if eq (include "adapter-service.role" .) "network-adapter-layer" -}}true{{- end -}} +{{- if and (eq (include "adapter-service.role" .) "network") (eq (include "adapter-service.deploymentMode" .) "single") -}}true{{- end -}} {{- end }} {{/* -Fails unless every private-key VALUE in a network-adapter-layer config is one of the +Fails unless every private-key VALUE in a single-mode config is one of the tier placeholders. Parsed, not grepped: it reads each module's keyManager. Usage: include "adapter-service.networkAdapterLayer.checkKeys" (dict "ctx" $ "cfg" $parsedConfig) */}} @@ -28,7 +41,7 @@ Usage: include "adapter-service.networkAdapterLayer.checkKeys" (dict "ctx" $ "cf {{- range $field := list "signingPrivateKey" "encrPrivateKey" -}} {{- $v := index $km $field | default "" | toString -}} {{- if and $v (not (regexMatch "^__(CONSUMER|NETWORK|PROVIDER)_(SIGNING|ENCR)_PRIVATE__$" $v)) -}} -{{- fail (printf "%s: network-adapter-layer module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} +{{- fail (printf "%s: single-mode module %s sets %s to a value that is not a tier placeholder. A private key written into config reaches git, the ConfigMap and `helm get values`; use ___%s__ and the init container fills it from that tier's Secret." $.ctx.Chart.Name $m.name $field (ternary "SIGNING_PRIVATE" "ENCR_PRIVATE" (eq $field "signingPrivateKey"))) -}} {{- end -}} {{- end -}} {{- end -}} @@ -59,14 +72,14 @@ modules across pods again changes only the hosts. {{- $src := index $pair 0 -}} {{- $doc := $ctx.Files.Get (printf "config/%s" $src) | fromYaml -}} {{- if not $doc.routingRules -}} -{{- fail (printf "%s: config/%s parsed to no routingRules; the network-adapter-layer routing is generated from it." $ctx.Chart.Name $src) -}} +{{- fail (printf "%s: config/%s parsed to no routingRules; the single-mode routing is generated from it." $ctx.Chart.Name $src) -}} {{- end -}} {{- range $i, $rule := $doc.routingRules -}} {{- if $rule.target -}} {{- $url := $rule.target.url | default "" -}} {{- $url = regexReplaceAll $tierHost $url $ownListener -}} {{- if hasSuffix "/" $url -}} -{{- fail (printf "%s: network-adapter-layer routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} +{{- fail (printf "%s: single-mode routing from config/%s routingRules[%d] ends in a slash (%q). The router appends the action, so that produces //discover." $ctx.Chart.Name $src $i $url) -}} {{- end -}} {{- $_ := set $rule.target "url" $url -}} {{- end -}} diff --git a/charts/adapter-service/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml index ea9534fa..aa7fa109 100644 --- a/charts/adapter-service/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -10,7 +10,7 @@ Everything that differs between the provider, network and consumer adapters is a value: they are one image with one config format, and the role decides the identity, the handler role, the step list and where requests are routed. -role network-adapter-layer reads config/network-adapter-config.yaml; its routing +role network in single mode reads config/network-adapter-config.yaml; its routing files are generated from the tier routing files (_network-adapter.tpl), and it renders more than one routing file. */}} @@ -31,20 +31,20 @@ in `helm get values` -- so the render refuses it. {{- $networkAdapterLayer := include "adapter-service.isNetworkAdapterLayer" . }} {{- $nlCfg := "" }} {{- if $networkAdapterLayer }} -{{- /* role network-adapter-layer: the config is config/network-adapter-config.yaml, +{{- /* single mode: the config is config/network-adapter-config.yaml, unless `config` replaces it whole. Either way every private key must be a tier placeholder; checked on the parsed config. */}} {{- $nlCfg = .Values.config | default (.Files.Get "config/network-adapter-config.yaml") }} {{- if not $nlCfg }} -{{- fail (printf "%s: role network-adapter-layer reads config/network-adapter-config.yaml, which is missing from the chart." .Chart.Name) }} +{{- fail (printf "%s: role network in single mode reads config/network-adapter-config.yaml, which is missing from the chart." .Chart.Name) }} {{- end }} {{- $nlParsed := $nlCfg | fromYaml }} {{- if or (not $nlParsed) (hasKey $nlParsed "Error") }} -{{- fail (printf "%s: the network-adapter-layer adapter config (`config` override or config/network-adapter-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} +{{- fail (printf "%s: the single-mode adapter config (`config` override or config/network-adapter-config.yaml) does not parse as YAML, so its private keys cannot be checked." .Chart.Name) }} {{- end }} {{- include "adapter-service.networkAdapterLayer.checkKeys" (dict "ctx" . "cfg" $nlParsed) }} {{- if ne (toString (dig "http" "port" "" $nlParsed)) (toString .Values.service.targetPort) }} -{{- fail (printf "%s: the network-adapter-layer config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} +{{- fail (printf "%s: the single-mode config listens on http.port %v but service.targetPort is %v. The generated routing sends tier hops to localhost:, so the two must match." .Chart.Name (dig "http" "port" "unset" $nlParsed) .Values.service.targetPort) }} {{- end }} {{- else }} {{- $cfgFile := printf "config/%s-config.yaml" (include "adapter-service.role" .) }} @@ -72,7 +72,7 @@ data: adapter.yaml: | {{- $nlCfg | nindent 4 }} {{- if .Values.routing.config }} - {{- fail (printf "%s: routing.config is not used by the network-adapter-layer role. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} + {{- fail (printf "%s: routing.config is not used in the single deployment mode. Its routing files are generated from config/routing-{consumer,network,provider}.yaml. To change a route edit those files; to replace the whole adapter config set `config`." .Chart.Name) }} {{- end }} {{- range $file, $doc := include "adapter-service.networkAdapterLayer.routing" . | fromYaml }} {{ $file }}: | diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index b8664fc2..b59a5256 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -87,7 +87,7 @@ spec: - | set -eu {{- if include "adapter-service.isNetworkAdapterLayer" . }} - # network-adapter-layer: adapter.yaml plus one routing file per routed module. + # single mode: adapter.yaml plus one routing file per routed module. cp /config-template/*.yaml /rendered/ {{- else }} cp /config-template/adapter.yaml /rendered/adapter.yaml diff --git a/charts/adapter-service/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml index ab0044fa..98bacb0d 100644 --- a/charts/adapter-service/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -1,7 +1,7 @@ {{- if .Values.ingress.enabled }} {{- $ann := merge (dict) (.Values.ingress.annotations | default dict) (.Values.commonAnnotations | default dict) }} {{- if include "adapter-service.isNetworkAdapterLayer" . }} -{{- /* role network-adapter-layer: the public surface is flat -- /discover, +{{- /* single mode: the public surface is flat -- /discover, / and /health, all served by the consumer module at /. Two things must never be public: /_internal/ (the network and provider tiers, called only by this process) and /publish (no validateSign, @@ -12,20 +12,20 @@ on ingress-nginx, and regex or rewrite annotations turn paths into patterns, so either could reach /_internal or /publish. */}} {{- if eq (index $ann "nginx.ingress.kubernetes.io/use-regex" | default "" | toString | trim | lower) "true" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed for the network-adapter-layer role. A regex path can match /_internal or /publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/use-regex is not allowed in the single deployment mode. A regex path can match /_internal or /publish." .Chart.Name) }} {{- end }} {{- if hasKey $ann "nginx.ingress.kubernetes.io/rewrite-target" }} -{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed for the network-adapter-layer role. It turns paths into regexes and can rewrite a request onto /_internal or /publish." .Chart.Name) }} +{{- fail (printf "%s: ingress annotation nginx.ingress.kubernetes.io/rewrite-target is not allowed in the single deployment mode. It turns paths into regexes and can rewrite a request onto /_internal or /publish." .Chart.Name) }} {{- end }} {{- range .Values.ingress.hosts }} {{- range .paths }} {{- $p := .path | default "" }} {{- $pt := .pathType | default "Prefix" }} {{- if not (has $pt (list "Prefix" "Exact")) }} -{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed for the network-adapter-layer role. Use Prefix or Exact." $.Chart.Name $p $pt) }} +{{- fail (printf "%s: ingress path %q has pathType %s, which is not allowed in the single deployment mode. Use Prefix or Exact." $.Chart.Name $p $pt) }} {{- end }} {{- if or (not (regexMatch "^/[A-Za-z0-9_.-]+$" $p)) (hasPrefix "/_internal" $p) (eq $p "/publish") }} -{{- fail (printf "%s: ingress path %q is not allowed for the network-adapter-layer role. Allowed: one literal segment, / (e.g. /discover, /select). /, /publish and /_internal* are never public." $.Chart.Name $p) }} +{{- fail (printf "%s: ingress path %q is not allowed in the single deployment mode. Allowed: one literal segment, / (e.g. /discover, /select). /, /publish and /_internal* are never public." $.Chart.Name $p) }} {{- end }} {{- end }} {{- end }} @@ -35,7 +35,7 @@ Off by default, and worth a thought before turning on. The provider and network roles take SIGNED requests -- validateSign is their first step -- so publishing them is not the same risk as publishing an adapter -that does not. The consumer role, and the network-adapter-layer consumer module +that does not. The consumer role, and the single-mode consumer module at /, take UNSIGNED requests (NOTES says so when ingress is on). Either way it is still a Beckn peer endpoint, and the thing that makes it safe is that every caller must hold a key the registry publishes. That is a property of the diff --git a/charts/adapter-service/values.yaml b/charts/adapter-service/values.yaml index 828a9943..bc5f3521 100644 --- a/charts/adapter-service/values.yaml +++ b/charts/adapter-service/values.yaml @@ -6,23 +6,22 @@ # requests are routed. # # single (the DEFAULT): ONE release, the Network Adapter Layer, serving all -# three tiers as modules of one process (role network-adapter-layer): +# three tiers as modules of one process (role network, +# networkAdapterDeploymentMode single): # -# helm install network-adapter . -n network-adapter-layer --create-namespace \ +# helm install network-adapter . -n network-adapter --create-namespace \ # -f examples/network-adapter.yaml # -# multi (OPTIONAL): one release per tier. INSTALL ONCE PER ROLE, and keep the -# RELEASE name per-role: +# multi (OPTIONAL): one release per tier, with networkAdapterDeploymentMode +# multi on the network release. INSTALL ONCE PER ROLE, and keep the RELEASE +# name per-role: # # helm install provider-adapter . -f examples/provider.yaml # helm install network-adapter . -f examples/network.yaml # helm install consumer-adapter . -f examples/consumer.yaml # -# Run one deployment mode per environment. The Network Adapter Layer is released as -# `network-adapter` but in its own namespace, `network-adapter-layer`, so the -# tier identities openbao-secrets mirrors into it never land in the multi -# network tier's namespace, and the two deployment modes can coexist during a -# switch-over. +# Run one deployment mode per environment. Both release `network-adapter` into +# namespace `network-adapter`; switching modes upgrades that release in place. # # The release name is what the Service is called, which is the DNS name the # other adapters address. In multi, rename a release and you must update @@ -40,7 +39,7 @@ # =========================================================================== # --------------------------------------------------------------------------- -# Role: provider | network | consumer | network-adapter-layer +# Role: provider | network | consumer # # Required, with no default. A default would hand one role's handler role and # step list to a different adapter, which renders, starts, reports Ready and @@ -48,6 +47,12 @@ # --------------------------------------------------------------------------- role: "" +# single | multi. Read by role network only: single (the default) runs all +# three tiers in one adapter (networkAdapterLayer below); multi is the network +# tier alone. A multi network release MUST set multi, or an upgrade turns it +# into single. provider and consumer ignore it. +networkAdapterDeploymentMode: single + # Overrides for what the role implies. Leave empty to take the role's default. appName: "" @@ -436,7 +441,7 @@ extraEnv: [] envFromSecrets: [] # --------------------------------------------------------------------------- -# role: network-adapter-layer -- the Network Adapter Layer. +# role network, networkAdapterDeploymentMode single -- the Network Adapter Layer. # # The consumer, network and provider tiers as four modules of ONE process # (/ for the consumer, /publish, /_internal/network/, /_internal/provider/). Its adapter config is @@ -447,9 +452,9 @@ envFromSecrets: [] # listener (http://localhost:/_internal/): any adapter image # works. # -# Ignored by the provider, network and consumer roles. +# Ignored by provider, consumer and a multi network release. # -# NOT USED by this role: `routing.config` (setting it fails the render, because +# NOT USED in single mode: `routing.config` (setting it fails the render, because # the routing is generated) and `handler.*` (role, steps and path are in the # config file). `config` replaces config/network-adapter-config.yaml # WHOLE: the private-key guard still runs on it, and it must point its routers diff --git a/charts/openbao-secrets/CHANGELOG.md b/charts/openbao-secrets/CHANGELOG.md index 8378f538..7e2160b0 100644 --- a/charts/openbao-secrets/CHANGELOG.md +++ b/charts/openbao-secrets/CHANGELOG.md @@ -8,14 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.1] - 2026-10-09 ### Added -- `network-adapter-layer` as a target namespace of `consumer-adapter-keys`, - `network-adapter-keys`, `provider-adapter-keys` and `provider-upstream`, for - `adapter-service` role `network-adapter-layer`: the Network Adapter Layer - (the default deployment mode) runs all three tiers, so it reads all three - identities. It has a namespace of its own rather than the multi network - tier's `network-adapter`, so multi environments do not get the consumer and - provider keys or the upstream credentials copied next to the network tier. - Inert where that namespace does not exist. +- `network-adapter` as a target namespace of `consumer-adapter-keys`, + `provider-adapter-keys` and `provider-upstream`: in the single deployment + mode (the default) `adapter-service` role `network` runs all three tiers + there. In multi mode, remove `network-adapter` from those three entries. ## [0.1.0] - 2026-09-27 diff --git a/charts/openbao-secrets/values.yaml b/charts/openbao-secrets/values.yaml index c8966a59..24ee4686 100644 --- a/charts/openbao-secrets/values.yaml +++ b/charts/openbao-secrets/values.yaml @@ -58,23 +58,21 @@ secrets: # resolve-key-ids.sh AFTER the registry has been seeded, so these entries # change once per environment after the first bring-up. # - # network-adapter-layer is the namespace of the Network Adapter Layer (the - # single deployment mode, role network-adapter-layer): it runs all three tiers, so - # it reads all three identities. It is a namespace of its own, NOT the multi - # network tier's `network-adapter`, so a multi environment never gets the - # consumer and provider keys copied next to the network tier. Reflector - # mirrors only into namespaces that exist, so this is inert where the layer - # is not installed. + # network-adapter runs the Network Adapter Layer in the single deployment + # mode (the default), which runs all three tiers, so it reads all three + # identities. In multi mode drop network-adapter from the consumer and + # provider entries, so the network tier never holds their keys + # (charts/adapter-service/README.md, "Multi mode"). - name: consumer-adapter-keys - namespaces: [consumer-adapter, network-adapter-layer] + namespaces: [consumer-adapter, network-adapter] - name: network-adapter-keys - namespaces: [network-adapter, network-adapter-layer] + namespaces: [network-adapter] - name: provider-adapter-keys - namespaces: [provider-adapter, network-adapter-layer] + namespaces: [provider-adapter, network-adapter] # The provider adapter's upstream credentials, named as environment variables # by its config rather than substituted into it. The Network Adapter Layer - # (namespace network-adapter-layer) runs the provider tier too, so it reads + # (namespace network-adapter, single mode) runs the provider tier too, so it reads # them as well. - name: provider-upstream - namespaces: [provider-adapter, network-adapter-layer] + namespaces: [provider-adapter, network-adapter] diff --git a/docs/adr/0001-single-network-adapter-deployment-mode.md b/docs/adr/0001-single-network-adapter-deployment-mode.md index aab879f8..02afdca8 100644 --- a/docs/adr/0001-single-network-adapter-deployment-mode.md +++ b/docs/adr/0001-single-network-adapter-deployment-mode.md @@ -37,13 +37,13 @@ In `single`, one Beckn OAN network adapter process listens on **9201** (the mult A change complies with this decision if: - **Paths:** the paths callers see stay `/discover`, `/publish`, `/` and `/health`, the same as multi. Tier-to-tier paths stay under `/_internal/`, called only by the process itself. - **Routing:** each routing module keeps its own routing file (`routing-network-adapter-consumer.yaml`, `routing-network-adapter-provider.yaml`; network reuses `routing-network.yaml`). A module's routing file is its list of allowed actions: `addRoute` runs before `sign`, so an action with no rule errors instead of being forwarded. -- **Helm:** the role is `network-adapter-layer`, released once as `network-adapter` in its own namespace, `network-adapter-layer`. Routing is generated at render time from the existing `config/routing-*.yaml`, with hosts rewritten to `localhost`. The render fails on: +- **Helm:** no new role. Role `network` with `networkAdapterDeploymentMode: single` (the default) is the single adapter; with `multi` it is the network tier. `provider` and `consumer` ignore the value. Both modes release `network-adapter` into namespace `network-adapter`. Routing is generated at render time from the existing `config/routing-*.yaml`, with hosts rewritten to `localhost`. The render fails on: - a literal private key; - `http.port` ≠ `service.targetPort`; - an unset tier Secret name; - an Ingress path other than single-segment `/`. -- **Secrets:** `openbao-secrets` reflects the three tier key Secrets and `provider-upstream` into `network-adapter-layer` only. The multi network tier's namespace never receives the consumer or provider keys. -- **Choosing the mode:** a helmfile value `networkAdapterDeploymentMode: single | multi` selects which releases are installed. In quick-start, compose profiles `network` (single) and `multi`, via `make up` and `make up-multi`. +- **Secrets:** `openbao-secrets` reflects the consumer and provider key Secrets and `provider-upstream` into `network-adapter` for single mode. A multi environment removes those targets and deletes the copies, so the network tier holds only its own key. +- **Choosing the mode:** the chart value `networkAdapterDeploymentMode: single | multi` on the `network-adapter` release; multi also installs `consumer-adapter` and `provider-adapter`. In quick-start, `NETWORK_ADAPTER_DEPLOYMENT_MODE` picks `network-adapter`'s config, and profile `multi` adds the other two adapters, via `make up` and `make up-multi`. ## Alternatives considered From 8399f2986d29331d75d8abbce6a23ffd066083c7 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 22 Oct 2026 16:11:40 +0530 Subject: [PATCH 42/43] refactor(deployment): enhance log message for single network adapter mode --- charts/adapter-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/adapter-service/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml index b59a5256..9726b5af 100644 --- a/charts/adapter-service/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -165,7 +165,7 @@ spec: grep -oE '__[A-Z][A-Z0-9_]*__' /rendered/adapter.yaml | sort -u >&2 exit 1 fi - echo "render-config: config rendered for the {{ include "adapter-service.role" . }} role" + echo "render-config: config rendered for the {{ include "adapter-service.role" . }} role{{ if include "adapter-service.isNetworkAdapterLayer" . }} (single mode: all three tiers){{ end }}" volumeMounts: - name: config-template mountPath: /config-template From 04dd0da821ec3a7200f05ab13e0538348e1ae920 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 22 Oct 2026 16:30:50 +0530 Subject: [PATCH 43/43] fix(quick-start): one network-adapter service; the deployment mode picks its config at startup (#55, #66) - docker-compose: merge network-adapter-layer into network-adapter, always on. It mounts config/adapters/${NETWORK_ADAPTER_CONFIG}.yaml: network-adapter.yaml (all three tiers, the default) or network.yaml (the network tier alone). provider- and consumer-adapter stay behind profile multi. Drop COMPOSE_PROFILES=network. - stack.sh: read NETWORK_ADAPTER_DEPLOYMENT_MODE from .env at startup (absent key, quotes, inline comments and CR handled; MODE overrides) and map it to NETWORK_ADAPTER_CONFIG. `up` starts the requested mode; other commands keep the running one, detected from the config network-adapter mounts. Remove a pre-merge network-adapter-layer container before starting. - Only the chosen config is mounted, so in multi the network container never holds the consumer or provider keys. - README and .env.example: modes, switching, upgrade and multi notes. --- quick-start/.env.example | 9 +- quick-start/README.md | 41 ++++--- quick-start/bin/stack.sh | 93 ++++++++------ .../config/adapters/network-adapter.yaml.tmpl | 2 +- quick-start/docker-compose.yml | 115 +++++++----------- 5 files changed, 132 insertions(+), 128 deletions(-) diff --git a/quick-start/.env.example b/quick-start/.env.example index 0440faa0..7adfacdd 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -220,7 +220,8 @@ POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in POCRA_PATH=/search POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/0.0.1/quick-start/config/mappings/pocra/agriculture-facility.select.yaml -# Which adapter deployment mode a bare `docker compose up -d` starts: "network" (the -# single deployment mode: one adapter for all three tiers) or "multi" (one per tier). bin/stack.sh passes -# --profile explicitly and ignores this; it is for plain compose commands. -COMPOSE_PROFILES=network +# Adapter deployment mode, read by bin/stack.sh at startup: "single" (network-adapter +# runs all three tiers, config network-adapter.yaml) or "multi" (network-adapter +# runs config network.yaml, the network tier alone, and provider- and +# consumer-adapter start beside it). MODE=multi on the command line overrides it. +NETWORK_ADAPTER_DEPLOYMENT_MODE=single diff --git a/quick-start/README.md b/quick-start/README.md index 92d32692..920bdfe1 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -208,8 +208,9 @@ collection. → Appendix N. | | single (default) | multi (optional) | |---|---|---| -| What runs | one container, `network-adapter` (compose service `network-adapter-layer`) | `consumer-adapter`, `network-adapter`, `provider-adapter` | -| Compose profile | `network` | `multi` | +| What runs | one container, `network-adapter` | `consumer-adapter`, `network-adapter`, `provider-adapter` | +| `NETWORK_ADAPTER_DEPLOYMENT_MODE` | `single` | `multi` | +| `network-adapter` config | `config/adapters/network-adapter.yaml` | `config/adapters/network.yaml` | | Start | `make up-core` / `make up` | `make up-core-multi` / `make up-multi` | | Ports | `127.0.0.1:9201` | 9200 provider, 9201 network, 9202 consumer | | App calls | `:9201/discover`, `:9201/select` | `:9202/discover`, `:9202/select` | @@ -218,22 +219,28 @@ collection. → Appendix N. Single is the everyday choice: one container, the same requests. Use multi to reproduce a deployment where the tiers run separately. -**Switching** is running the other target. It removes the running mode's -adapters first (both use 9201, and both have a container named -`network-adapter`), then starts the other's. `MODE=multi` also works with -`make up` and `make up-core`; `make restart`, `make down`, `make ps` and -`make logs` follow whichever mode is running. Plain compose: -`docker compose --profile network up -d` (or `--profile multi`); a bare -`docker compose up -d` follows `COMPOSE_PROFILES` in `.env` (`network` in -`.env.example`). - -**Upgrading a multi stack.** `make up` now starts single mode: it removes the -9200/9202 adapters, so repoint NPM proxy hosts at `network-adapter:9201`, or -use `make up-multi`. An older `.env` without `COMPOSE_PROFILES` makes a bare -`docker compose up -d` start no adapter; add `COMPOSE_PROFILES=network`. +**Switching** is running the other target: it recreates `network-adapter` with +the other config, and starts (multi) or removes (single) `provider-adapter` and +`consumer-adapter`. `MODE=multi` also works with `make up` and `make up-core`; +`make restart`, `make down`, `make ps` and `make logs` follow whichever mode is +running. The mode is read once, at startup, by `bin/stack.sh`, which passes +compose the config to mount (`NETWORK_ADAPTER_CONFIG`). Plain compose mounts +`network-adapter.yaml` unless told otherwise: +`NETWORK_ADAPTER_CONFIG=network docker compose --profile multi up -d provider-adapter network-adapter consumer-adapter`. + +**Upgrading a stack.** The compose service `network-adapter-layer` and +`COMPOSE_PROFILES=network` are gone: add `NETWORK_ADAPTER_DEPLOYMENT_MODE=single` +to `.env` (optional; single is the default). `make up` starts single mode +and removes the 9200/9202 adapters, so repoint NPM proxy hosts at +`network-adapter:9201`, or use `make up-multi`. + +**Multi mode.** `make up-multi`, or `NETWORK_ADAPTER_DEPLOYMENT_MODE=multi` in +`.env` and `make up`. Only `network.yaml` is mounted into `network-adapter`, so +the network tier never holds the consumer or provider keys. The app calls +`consumer-adapter:9202`, the catalogue publishes to `provider-adapter:9200`. **How single mode is wired.** `config/adapters/network-adapter.yaml.tmpl`, -rendered by `setup.py`, holds four modules, each its tier's module from the +rendered by `setup.py` to `network-adapter.yaml`, holds four modules, each its tier's module from the tier templates, each with its own tier identity: | Module | Path | Routes to | @@ -807,7 +814,7 @@ an id is a `.so` basename. If `ADAPTER_IMAGE` names a **new tag**, set it before nothing in `stack.sh` pulls, so the stack quietly comes back on the old image: ```sh -docker compose pull network-adapter-layer provider-adapter network-adapter consumer-adapter +docker compose --profile multi pull provider-adapter network-adapter consumer-adapter ``` Build from the adapter repo at the commit the config expects, and check the diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index aea28aca..4c020653 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -37,35 +37,56 @@ cd "$ROOT" # down` leaves the gateway, hyperdx and the adapters (both deployment modes are # profiled) running and then reports success. Naming them on teardown is what # makes "down" mean down. -PROFILES=(--profile reverse-proxy --profile observability --profile network --profile multi) - -# Adapter deployment mode: "single" (default) runs the Network Adapter Layer (compose -# service network-adapter-layer, container network-adapter), one process for -# all three tiers; "multi" runs provider-, network- and consumer-adapter. Both -# publish 127.0.0.1:9201, and the single container shares the multi network -# tier's name, so only one deployment mode exists at a time: the other is REMOVED -# (not just stopped) before this one starts. +PROFILES=(--profile reverse-proxy --profile observability --profile multi) + +# Adapter deployment mode: "single" (default) runs network-adapter as the +# Network Adapter Layer, one process for all three tiers; "multi" runs +# network-adapter as the network tier plus provider- and consumer-adapter +# (profile multi). The mode decides, at startup, which config network-adapter +# mounts (NETWORK_ADAPTER_CONFIG below). +# +# MODE is the mode `up` starts: the shell's MODE, else .env, else single. +# Every other compose call exports the RUNNING mode, so a command that touches +# network-adapter only through depends_on (reverse-proxy, observability) never +# recreates it with the other config. # Validated only by the commands that use it (check_mode), so an unrelated # MODE in the shell never blocks `down`, `destroy` or `help`. -MODE="${MODE:-single}" -check_mode() { - case "$MODE" in single|multi) ;; *) die "MODE must be single or multi, not '$MODE'" ;; esac + +# A key's value in .env: last occurrence, quotes, inline comment and CR +# stripped, empty when absent (never an error under set -e). +dotenv_get() { + local v + v=$(grep -E "^$1=" .env 2>/dev/null | tail -1 | cut -d= -f2-) || true + v=${v%%#*} + printf '%s' "$v" | tr -d '"'"'"' \t\r' } -# The deployment mode whose adapters are running now: single, multi, or empty. -# Both deployment modes have a container called network-adapter, so single is told -# apart by its compose service label. +# The mode network-adapter is running in, from the config it mounts +# (network.yaml is the network tier alone): single, multi, or empty when it is +# not running. running_mode() { - if [ "$(docker inspect -f '{{.State.Running}}' consumer-adapter 2>/dev/null)" = true ]; then echo multi - elif [ "$(docker inspect -f '{{.State.Running}}' network-adapter 2>/dev/null)" = true ] \ - && [ "$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' network-adapter 2>/dev/null)" = network-adapter-layer ]; then - echo single - fi + [ "$(docker inspect -f '{{.State.Running}}' network-adapter 2>/dev/null)" = true ] || return 0 + case "$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/app/config/adapter.yaml"}}{{.Source}}{{end}}{{end}}' network-adapter 2>/dev/null)" in + */network.yaml) echo multi ;; + *) echo single ;; + esac +} + +MODE="${MODE:-$(dotenv_get NETWORK_ADAPTER_DEPLOYMENT_MODE)}" +MODE="${MODE:-single}" +RUNNING_MODE="$(running_mode)" + +# The config network-adapter mounts for a mode; compose reads it as +# NETWORK_ADAPTER_CONFIG. +network_config_for() { [ "$1" = multi ] && echo network || echo network-adapter; } +export NETWORK_ADAPTER_CONFIG="$(network_config_for "${RUNNING_MODE:-$MODE}")" +check_mode() { + case "$MODE" in single|multi) ;; *) die "MODE must be single or multi, not '$MODE'" ;; esac } adapters_for() { # $1 mode case "$1" in - single) echo network-adapter-layer ;; + single) echo network-adapter ;; multi) echo provider-adapter network-adapter consumer-adapter ;; esac } @@ -76,11 +97,6 @@ configs_for() { # $1 mode multi) echo provider network consumer ;; esac } -other_mode() { [ "$1" = single ] && echo multi || echo single; } - -# The compose profile for a deployment mode: single runs under profile "network", -# multi under profile "multi". -profile_for() { [ "$1" = single ] && echo network || echo multi; } # ------------------------------------------------------------------ output @@ -192,19 +208,27 @@ up_adapters() { # new participant id and base URL in .env plus a setup.py re-run, because # the registry cannot repoint an existing row. check_mode - local other f; other=$(other_mode "$MODE") + local f + # `up` starts the requested mode, not the running one. + NETWORK_ADAPTER_CONFIG="$(network_config_for "$MODE")" step 3 "$1" "adapters, mode $MODE ($(adapters_for "$MODE"))" - # The other deployment mode also publishes 127.0.0.1:9201, and the single container - # shares the multi network tier's name: remove it (stopped is not enough). - # shellcheck disable=SC2046 - docker compose --profile "$(profile_for "$other")" rm -sf $(adapters_for "$other") >/dev/null 2>&1 || true + # A stack from before the merge has container network-adapter under the + # old service name network-adapter-layer, which compose no longer knows + # and would not remove: the name would clash. + if [ "$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' network-adapter 2>/dev/null)" = network-adapter-layer ]; then + docker rm -f network-adapter >/dev/null + fi + # Single runs network-adapter alone: remove the multi-only adapters. + if [ "$MODE" = single ]; then + docker compose --profile multi rm -sf provider-adapter consumer-adapter >/dev/null 2>&1 || true + fi # Never let compose create a directory where a bind-mounted config belongs. for f in $(configs_for "$MODE"); do [ -f "config/adapters/$f.yaml" ] \ || die "config/adapters/$f.yaml was not rendered (a directory there? rm -r it, then re-run)" done # shellcheck disable=SC2046 - docker compose --profile "$(profile_for "$MODE")" up -d --force-recreate $(adapters_for "$MODE") + docker compose --profile multi up -d --force-recreate $(adapters_for "$MODE") # NPM resolves adapter names when nginx loads and caches the addresses, so # a recreated or switched adapter 502s through it until it restarts. if [ "$(docker inspect -f '{{.State.Running}}' nginx-proxy-manager 2>/dev/null)" = true ]; then @@ -360,14 +384,13 @@ NEXT # up is a changed image or a changed environment, both of which need the # container recreated -- use `make up` for those. restart_app() { - local running; running=$(running_mode) - [ -n "$running" ] && MODE=$running + [ -n "$RUNNING_MODE" ] && MODE=$RUNNING_MODE check_mode step 1 1 "restarting registry, discovery and the $MODE-mode adapters" info "keycloak, both databases and the edge are left alone" # shellcheck disable=SC2046 - docker compose --profile "$(profile_for "$MODE")" restart sunbird-registry-service discovery-service $(adapters_for "$MODE") - docker compose --profile "$(profile_for "$MODE")" ps --format 'table {{.Service}}\t{{.Status}}' + docker compose --profile multi restart sunbird-registry-service discovery-service $(adapters_for "$MODE") + docker compose --profile multi ps --format 'table {{.Service}}\t{{.Status}}' } # ----------------------------------------------------------------- misc diff --git a/quick-start/config/adapters/network-adapter.yaml.tmpl b/quick-start/config/adapters/network-adapter.yaml.tmpl index 5550b914..7cb75653 100644 --- a/quick-start/config/adapters/network-adapter.yaml.tmpl +++ b/quick-start/config/adapters/network-adapter.yaml.tmpl @@ -1,5 +1,5 @@ # Network Adapter: the consumer, network and provider tiers as -# four modules of ONE adapter process (docker-compose.yml, profile "network"). +# four modules of ONE adapter process (docker-compose.yml service network-adapter, single mode). # # / the consumer (bap); the app posts here, unsigned # /publish the catalogue publishes here diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 8d58c08f..9f41c9fa 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -3,12 +3,6 @@ # # Every service name is -, and every container_name matches it, so # what `docker ps` shows is what `docker logs` and the in-stack DNS name take. -# One exception: the Network Adapter Layer is service `network-adapter-layer` -# (profile network) with container `network-adapter`, the same container name -# as the multi network tier's service `network-adapter`. Only one deployment mode runs -# at a time, so `docker ps` / `docker logs network-adapter` mean whichever runs; -# compose commands (`docker compose logs`, `make logs SVC=`) take the service -# name, `network-adapter-layer` in the single deployment mode. # The sunbird-registry-* three are Sunbird RC's, not OAN's; the keycloak among # them authenticates nothing but that registry, and shares its postgres. # @@ -17,8 +11,8 @@ # registry sunbird-registry-postgres, sunbird-registry-keycloak, # sunbird-registry-service # discovery discovery-postgres, discovery-service -# adapters network-adapter-layer (profile network, default) | -# provider, network, consumer (profile multi) +# adapters network-adapter (all three tiers by default) + +# provider, consumer (profile multi) # observability hyperdx profile: observability # edge nginx-proxy-manager profile: reverse-proxy # @@ -33,8 +27,9 @@ # everything EXCEPT the adapters # 3. python3 bin/setup.py keys, the three tier identities, and # every adapter config they mount -# 4. docker compose --profile network up -d the Network Adapter -# (or --profile multi: provider-, network-, consumer-adapter) +# 4. docker compose up -d network-adapter the Network Adapter +# (multi: NETWORK_ADAPTER_CONFIG=network docker compose --profile multi +# up -d provider-adapter network-adapter consumer-adapter) # 5. docker compose --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack # @@ -541,19 +536,22 @@ services: condition: service_healthy # ========================================================================== - # adapters -- TWO deployment modes, chosen with a compose profile. Same image. + # adapters -- TWO deployment modes, chosen at startup by + # NETWORK_ADAPTER_DEPLOYMENT_MODE in .env (or MODE): bin/stack.sh turns it + # into NETWORK_ADAPTER_CONFIG, the config network-adapter mounts. Same image. # - # profile "network" (the single deployment mode, the default; `make up`) - # network-adapter-layer (container network-adapter): the consumer, network and provider tiers - # as four modules of ONE process, one config, one port (127.0.0.1:9201, the multi network tier's port). - # Tier-to-tier hops stay inside the container. + # single (the default; `make up`) + # network-adapter mounts config/adapters/network-adapter.yaml: the + # consumer, network and provider tiers as four modules of ONE process on + # 127.0.0.1:9201. Tier-to-tier hops stay inside the container. # - # profile "multi" (`make up-multi`) - # provider-adapter, network-adapter, consumer-adapter: one container per - # tier, tier-to-tier hops over the compose network. + # multi (`make up-multi`) + # network-adapter mounts config/adapters/network.yaml (the network tier + # alone), and provider-adapter and consumer-adapter (profile multi) run + # beside it. Only the network tier's config is mounted, so the network + # container never holds the consumer or provider keys. # - # Both publish 127.0.0.1:9201, so run one profile at a time (bin/stack.sh - # stops the other first). Both use the same keys and registry rows. + # Both use the same keys and registry rows. # # Their configs are bind-mounted FILES that do not exist until bin/setup.py # renders them from the .tmpl beside them, which is what the step ordering @@ -606,50 +604,6 @@ services: ports: - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" - # The Network Adapter Layer: the consumer, network and provider tiers as - # modules of ONE adapter process, on ONE config rendered by bin/setup.py from - # config/adapters/network-adapter.yaml.tmpl, with one routing file per - # module. - # - # / the consumer, the caller's entry (bap), takes unsigned requests - # /publish sends the provider's catalogue to /_internal/network/publish - # /_internal/network/* hands discover and publish on to discovery - # /_internal/provider/* calls the upstream, answers synchronously - # - # Each module carries its own tier identity (consumer, network, provider): - # the same keys and registry rows as the multi-adapter services. Signing is - # off, as in every quick-start template, so /_internal/ and /publish accept - # unsigned requests: keep :9201 on 127.0.0.1. - network-adapter-layer: - <<: *adapter - profiles: ["network"] - container_name: network-adapter - depends_on: - sunbird-registry-service: - condition: service_healthy - discovery-service: - condition: service_started - environment: - <<: *adapter-env - OTEL_SERVICE_NAME: network-adapter - # Upstream credentials, read at call time -- the same variables and - # reasons as provider-adapter below. - MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} - AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} - AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} - MOCKAGMARKNET_TOKEN: ${MOCKAGMARKNET_TOKEN:-local-mandi-token} - VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} - VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} - volumes: - - ./config/adapters/network-adapter.yaml:/app/config/adapter.yaml:ro - # One routing file per module. The network module reuses the - # multi-adapter routing-network.yaml unchanged. - - ./config/adapters/routing-network-adapter-consumer.yaml:/app/config/routing-network-adapter-consumer.yaml:ro - - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro - - ./config/adapters/routing-network-adapter-provider.yaml:/app/config/routing-network-adapter-provider.yaml:ro - ports: - - "127.0.0.1:${NETWORK_ADAPTER_PORT}:9201" - # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, # so repointing it at a different upstream is a registry write, not a @@ -700,9 +654,21 @@ services: - "127.0.0.1:${PROVIDER_ADAPTER_PORT}:9200" # Verifies the caller, hands discovery on, re-signs as itself. + # + # In single mode (the default) it is the Network Adapter Layer instead: the + # consumer, network and provider tiers as modules of one process. + # + # / the consumer, the caller's entry (bap), takes unsigned requests + # /publish sends the provider's catalogue to /_internal/network/publish + # /_internal/network/* hands discover and publish on to discovery + # /_internal/provider/* calls the upstream, answers synchronously + # + # Signing is off, as in every quick-start template, so /_internal/ and + # /publish accept unsigned requests: keep :9201 on 127.0.0.1. Every routing + # file and the upstream credentials are mounted in both modes; the multi + # config does not read them. network-adapter: <<: *adapter - profiles: ["multi"] container_name: network-adapter depends_on: sunbird-registry-service: @@ -712,9 +678,19 @@ services: environment: <<: *adapter-env OTEL_SERVICE_NAME: network-adapter + # Upstream credentials for the single-mode provider module, read at call + # time -- the same variables and reasons as provider-adapter. + MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} + AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} + AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} + MOCKAGMARKNET_TOKEN: ${MOCKAGMARKNET_TOKEN:-local-mandi-token} + VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} + VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} volumes: - - ./config/adapters/network.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/${NETWORK_ADAPTER_CONFIG:-network-adapter}.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro + - ./config/adapters/routing-network-adapter-consumer.yaml:/app/config/routing-network-adapter-consumer.yaml:ro + - ./config/adapters/routing-network-adapter-provider.yaml:/app/config/routing-network-adapter-provider.yaml:ro ports: - "127.0.0.1:${NETWORK_ADAPTER_PORT}:9201" @@ -1006,11 +982,8 @@ services: # `make observability` -- NPM was started before the name existed, so the # HyperDX host 502s until it re-resolves. # - # required: false -- only one deployment mode's adapters exist at a time - # (profiles "network" / "multi"); NPM starts with whichever is running. - network-adapter-layer: - condition: service_started - required: false + # required: false -- consumer- and provider-adapter exist only in multi + # mode (profile "multi"); NPM starts with whichever is running. consumer-adapter: condition: service_started required: false