From cee3ee5ce19f1a2bd9804bf91d60876fa40e30c2 Mon Sep 17 00:00:00 2001 From: Prajwal TD <157502711+tdprajwal@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:06:01 +0530 Subject: [PATCH 1/5] feat: host the Network Descriptor and JWKS for network-of-networks [OpenAgriNet/network-adapter#74] Adds networkofnetwork/: two OAN networks on one machine, federated. oanfed.py publish builds each network's Network Descriptor from its registry entry, signs it, and writes it with the JWKS that verifies it. nginx serves both as static files at the fixed paths /.well-known/openagrinet and /.well-known/jwks.json. Signature: Ed25519. Canonical form: RFC 8785 (JCS), covering every field except signature. Generated per-network material (keys, TLS, .env) stays out of git via networkofnetwork/.gitignore. Co-Authored-By: Claude Opus 5.5 (1M context) --- networkofnetwork/.env.example | 229 ++ networkofnetwork/.gitignore | 8 + networkofnetwork/Makefile | 69 + networkofnetwork/bin/network-certs.sh | 70 + networkofnetwork/bin/network-env.py | 211 ++ networkofnetwork/bin/network-ports.py | 108 + networkofnetwork/bin/network.sh | 231 ++ networkofnetwork/bin/oanfed.py | 849 ++++++ networkofnetwork/bin/render-adapter.py | 120 + networkofnetwork/bin/setup.py | 680 +++++ networkofnetwork/config/adapter.yaml.tmpl | 505 ++++ networkofnetwork/config/default.conf.template | 152 ++ networkofnetwork/config/init-crawler-db.sh | 11 + networkofnetwork/config/proxy-headers.inc | 7 + .../config/registry/imports/realm-export.json | 2321 +++++++++++++++++ .../config/registry/schemas/Participant.json | 217 ++ .../registry/schemas/ProviderSchema.json | 81 + .../registry/schemas/SchemaRegistry.json | 35 + networkofnetwork/config/routing-consumer.yaml | 37 + networkofnetwork/config/routing-network.yaml | 10 + networkofnetwork/config/routing-publish.yaml | 12 + networkofnetwork/demo/README.md | 267 ++ networkofnetwork/demo/discover-farmer.json | 19 + networkofnetwork/demo/discover.json | 20 + ...etwork-of-networks.postman_collection.json | 258 ++ .../demo/publish-catalog-mandi.json | 82 + networkofnetwork/demo/publish-catalog.json | 82 + .../demo/select-cross-network.json | 74 + networkofnetwork/demo/select-local.json | 73 + networkofnetwork/demo/select-mandi-local.json | 82 + networkofnetwork/docker-compose.yml | 326 +++ 31 files changed, 7246 insertions(+) create mode 100644 networkofnetwork/.env.example create mode 100644 networkofnetwork/.gitignore create mode 100644 networkofnetwork/Makefile create mode 100755 networkofnetwork/bin/network-certs.sh create mode 100755 networkofnetwork/bin/network-env.py create mode 100755 networkofnetwork/bin/network-ports.py create mode 100755 networkofnetwork/bin/network.sh create mode 100644 networkofnetwork/bin/oanfed.py create mode 100755 networkofnetwork/bin/render-adapter.py create mode 100755 networkofnetwork/bin/setup.py create mode 100644 networkofnetwork/config/adapter.yaml.tmpl create mode 100644 networkofnetwork/config/default.conf.template create mode 100755 networkofnetwork/config/init-crawler-db.sh create mode 100644 networkofnetwork/config/proxy-headers.inc create mode 100644 networkofnetwork/config/registry/imports/realm-export.json create mode 100644 networkofnetwork/config/registry/schemas/Participant.json create mode 100644 networkofnetwork/config/registry/schemas/ProviderSchema.json create mode 100644 networkofnetwork/config/registry/schemas/SchemaRegistry.json create mode 100644 networkofnetwork/config/routing-consumer.yaml create mode 100644 networkofnetwork/config/routing-network.yaml create mode 100644 networkofnetwork/config/routing-publish.yaml create mode 100644 networkofnetwork/demo/README.md create mode 100644 networkofnetwork/demo/discover-farmer.json create mode 100644 networkofnetwork/demo/discover.json create mode 100644 networkofnetwork/demo/oan-network-of-networks.postman_collection.json create mode 100644 networkofnetwork/demo/publish-catalog-mandi.json create mode 100644 networkofnetwork/demo/publish-catalog.json create mode 100644 networkofnetwork/demo/select-cross-network.json create mode 100644 networkofnetwork/demo/select-local.json create mode 100644 networkofnetwork/demo/select-mandi-local.json create mode 100644 networkofnetwork/docker-compose.yml diff --git a/networkofnetwork/.env.example b/networkofnetwork/.env.example new file mode 100644 index 00000000..d5b1fb93 --- /dev/null +++ b/networkofnetwork/.env.example @@ -0,0 +1,229 @@ +# Copy to .env and fill in every value. bin/setup.py refuses +# to touch the registry while any of them is still a placeholder -- the registry +# is append-only, so a row seeded with a placeholder burns that id for good. +# +# One line per variable, and one meaning per name. Where a name carries a +# prefix, the prefix says WHOSE the value is: MAUSAMGRAM_/AGMARKNET_/VISTAAR_/ +# POCRA_ are the four upstream entities, CONSUMER_/NETWORK_/PROVIDER_ are the +# three adapters, REGISTRY_/KEYCLOAK_/DISCOVERY_ are the services. +# +# Every credential below is a shipped default, which means it is public. Change +# all of them before the stack is reachable by anyone but you. The adapter +# keypairs are the exception: setup.py generates those into keys/keys.json. + +# ---- images ---------------------------------------------------------------- +# Pinned, not latest, so what is deployed is in the diff. The adapter image and +# the adapter configs in this repo move together: a config names plugins by id +# and an id is a .so inside the image. +ADAPTER_IMAGE=ghcr.io/openagrinet/network-adapter:0.0.1 +DISCOVERY_IMAGE=ghcr.io/openagrinet/discovery-service:0.0.1 +# Source dir for `docker compose build discovery`; nothing else reads it. +DISCOVERY_CONTEXT=../../discovery-service +# That build's base images. Compose already defaults them to public ones because +# the Dockerfile's own dhi.io/* defaults 401 without a Hardened Images +# subscription. Uncomment only if you have one. +#DISCOVERY_BUILD_IMAGE=dhi.io/golang:1.27-alpine-dev +#DISCOVERY_RUNTIME_IMAGE=dhi.io/static:20260611-musl-alpine +# The two mock upstreams. No capability uses one; `make up` does not start them. +MOCKIMD_IMAGE=ghcr.io/nisargabd/oan-mockimd:latest +MOCKAGMARKNET_IMAGE=ghcr.io/nisargabd/oan-mockagmarknet:latest + +# ---- ports ----------------------------------------------------------------- +# All published on 127.0.0.1 except the reverse proxy's 80/443, and the loopback +# binding is written literally in docker-compose.yml rather than taken from a +# variable. Reach the rest over an SSH tunnel. +REGISTRY_PORT=8081 +KEYCLOAK_PORT=8080 +KEYCLOAK_ADMIN_PORT=9990 +DISCOVERY_PORT=8090 +PROVIDER_ADAPTER_PORT=9200 +NETWORK_ADAPTER_PORT=9201 +CONSUMER_ADAPTER_PORT=9202 +# Only for looking at a mock directly; the adapter reaches it by service name. +MOCKIMD_PORT=9100 +MOCKAGMARKNET_PORT=9101 +# The reverse-proxy profile (nginx-proxy-manager). 80/443 are public, admin is not. +REVERSE_PROXY_HTTP_PORT=80 +REVERSE_PROXY_HTTPS_PORT=443 +NPM_ADMIN_PORT=81 +# The observability profile (hyperdx + clickhouse). +HYPERDX_UI_PORT=8085 +HYPERDX_OTLP_GRPC_PORT=4317 +HYPERDX_OTLP_HTTP_PORT=4318 +CLICKHOUSE_HTTP_PORT=8123 + +# ---- registry tier --------------------------------------------------------- +# CHANGE every credential in this block before the stack is exposed. +REGISTRY_IMAGE_TAG=v2.0.0 +# registry-db. Named REGISTRY_DB_ because discovery-db is a separate database. +REGISTRY_DB_NAME=registry +REGISTRY_DB_USER=postgres +REGISTRY_DB_PASSWORD=postgres +# The Keycloak login setup.py exchanges for the registry write token. +REGISTRY_ADMIN_USER=no-user +REGISTRY_ADMIN_PASSWORD=no-user-password +KEYCLOAK_REALM=sunbird-rc +KEYCLOAK_ADMIN_USER=admin +KEYCLOAK_ADMIN_PASSWORD=admin +# Two Keycloak clients: admin-api mints tokens, registry-frontend logs users in. +KEYCLOAK_ADMIN_CLIENT_ID=admin-api +KEYCLOAK_ADMIN_CLIENT_SECRET=1b46c0f2-6a1c-4a2f-8b0f-2b6b30bcd2b2 +KEYCLOAK_CLIENT_ID=registry-frontend +# Password Keycloak assigns to registry users it provisions. +KEYCLOAK_DEFAULT_USER_PASSWORD=abcd@123 + +# ---- telemetry ------------------------------------------------------------- +# Both components ship to this collector; a bare host:port, no scheme. +OTLP_ENDPOINT=hyperdx:4317 +# The deployment name on every stream, from both components. +OTEL_ENVIRONMENT=dev +# ADAPTERS ONLY, all three signals. false builds no exporter and never dials -- +# set it for `make up-core`, which does not start hyperdx. +ADAPTER_OTEL_ENABLED=true +# DISCOVERY ONLY, and a separate switch: `none` for `make up-core`. +DISCOVERY_OTEL_EXPORTER=otlp + +# ---- discovery ------------------------------------------------------------- +# The network this deployment belongs to; pairs with the spec it validates against. +BECKN_NETWORK_ID=oan-dev +BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml +# Required when DISCOVERY_OTEL_EXPORTER is otlp: it attributes discovery's spans. +# Needs no keypair and no registry row -- discovery does not sign. +DISCOVERY_SUBSCRIBER_ID=discovery.oan.dev +# The sector. Checked against the build's sector list; every OAN component emits it. +DISCOVERY_DOMAIN=Agriculture +# GeoJSON shapes ONE catalog may add to the spatial index; past it, publish is +# PARTIAL and the excess is unsearchable. Count shapes, not resources. +# +# 50000 against the image's default of 256, because that number counts SHAPES +# while the cost it guards is incurred per H3 CELL -- and the two differ by up +# to 8192 apiece. A Point is one cell; a state-sized polygon is the ceiling. So +# a points-only catalog can raise this a long way and a polygon-heavy one +# cannot. The mandi catalog is 272 markets carrying a Point under both +# coverageAreas and market.location: 544 shapes, 544 cells, against the ~2.1M +# cells the default stands in for. Measured 2026-09-12 -- 50000 Points is +# roughly 3 s and 18 MB of index, so this is sized for where the catalog is +# going rather than where it is. +# +# RAISE IT BEFORE A BULK PUBLISH, not after. While a catalog is over the +# ceiling, republishing a resource that sits past the cut-off clears its +# geometry rows and writes none back -- it drops off the map silently. +# +# Interim: discovery-service#32 moves the budget onto cells, where it stops +# depending on what you publish and this line goes away. +DISCOVERY_MAX_GEOMETRIES_PER_CATALOG=50000 +# Requests per second one client may make, and the bucket it may burst to. +# Effectively off: the image defaults to 20/40, which is under any scripted +# sweep, and a throttled request NACKs with AUT_RATE_LIMITED and no `catalogs` +# -- so it reads as an empty result, not as backpressure. Cannot be 0: the +# boot requires burst >= rps > 0. +DISCOVERY_RATE_LIMIT_RPS=100000 +DISCOVERY_RATE_LIMIT_BURST=100000 + +# ---- capability schemas ---------------------------------------------------- +# The ref in OpenAgriNet/network-specs that setup.py builds the four +# SchemaRegistry rows from. Those rows say what a capability MEANS -- the pack +# defining its attributes -- as against the *_MAPPING_URL values below, which +# say how one upstream serves it. A branch, tag or commit; the path is derived +# from the capability code, so schema///attributes.yaml. +# +# GET THIS RIGHT BEFORE THE FIRST RUN. The registry checks the URL's shape and +# never fetches it, so a ref that does not serve the pack stores happily and +# fails at whoever resolves it. It also cannot be corrected: SchemaRegistry +# rejects updates and its delete is soft, keeping the unique index. Today main +# does NOT serve these packs -- they are on schema-packs-v0.1 until that merges. +SCHEMA_PACK_REF=main + +# ---- the three adapter identities ------------------------------------------ +# setup.py registers these three and generates a keypair for each. A participant +# id IS the network identity a signature is verified against, so it must be +# hostname-shaped. Pick them once: the registry cannot update or truly delete a +# row, so an id is never reusable. +CONSUMER_SUBSCRIBER_ID=consumer.oan.dev +NETWORK_SUBSCRIBER_ID=network.oan.dev +PROVIDER_SUBSCRIBER_ID=provider.oan.dev +# The address PEERS are told to POST to, published in each registry row. Nothing +# inside this stack resolves it -- routing here is by compose service name -- so +# a wrong value costs nothing locally and everything on a shared network. +CONSUMER_BASE_URL=https://consumer.oan.dev +NETWORK_BASE_URL=https://network.oan.dev +PROVIDER_BASE_URL=https://provider.oan.dev + +# ---- the four upstream providers ------------------------------------------- +# setup.py creates all four plus a capability binding each -- seven participants +# and four bindings in all. The registry is unreachable from outside the stack, +# so there is no second way to create them. +# +# THE BINDING KEY IS |. The provider adapter builds +# it from the incoming payload and matches it against its own config; setup.py +# renders that config AND seeds the registry from these same values, which is +# what keeps the two from disagreeing. Change one, re-run setup.py. +# A mismatch is a 404, either NET_ENTITY_NOT_FOUND or "no active record". +# +# All four are REAL external APIs -- the stack needs egress to each. + +# WeatherObservation, from IMD Mausamgram. One ready-made header value, "Basic " +# prefix included, sent verbatim (authScheme header) -- IMD issues a single +# opaque token, not a user/password pair. +MAUSAMGRAM_PARTICIPANT_ID=mausamgram +MAUSAMGRAM_CAPABILITY=openagrinet:WeatherObservation +MAUSAMGRAM_BASE_URL=https://mausamgram.imd.gov.in +MAUSAMGRAM_PATH=/nwpapi/get-daily +MAUSAMGRAM_AUTH= +MAUSAMGRAM_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/mausamgram/weather-observation.select.yaml + +# MandiPrice, from Agmarknet. authScheme tokenQuery: the adapter POSTs the +# access name and password to the token URL and sends the token it gets back as +# a query parameter. Those two reach the adapter as environment variables only. +# TOKEN_URL IS ABSOLUTE, not a path: it is rendered into the config verbatim and +# POSTed as given, so a bare path fails with unsupported protocol scheme "". +# KNOWN AND UNFIXED: "no records" comes back as HTTP 400, which the adapter +# treats as permanent -- so an empty market reads as ERROR, not "no prices". +AGMARKNET_PARTICIPANT_ID=agmarknet +AGMARKNET_CAPABILITY=openagrinet:MandiPrice +AGMARKNET_BASE_URL= +AGMARKNET_PATH=/v1/fetch-agmarknet-vistaar +AGMARKNET_TOKEN_URL= +AGMARKNET_ACCESS_NAME= +AGMARKNET_PASSWORD= +AGMARKNET_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/agmarknet/mandi-price.select.yaml + +# KnowledgeAdvisory, from Bharat Vistaar -- the entity that runs the corpus, not +# the docs-pipeline that filled it. authScheme oauth2 client credentials, and a +# POST. The two secrets reach the adapter as environment variables only. +VISTAAR_PARTICIPANT_ID=bharat-vistaar +VISTAAR_CAPABILITY=openagrinet:KnowledgeAdvisory +VISTAAR_BASE_URL= +VISTAAR_PATH=/docs-pipeline-api/search +VISTAAR_TOKEN_URL= +VISTAAR_CLIENT_ID= +VISTAAR_CLIENT_SECRET= +VISTAAR_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/knowledge/knowledge-advisory.select.yaml + +# AgricultureFacility, from POCRA. A POST, and the only capability needing no +# credential -- its registry row publishes authScheme none. +# TWO MEASURED BEHAVIOURS: it takes exactly ONE facility type per call and +# caches on message_id for ten minutes while IGNORING location; and after ~20 +# calls it answers 200 with an empty catalogue, indistinguishable from "nothing +# nearby". That is why searchConcurrency is pinned to 1. Do not raise it. +POCRA_PARTICIPANT_ID=pocra +POCRA_CAPABILITY=openagrinet:AgricultureFacility +POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in +POCRA_PATH=/search +POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/pocra/agriculture-facility.select.yaml +FEDERATION_INTENT={"filters": {"type": "jsonpath", "expression": "$.catalogs[*] ? (@.isActive == true)"}} + +# on|off. off means this network crawls nobody -- it still ANSWERS another +# network's crawl, which is the network module's job and unaffected. +FEDERATION_CRAWL_PEERS=on + +# How long a peer may serve a copy of our catalogues, in seconds. It is also +# what paces their re-crawl: the crawler refreshes at projectionTtl/3, floored +# at 30s, so 30 here means a peer re-crawls us every 30 seconds. Short on +# purpose -- it is what makes a catalog visibly cross during a demo. +FEDERATION_PROJECTION_TTL=30 + +# Upper bound on the re-crawl interval. Recomputed from the peers found on +# each pass, so this is also how long a deployment with no peers at startup +# waits before looking again. +FEDERATION_REFRESH_SECONDS=30 diff --git a/networkofnetwork/.gitignore b/networkofnetwork/.gitignore new file mode 100644 index 00000000..8d7fbe83 --- /dev/null +++ b/networkofnetwork/.gitignore @@ -0,0 +1,8 @@ +# One directory per network, and all of it generated: .env, the keypair, TLS +# material, the rendered adapter config and the signed documents. +# +# Ignored as a WHOLE rather than by pattern -- a rule naming .env and keys/ +# would not catch networks//tls/server.key, and the cost of +# missing one is a private key in a public repository. +networks/ +__pycache__/ diff --git a/networkofnetwork/Makefile b/networkofnetwork/Makefile new file mode 100644 index 00000000..0b26a7cf --- /dev/null +++ b/networkofnetwork/Makefile @@ -0,0 +1,69 @@ +# Network of networks, locally. +# +# Runs a whole OAN network by name, as many times as you like on one machine -- +# each with its own registry, its own discovery, its own adapter and a front +# door the others reach it at. Two of them is federation with nothing stubbed. +# +# make network NETWORK_ID=bharatvistar.oan.local +# make network NETWORK_ID=mahavistara.oan.local +# make ls +# +# Self-contained: its own .env.example, registry schemas, setup and compose +# file. Nothing here reads or writes anything outside this directory. +# +# Anchored to this file's own directory rather than $(PWD), so +# `make -C networkofnetwork network ...` works from the repo root. + +NET := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/network.sh + +.PHONY: help network setup host-descriptor admit down destroy teardown ps logs ls + +# Default target: running a bare `make` in a directory that can delete a +# Postgres volume should print the menu, not pick something from it. +help: + @$(NET) + +# ------------------------------------------------------------- one network + +# Everything: ports, certificates, .env, containers, registry seed, front door. +# Safe to re-run -- it is also how you apply a config change. +network: ; @$(NET) up $(NETWORK_ID) + +# Re-seed the registry and re-render the adapter config only. +setup: ; @$(NET) setup $(NETWORK_ID) + +# Generate and sign the two documents this network HOSTS at its own domain -- +# the descriptor and its jwks -- from its registry entry. Run it after +# `network`, and again whenever the entry changes. +# +# Named for the documents, not for the verb: "publish" on its own reads as +# publishing a CATALOG, which is a different thing entirely and goes to the +# adapter's /publish/ mount. +host-descriptor: ; @$(NET) host-descriptor $(NETWORK_ID) + +# Record our decision to deal with another network: fetch its descriptor over +# TLS, verify the signature against the key it publishes, and store it. +# +# Two networks, so two variables. NETWORK_ID is US -- whose registry is written +# -- and PEER is them. It is one direction only; run it both ways for a pair. +# +# make admit NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +admit: ; @$(NET) admit $(NETWORK_ID) $(PEER) + +down: ; @$(NET) down $(NETWORK_ID) + +# Deletes the data AND releases the ports, so the block can be reused. +destroy: ; @$(NET) destroy $(NETWORK_ID) + +# Destroy EVERY deployed network. Asks first; TEARDOWN_YES=1 skips the prompt. +# +# make teardown +teardown: ; @$(NET) teardown + +ps: ; @$(NET) ps $(NETWORK_ID) +logs: ; @$(NET) logs $(NETWORK_ID) $(SVC) + +# ----------------------------------------------------------- every network + +# What is deployed on this machine, and which ports each one holds. +ls: ; @$(NET) ls diff --git a/networkofnetwork/bin/network-certs.sh b/networkofnetwork/bin/network-certs.sh new file mode 100755 index 00000000..b45d2725 --- /dev/null +++ b/networkofnetwork/bin/network-certs.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Give each network a TLS certificate for its own name. +# +# The contract says a network is reached over https, and the registry's +# Participant schema pins baseUrl to ^https://. Serving plain HTTP locally would +# mean either weakening that schema or storing a URL the stack cannot use -- both +# of which hide the thing a POC is supposed to exercise. +# +# So: one local CA, and a certificate per network signed by it. Peers verify +# against the CA, which is real certificate verification. What a laptop still +# cannot do is prove it OWNS the domain -- that is the one property this setup +# does not and cannot demonstrate, and it is why the CA is trusted explicitly by +# path rather than installed into any trust store. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Everything written here is private key material, so it all lands under +# networks/, which is gitignored as a whole. +NETWORKS="$(dirname "$HERE")/networks" +CA_DIR="$NETWORKS/ca" +DAYS=825 + +mkdir -p "$CA_DIR" + +if [ ! -f "$CA_DIR/ca.crt" ]; then + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$CA_DIR/ca.key" -out "$CA_DIR/ca.crt" -days "$DAYS" \ + -subj "/CN=OAN federation local CA" 2>/dev/null + chmod 600 "$CA_DIR/ca.key" + echo "created the local CA at $CA_DIR/ca.crt" +fi + +for network in "$@"; do + dir="$NETWORKS/$network/tls" + mkdir -p "$dir" + + # The name must be in a subjectAltName: common name alone has not been + # accepted by anything for years, and a cert that fails only at verification + # time is worse than one that fails here. + openssl req -newkey rsa:2048 -nodes \ + -keyout "$dir/server.key" -out "$dir/server.csr" \ + -subj "/CN=$network" 2>/dev/null + + # The upstream names as well as the network's own. This network's mock + # providers are reached at https://., because the registry + # pins a provider's baseUrl to ^https:// just as it does a network's -- so the + # front door has to answer for them, and that means holding their names. + san="DNS:$network" + for upstream in mausamgram agmarknet vistaar pocra; do + san="$san,DNS:$upstream.$network" + done + + openssl x509 -req -in "$dir/server.csr" \ + -CA "$CA_DIR/ca.crt" -CAkey "$CA_DIR/ca.key" -CAcreateserial \ + -out "$dir/server.crt" -days "$DAYS" \ + -extfile <(printf 'subjectAltName=%s\nbasicConstraints=CA:FALSE\n' "$san") 2>/dev/null + + rm -f "$dir/server.csr" + chmod 600 "$dir/server.key" + + # A copy of the CA beside this network's own material, so the compose file + # mounts one directory rather than reaching across to a shared one. The + # adapter trusts it by path, which is what lets an outbound call verify + # another network's certificate with no code that knows about local CAs. + cp "$CA_DIR/ca.crt" "$dir/ca.crt" + echo "issued a certificate for $network" +done + +echo +echo "Other networks verify against $CA_DIR/ca.crt. Pass it with --ca-file." diff --git a/networkofnetwork/bin/network-env.py b/networkofnetwork/bin/network-env.py new file mode 100755 index 00000000..e3f7642e --- /dev/null +++ b/networkofnetwork/bin/network-env.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Write one network's .env, from quick-start/.env.example. + +Every network runs the SAME docker-compose.network.yml. What differs between +them is this file: the network's identity, the host ports it publishes, and +where its own state lives. Everything else -- images, the spec URL, the mapping +URLs -- is read from the example rather than restated here, so this does not +drift when the example gains a variable. + +Ports come from bin/network-ports.py, which allocates a block per network and +remembers it. Nothing here picks a port. + +Upstream capabilities point at this network's own mock servers. The federation +demo never calls them; they exist because bin/setup.py refuses to seed the +registry while any required value is still a placeholder. No real credential is +used, and none belongs in a file like this. + +Run: network-env.py +""" +import pathlib +import socket +import subprocess +import sys + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent +EXAMPLE = ROOT / ".env.example" +# Generated per-network state, in one place and gitignored as a whole -- .env, +# keys, TLS material and the rendered adapter config all carry secrets. +STATE = ROOT / "networks" + +# Every value whose placeholder would stop setup.py's preflight. None is secret. +# +# They must be https: the registry's Participant schema pins baseUrl to +# ^https://..., so an http:// value is refused at seeding time with a validation +# error naming the pattern. That is why these are hostnames under the network +# rather than the "http://mock-imd:9100" the containers actually answer on -- +# the registry record is metadata, and this demo never invokes these upstreams. +MOCK_CREDENTIALS = { + "MAUSAMGRAM_BASE_URL": "https://mausamgram.{network}", + # The path the MOCK serves, not IMD's real /nwpapi/get-daily. The front door + # answers this hostname and proxies to mock-imd, so the path has to be the + # mock's or the call 404s after everything else worked. + "MAUSAMGRAM_PATH": "/get-daily", + "MAUSAMGRAM_AUTH": "Basic bW9jazptb2Nr", + "AGMARKNET_BASE_URL": "https://agmarknet.{network}", + "AGMARKNET_TOKEN_URL": "https://agmarknet.{network}/token", + "AGMARKNET_ACCESS_NAME": "mock", + "AGMARKNET_PASSWORD": "mock", + "VISTAAR_BASE_URL": "https://vistaar.{network}", + "VISTAAR_TOKEN_URL": "https://vistaar.{network}/token", + "VISTAAR_CLIENT_ID": "mock", + "VISTAAR_CLIENT_SECRET": "mock", + "POCRA_BASE_URL": "https://pocra.{network}", +} + +# Upstream participant ids are per network: two networks seeding the same +# registry ids would be two different participants with one name the moment +# anything looked across them. +UPSTREAM_IDS = ("MAUSAMGRAM_PARTICIPANT_ID", "AGMARKNET_PARTICIPANT_ID", + "VISTAAR_PARTICIPANT_ID", "POCRA_PARTICIPANT_ID") + + +def read_example(): + """The example's keys in order, and their values.""" + if not EXAMPLE.exists(): + sys.exit(f"network-env: {EXAMPLE} is missing") + order, values = [], {} + for line in EXAMPLE.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, _, value = stripped.partition("=") + key = key.strip() + order.append(key) + values[key] = value.strip() + return order, values + + +def ports_for(network): + """This network's allocated block, as the variables it fills.""" + out = subprocess.run([sys.executable, str(HERE / "network-ports.py"), "ports", network], + capture_output=True, text=True, check=True).stdout + return dict(line.split("=", 1) for line in out.splitlines() if line) + + +def resolve(values, network): + """One network's values: the example, with what must differ overridden.""" + out = dict(values) + out.update({k: v.format(network=network) for k, v in MOCK_CREDENTIALS.items()}) + out.update(ports_for(network)) + + # The identity. ONE participant: the network IS the party, however many + # layers it serves, so there are no per-role subscriber ids. + out["BECKN_NETWORK_ID"] = network + out["DISCOVERY_SUBSCRIBER_ID"] = f"discovery.{network}" + + # The address another network reaches this one at, and what the descriptor + # publishes as services.discovery. + # + # It ends in /discover, and that is not decoration: the adapter takes the + # ACTION FROM THE URL -- the router strips the module's mount path and + # routes on what is left. A base url ending at /network arrives as an empty + # action and is refused with "endpoint '' is not supported". + # + # https and a hostname because the Participant schema refuses anything else. + # Inside the stack the modules reach each other by localhost, which their + # own config carries. + out["NETWORK_BASE_URL"] = f"https://{network}/network/discover" + + # Where this network's own state lives. Read by docker-compose.network.yml + # for its bind mounts, and by setup.py for keys and the rendered config. + out["NETWORK_DIR"] = str(STATE / network) + + # The crawl's own settings. Defaults rather than questions: a demo that + # asked for a page size before it would start would be asking about the one + # thing it is least likely to need changed. + # The UPPER bound on how often peers are re-crawled. A peer's own + # projectionTtl can shorten it, but never lengthen it -- and the interval is + # recomputed from the peers found on each pass, so a deployment that had NO + # peers when it started waits this long before looking again. 900 meant a + # network admitted after startup was not crawled for fifteen minutes. + out["FEDERATION_REFRESH_SECONDS"] = "30" + out["FEDERATION_PAGE_SIZE"] = "0" + out["FEDERATION_UPDATE_MODE"] = "MERGE" + # How long a peer may serve a copy of our catalogues. The crawler refreshes + # at projectionTtl/3 floored at 30s, so 30 means a peer re-crawls us every + # 30 seconds -- short enough to watch a catalog cross during a demo. + out["FEDERATION_PROJECTION_TTL"] = "30" + # What breadth of a peer's catalogue to mirror: all of it. + # + # The star does the work. Discovery will not take a bare selector -- it + # wants a test, not a projection -- and it refuses a test that narrows + # nothing, so exists(), like_regex, "starts with" and != are all out. But a + # star with ONE equality is fine, and isActive is the one field every + # catalog has. + # + # Deliberately not a list of the capabilities we speak: that would be a + # fifth place the capability set is written, and adding a sixth capability + # would silently stop it being mirrored. + # + # Broad here does not mean mirroring a peer's whole index: context.networkId + # and schemaContext still narrow what comes back. + out["FEDERATION_INTENT"] = '{"filters": {"type": "jsonpath", "expression": "$.catalogs[*] ? (@.isActive == true)"}}' + out["FEDERATION_PUBLISH_RETRY_SECONDS"] = "30" + + # No collector runs here, and discovery REFUSES to start with an exporter + # configured and nowhere to export to. The example turns telemetry on for a + # stack that has hyperdx; this one does not. + out["DISCOVERY_OTEL_EXPORTER"] = "none" + out["ADAPTER_OTEL_ENABLED"] = "false" + + # Built locally, because federation runs ahead of any published image. + out["ADAPTER_IMAGE"] = "oan/network-adapter:local" + + # One label, used where a value has to be short and unique per network. + short = network.split(".")[0] + for key in UPSTREAM_IDS: + if key in out: + out[key] = f"{out[key]}.{short}" + + return out + + +def warn_if_bound(network, resolved): + """Say which of this network's ports something else already holds. + + A warning, not an error: a port can be released between writing this file + and starting the stack, so refusing here would be refusing on stale + information. But finding out now beats finding out from a compose failure + that names the port and not the service. + """ + busy = [] + for name, port in sorted(ports_for(network).items()): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", int(port))) + except OSError: + busy.append(f" {port} {name}") + if busy: + print(f"network-env: these ports are already in use on this machine:") + print("\n".join(busy)) + print("network-env: free them, or `network destroy` whatever holds them.") + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: network-env.py ") + network = argv[1].strip() + if not network: + sys.exit("network-env: which network?") + + order, values = read_example() + resolved = resolve(values, network) + warn_if_bound(network, resolved) + + directory = STATE / network + directory.mkdir(parents=True, exist_ok=True) + path = directory / ".env" + # Keys the example does not have go last, in a stable order, so a + # regenerated file diffs cleanly against the one it replaces. + extra = [k for k in sorted(resolved) if k not in order] + path.write_text("\n".join(f"{k}={resolved[k]}" for k in list(order) + extra if k in resolved) + "\n") + path.chmod(0o600) + print(f"network-env: wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/network-ports.py b/networkofnetwork/bin/network-ports.py new file mode 100755 index 00000000..abb61ea5 --- /dev/null +++ b/networkofnetwork/bin/network-ports.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Give each network its own block of host ports, once, and remember it. + +Two networks on one machine cannot both publish 9200. Each gets a contiguous +block, and the block it got is RECORDED -- so bringing a network back up lands +on the ports it had, and `make network-ls` can say what is where. + +Recorded rather than derived from the name. Hashing a network id into a block +needs no file, but two names can hash to the same block and the only symptom is +a port already in use at start-up, on whichever network happened to come second. +A file costs one more thing to keep, and cannot collide. + +The file is the allocation, not a cache: deleting it and bringing a network back +up moves it to a different block, which is why `network destroy` clears its own +entry rather than the whole file. +""" +import json +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +STATE = HERE.parent / "networks" +ALLOCATIONS = STATE / "ports.json" + +# Where the blocks start, and how wide one is. 100 apart so a block's ports stay +# readable -- 9300, 9301, 9302 belongs to one network at a glance, which matters +# when two stacks are running and a log line names a port. +BASE = 9300 +WIDTH = 100 + +# What a block holds, in order. The names are the .env variables they fill. +PORTS = [ + "ADAPTER_PORT", + "DISCOVERY_PORT", + "REGISTRY_PORT", + "KEYCLOAK_PORT", + "MOCKIMD_PORT", + "MOCKAGMARKNET_PORT", + "FRONT_DOOR_PORT", +] + + +def load(): + if not ALLOCATIONS.exists(): + return {} + return json.loads(ALLOCATIONS.read_text()) + + +def save(allocations): + ALLOCATIONS.parent.mkdir(parents=True, exist_ok=True) + ALLOCATIONS.write_text(json.dumps(allocations, indent=2, sort_keys=True) + "\n") + + +def block_for(network): + """This network's block index, allocating the lowest free one if it has none.""" + allocations = load() + if network in allocations: + return allocations[network] + + taken = set(allocations.values()) + index = 0 + while index in taken: + index += 1 + allocations[network] = index + save(allocations) + return index + + +def ports_for(network): + """The network's ports, as the .env variables they fill.""" + start = BASE + block_for(network) * WIDTH + return {name: start + offset for offset, name in enumerate(PORTS)} + + +def release(network): + """Forget a network's block, so the next one may take it.""" + allocations = load() + if allocations.pop(network, None) is not None: + save(allocations) + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: network-ports.py [network]") + + command = argv[1] + if command == "list": + for network, index in sorted(load().items(), key=lambda kv: kv[1]): + start = BASE + index * WIDTH + print(f"{network:40} {start}-{start + len(PORTS) - 1}") + return 0 + + if len(argv) < 3: + sys.exit(f"usage: network-ports.py {command} ") + network = argv[2] + + if command == "ports": + for name, port in ports_for(network).items(): + print(f"{name}={port}") + return 0 + if command == "release": + release(network) + return 0 + sys.exit(f"unknown command: {command}") + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/network.sh b/networkofnetwork/bin/network.sh new file mode 100755 index 00000000..f5ad2d91 --- /dev/null +++ b/networkofnetwork/bin/network.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# Stand up one whole OAN network, locally, by name. +# +# Run it again with a different name and you have two, side by side, able to +# reach each other -- which is the whole point. Everything that differs between +# them comes from that network's own .env, generated here, so this script is the +# one place that knows how to point the shared compose file at one of them: +# +# COMPOSE_PROJECT_NAME which project the containers belong to +# COMPOSE_ENV_FILES which .env compose itself reads +# QUICKSTART_ROOT where setup.py reads .env and writes keys and configs +# +# Usage: network.sh +# +# up certs, env, config, containers, registry seed, front door +# setup re-seed the registry and re-render configs only +# host-descriptor sign the two documents this network hosts at its own domain +# admit agree to deal with another network: admit +# down stop it, keep the data +# destroy stop it and delete the data, and release its ports +# teardown destroy EVERY deployed network (asks first) +# ps|logs what it is doing +# ls every network on this machine and the ports it holds +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(dirname "$HERE")" +STATE="$ROOT/networks" +COMPOSE="$ROOT/docker-compose.yml" + +# The bridge every network shares. Created once, outside any project, because a +# project-owned network would vanish with whichever project was torn down first. +FEDERATION_BRIDGE=oan-federation + +die() { echo "network: $*" >&2; exit 1; } + +need_network() { + [ -n "${1:-}" ] || die "which network? e.g. make network NETWORK_ID=bharatvistar.oan.local" +} + +# project turns a network id into a compose project name. Docker allows a narrow +# set of characters, and a network id is a domain -- so the dots go. +project() { echo "oan-${1//./-}"; } + +network_dir() { echo "$STATE/$1"; } + +compose() { + local network="$1"; shift + COMPOSE_PROJECT_NAME="$(project "$network")" \ + COMPOSE_ENV_FILES="$(network_dir "$network")/.env" \ + docker compose -f "$COMPOSE" "$@" +} + +ensure_bridge() { + docker network inspect "$FEDERATION_BRIDGE" >/dev/null 2>&1 && return + echo "network: creating the shared bridge $FEDERATION_BRIDGE" + docker network create "$FEDERATION_BRIDGE" >/dev/null +} + +cmd_up() { + local network="$1" + ensure_bridge + # Every directory the compose file bind-mounts, created BEFORE anything + # starts. Docker creates a missing bind-mount source as a ROOT-OWNED + # directory, and then nothing in this script can write the file that was + # supposed to be there -- including on every later run. + mkdir -p "$(network_dir "$network")"/{.well-known,config,tls,keys} + "$HERE/network-certs.sh" "$network" + python3 "$HERE/network-env.py" "$network" + # The core first, and the adapter deliberately NOT yet: its config does not + # exist until setup renders it, and docker creates a DIRECTORY at a bind-mount + # source that is missing. The adapter would then fail with "adapter.yaml is a + # directory" on every restart, for ever, with nothing saying why. + echo "network: starting $network" + compose "$network" up -d registry-postgres registry-keycloak registry-service \ + discovery-postgres discovery-service mock-imd mock-agmarknet + cmd_setup "$network" + echo + echo "network: $network is up" + python3 "$HERE/network-ports.py" ports "$network" | sed 's/^/ /' +} + +cmd_setup() { + local network="$1" + echo "network: seeding the registry for $network" + # Keys, the registry seed and the schema registry. It writes into the + # network's own directory, which is what QUICKSTART_ROOT selects. + QUICKSTART_ROOT="$(network_dir "$network")" python3 "$HERE/setup.py" + # The adapter config, which setup.py deliberately does not render: this + # deployment runs ONE adapter and the three-role templates do not exist here. + python3 "$HERE/render-adapter.py" "$network" + # Now the config exists, so the adapter can start. Recreated rather than + # started, because a container reads its config once and this is also how a + # config change is applied. + compose "$network" up -d --force-recreate adapter wellknown +} + +cmd_host_descriptor() { + local network="$1" + local dir; dir="$(network_dir "$network")" + local descriptor="$dir/.well-known/openagrinet" + + # A revision must only ever INCREASE -- another network refuses a descriptor + # offering one at or below what it already holds, which is what stops a + # withdrawn declaration being replayed as current. So it is derived from the + # last one published rather than asked for: publishing twice without changing + # anything would otherwise be refused by everyone who already has it. + local revision=1 + if [ -f "$descriptor" ]; then + revision=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['revision'] + 1)" "$descriptor") + fi + + # How long another network may serve a copy of our catalogues, and -- the + # half that shows in a demo -- how often it must re-crawl to keep them. The + # crawler refreshes at projectionTtl/3, floored at 30s, so 30 here means a + # peer re-crawls us every 30 seconds. + # + # From the env so a demo can shorten it without editing anything. Absent + # leaves oanfed's own default of an hour. + local ttl_arg=() + local ttl; ttl="$(sed -n 's/^FEDERATION_PROJECTION_TTL=//p' "$dir/.env" 2>/dev/null)" + [ -n "$ttl" ] && ttl_arg=(--projection-ttl "$ttl") + + # The BASE registry url: oanfed appends /api/v1 to it itself. + python3 "$HERE/oanfed.py" publish \ + --network-id "$network" \ + --registry-url "http://127.0.0.1:$(port_of "$network" REGISTRY_PORT)" \ + --key "$dir/keys/network.key" \ + --revision "$revision" \ + "${ttl_arg[@]}" \ + --out "$dir" +} + +port_of() { + python3 "$HERE/network-ports.py" ports "$1" | sed -n "s/^$2=//p" +} + +# admit records OUR decision to deal with another network, in OUR registry. +# +# network.sh admit +# +# --resolve is what makes a network id reachable from this machine: the id is a +# domain that only resolves on the shared bridge, so the request carries it as +# its Host and the connection goes to that network's published front-door port. +# The certificate is still checked against the id, so nothing is relaxed -- only +# where the connection goes. +cmd_admit() { + local us="$1" them="$2" + need_network "${them:-}" + local env_file; env_file="$(network_dir "$us")/.env" + local value; value() { sed -n "s/^$1=//p" "$env_file"; } + + python3 "$HERE/oanfed.py" admit \ + --network-id "$them" \ + --ca-file "$STATE/ca/ca.crt" \ + --resolve "127.0.0.1:$(port_of "$them" FRONT_DOOR_PORT)" \ + --registry-url "http://127.0.0.1:$(port_of "$us" REGISTRY_PORT)" \ + --keycloak-url "http://127.0.0.1:$(port_of "$us" KEYCLOAK_PORT)" \ + --keycloak-issuer "registry-keycloak:8080" \ + --realm "$(value KEYCLOAK_REALM)" \ + --client-id "$(value KEYCLOAK_CLIENT_ID)" \ + --username "$(value REGISTRY_ADMIN_USER)" \ + --password "$(value REGISTRY_ADMIN_PASSWORD)" +} + +cmd_down() { compose "$1" down; } +cmd_ps() { compose "$1" ps; } +cmd_logs() { shift; compose "$@" logs -f --tail=100; } + +# cmd_teardown destroys EVERY deployed network. +# +# A separate command rather than `destroy` with no argument: destroy already +# refuses an empty network id, and making that spelling mean "all of them" +# would turn the commonest typo into the most destructive outcome. +# +# It names what it is about to delete and waits, because the data it removes -- +# every registry, every published catalog, both crawler databases -- cannot be +# rebuilt from this directory. Set TEARDOWN_YES=1 to skip the prompt in a +# script. +cmd_teardown() { + local networks + networks="$(python3 "$HERE/network-ports.py" list | awk '{print $1}')" + + if [ -z "$networks" ]; then + echo "nothing deployed" + return 0 + fi + + echo "this will DESTROY, with their data and ports:" + echo "$networks" | sed 's/^/ /' + + if [ "${TEARDOWN_YES:-}" != "1" ]; then + printf "type yes to continue: " + local answer; read -r answer + [ "$answer" = "yes" ] || { echo "nothing done"; return 1; } + fi + + local network + for network in $networks; do + cmd_destroy "$network" + done +} + +cmd_destroy() { + local network="$1" + compose "$network" down -v + python3 "$HERE/network-ports.py" release "$network" + rm -rf "$(network_dir "$network")" + echo "network: $network destroyed, and its ports released" +} + +cmd_ls() { python3 "$HERE/network-ports.py" list; } + +main() { + local command="${1:-}" + case "$command" in + ls) cmd_ls ;; + up|setup|down|destroy|ps) + need_network "${2:-}"; "cmd_$command" "$2" ;; + # Its own line rather than joining the list above: that branch dispatches + # to "cmd_$command", and a hyphen cannot appear in a shell function name. + host-descriptor) + need_network "${2:-}"; cmd_host_descriptor "$2" ;; + teardown) cmd_teardown ;; + admit) need_network "${2:-}"; cmd_admit "$2" "${3:-}" ;; + logs) need_network "${2:-}"; compose "$2" logs -f --tail=100 "${3:-}" ;; + *) sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//' ;; + esac +} + +main "$@" diff --git a/networkofnetwork/bin/oanfed.py b/networkofnetwork/bin/oanfed.py new file mode 100644 index 00000000..0d563c49 --- /dev/null +++ b/networkofnetwork/bin/oanfed.py @@ -0,0 +1,849 @@ +#!/usr/bin/env python3 +"""oanfed -- the operator's tool for federation. + +A network publishes exactly two files, at locations derived from its own domain: + + https://{networkId}/.well-known/openagrinet the signed Network Descriptor + https://{networkId}/.well-known/jwks.json the keys that verify it + +These are publication points, not APIs. Serving a static file and generating the +same JSON are equally valid, and nothing here assumes which. + +Admitting a peer is a DECISION, not a request, so it is a command an operator +runs and not an endpoint the adapter exposes. An adapter endpoint would mean +anything able to reach the adapter could widen who this network deals with, +which is exactly the thing admission exists to control. + +It lives beside setup.py because it does the same job: both mint a Keycloak +token and write Participant records into this network's registry. setup.py seeds +the network's own participants; this admits another network as a peer. + + oanfed.py publish generate and sign this network's two published documents + oanfed.py verify fetch another network's documents and check them + oanfed.py admit record the decision to deal with a peer; also the refresh + oanfed.py suspend stop dealing with a peer + oanfed.py reinstate undo a suspension; a peer cannot do this for itself + +Run a command with -h for its flags. +""" + +import argparse, base64, http.client, json, pathlib, ssl, sys, urllib.error, urllib.parse, urllib.request +from datetime import datetime, timezone + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives import serialization as ser + +# The two fixed paths. They are NOT configurable: a peer derives them from the +# networkId alone, which is the whole reason a network can be found knowing +# nothing but its domain. +WELL_KNOWN_DESCRIPTOR = "/.well-known/openagrinet" +WELL_KNOWN_JWKS = "/.well-known/jwks.json" + +# Caps what will be read from a peer. These documents are small and a peer is +# not yet trusted -- it has not been admitted at the point they are fetched. +# Without a cap, the first thing an unknown host can do to us is stream until we +# run out of memory. +MAX_DOCUMENT_BYTES = 1 << 20 + +# The only algorithm this implementation signs or verifies with. The contract +# leaves the choice open; narrowing it here means an unexpected value is refused +# rather than quietly treated as this one. +ALGORITHM = "Ed25519" + +# The kid on this network's one published key. Stable, because a descriptor's +# signature points at it and a peer resolves the key by it. +KEY_ID = "network-1" + +ROLE_NETWORK = "network" + + +def as_role_list(role): + """A participant's role, however it was written. + + A list is what the schema now wants; a plain string is what older records + carry, and nothing rewrites a record just to change its shape. + """ + if role is None: + return [] + return list(role) if isinstance(role, list) else [role] + + +def has_role(record, role): + return any(str(r).lower() == role.lower() for r in as_role_list(record.get("role"))) +STATUS_ACTIVE, STATUS_INACTIVE = "active", "inactive" + + +# --- canonical form ------------------------------------------------------- + +def canonical(descriptor): + """The exact bytes that get signed: RFC 8785 (JCS). + + The signature is excluded, because it cannot cover itself. Everything else + is included: narrowing the covered set would let an unsigned field be + changed in flight, and the fields most worth tampering with -- status, + revision, the discovery URL -- are exactly the ones a smaller set would be + tempted to omit. + + JCS, and not "whatever this language's JSON encoder happens to emit", + because the point of a canonical form is that a DIFFERENT implementation + rebuilds the same bytes. An encoder-specific form works perfectly until + someone else writes the second implementation, and then fails every + signature check on documents nobody tampered with. + + Our descriptors contain only strings, integers, arrays and objects, so the + parts of JCS that are genuinely hard -- number formatting -- do not arise. + What matters here is: keys sorted, no insignificant whitespace, UTF-8, and + no escaping beyond what JSON requires. + """ + without_signature = {k: v for k, v in descriptor.items() if k != "signature"} + return json.dumps( + without_signature, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + +# --- keys ----------------------------------------------------------------- + +def b64url(raw): + """base64url WITHOUT padding, which is what RFC 7517 says a JWKS carries.""" + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def unb64url(encoded): + return base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + + +def registry_key(jwk_x): + """A JWKS key in the form the registry stores. + + A JWKS carries base64url without padding. The registry's PublicKey schema + pins ^[A-Za-z0-9+/]{43}=$, which is standard base64 WITH padding. Both are + correct for their own format, so something has to convert -- and it is one + function rather than an inline re-encode at each call site, because getting + it wrong produces a key that is silently the wrong bytes rather than an + error. + """ + return base64.b64encode(unb64url(jwk_x)).decode() + + +def load_or_create_key(path): + """Read a private key, generating one if the file is absent. + + Stored as the raw 32-byte seed, base64url-unpadded -- the same encoding the + public half uses in the JWKS, so an operator reading either file sees one + format rather than two. + """ + path = pathlib.Path(path) + if path.exists(): + raw = unb64url(path.read_text().strip()) + # 32 is the seed, which is what this tool writes. 64 is Go's + # ed25519.PrivateKey -- the seed followed by the public half -- which is + # what the Go implementation this replaces wrote, and what every key + # already on disk is. Both are read so an existing network does not have + # to regenerate its identity to move tools. + if len(raw) == 64: + raw = raw[:32] + if len(raw) != 32: + raise FederationError( + f"{path} holds {len(raw)} bytes; an Ed25519 private key is 32 (a seed) " + f"or 64 (a seed followed by its public half)") + return Ed25519PrivateKey.from_private_bytes(raw) + + private = Ed25519PrivateKey.generate() + raw = private.private_bytes( + encoding=ser.Encoding.Raw, + format=ser.PrivateFormat.Raw, + encryption_algorithm=ser.NoEncryption(), + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(b64url(raw) + "\n") + path.chmod(0o600) + return private + + +def public_of(private): + return private.public_key().public_bytes( + encoding=ser.Encoding.Raw, format=ser.PublicFormat.Raw + ) + + +# --- sign and verify ------------------------------------------------------ + +class FederationError(Exception): + """Anything a caller is expected to act on rather than crash over.""" + + +class StaleRevision(FederationError): + """The descriptor does not advance the revision. + + Its own type because admit DOUBLES AS THE REFRESH JOB: when it runs on a + schedule, finding nothing new is the normal outcome, not a failure. Treating + it as an error would mean a cron that alerts on every quiet run -- and an + operator who stops reading the alerts. + + A LOWER revision is a replay and a real refusal; the same revision is just + nothing to do. Both arrive here, and the caller tells them apart by + comparing the numbers it already holds. + """ + + +def sign(descriptor, key_id, private): + """Return a copy of the descriptor carrying its signature. + + The governance key signs this and nothing else. The contract forbids it from + signing ordinary traffic, which is why admission deliberately keeps it out + of the registry record a peer's calls are verified against. + """ + if not descriptor.get("networkId"): + raise FederationError("descriptor carries no networkId") + signed = dict(descriptor) + signed["signature"] = { + "keyId": key_id, + "algorithm": ALGORITHM, + "value": b64url(private.sign(canonical(descriptor))), + } + return signed + + +def verify(descriptor, keyset): + """Check a descriptor against the keys its network published. + + It answers ONE question -- did the holder of a published key write this + document -- and deliberately not whether the network should be trusted. The + keys came from the same domain as the document, so a network can always + verify its own lies. Admission is what decides trust. + """ + signature = descriptor.get("signature") + if not signature: + raise FederationError("descriptor carries no signature") + if signature.get("algorithm") != ALGORITHM: + raise FederationError(f"unsupported signature algorithm: {signature.get('algorithm')!r}") + + key_id = signature.get("keyId") + jwk = next((k for k in keyset.get("keys", []) if k.get("kid") == key_id), None) + if jwk is None: + raise FederationError(f"no published key matches the signature's keyId {key_id!r}") + + raw = unb64url(jwk["x"]) + if len(raw) != 32: + raise FederationError(f"key {key_id!r} is {len(raw)} bytes, want 32") + + try: + Ed25519PublicKey.from_public_bytes(raw).verify( + unb64url(signature["value"]), canonical(descriptor) + ) + except Exception as err: + raise FederationError("signature does not verify") from err + + +# --- fetching a peer's documents ------------------------------------------ + +class ResolvingHTTPSConnection(http.client.HTTPSConnection): + """Connects to a given address while still verifying the peer's own name. + + The way `curl --resolve` works. The request still carries the networkId as + its Host and the certificate is still checked against it, so nothing about + verification is relaxed -- only where the connection goes. It exists because + a network id is a DNS domain, and an operator's own machine usually cannot + resolve one belonging to a stack running in containers. + """ + + def __init__(self, server_name, address, **kwargs): + host, _, port = address.partition(":") + super().__init__(host, int(port or 443), **kwargs) + self._server_name = server_name + + def connect(self): + super(http.client.HTTPSConnection, self).connect() + self.sock = self._context.wrap_socket(self.sock, server_hostname=self._server_name) + + +class Fetcher: + """Retrieves and verifies another network's published documents. + + scheme is "https" in any real deployment. It is settable because a local POC + has no certificate authority and cannot prove domain ownership -- which is + precisely the property a real deployment relies on, so this is a knob that + must never be turned in one. + """ + + def __init__(self, scheme="https", ca_file=None, resolve=None, timeout=10): + self.scheme, self.resolve, self.timeout = scheme or "https", resolve, timeout + # Real certificate verification against a DIFFERENT root, not skipping + # it: a peer presenting a certificate this CA did not sign, or one whose + # name does not match, is still refused. A local stack has no publicly + # trusted certificate, and the alternative -- turning verification off -- + # would quietly remove the check in the one place a POC is meant to + # prove it. + self.context = ssl.create_default_context(cafile=ca_file) if ca_file else None + + def fetch(self, network_id): + """Retrieve a network's descriptor and keys, and verify one against the other. + + A verified descriptor is a CANDIDATE, not a trusted peer. Both documents + came from the same host, so this proves self-consistency and nothing + more -- any network can sign its own claims. Admission is the separate, + local decision that turns a candidate into a peer. + """ + if not network_id: + raise FederationError("no networkId given") + + descriptor = self._get(network_id, WELL_KNOWN_DESCRIPTOR) + keyset = self._get(network_id, WELL_KNOWN_JWKS) + + # The descriptor says who it is for. A document served at one domain + # that names another is either a misconfiguration or an attempt to be + # admitted under someone else's name, and neither should get further. + if descriptor.get("networkId") != network_id: + raise FederationError( + f"{network_id} publishes a descriptor for {descriptor.get('networkId')!r}") + verify(descriptor, keyset) + return descriptor, keyset + + def _get(self, network_id, path): + where = f"{self.scheme}://{network_id}{path}" + try: + if self.scheme == "https": + connection = ( + ResolvingHTTPSConnection(network_id, self.resolve, + context=self.context, timeout=self.timeout) + if self.resolve else + http.client.HTTPSConnection(network_id, context=self.context, + timeout=self.timeout) + ) + else: + host = self.resolve or network_id + connection = http.client.HTTPConnection(host, timeout=self.timeout) + + connection.request("GET", path, headers={"Accept": "application/json", + "Host": network_id}) + response = connection.getresponse() + if response.status != 200: + raise FederationError(f"{where} returned {response.status} {response.reason}") + body = response.read(MAX_DOCUMENT_BYTES) + except FederationError: + raise + except Exception as err: + raise FederationError(f"fetching {where}: {err}") from err + finally: + try: + connection.close() + except Exception: + pass + + try: + return json.loads(body) + except json.JSONDecodeError as err: + raise FederationError(f"parsing {where}: {err}") from err + + +# --- admission ------------------------------------------------------------ + +DEFAULT_PROJECTION_TTL = 3600 # seconds; finite, and deliberately short + + +def projection_ttl(descriptor): + """What a descriptor permits, in seconds. + + The three cases are genuinely different and none may be folded into + another: absent is consent to decide for ourselves, zero is a refusal to be + cached, and a positive value is a bounded licence. + """ + declared = descriptor.get("projectionTtl") + if declared is None: + return DEFAULT_PROJECTION_TTL + if declared < 0: + raise FederationError(f"projectionTtl is negative: {declared}") + if declared == 0: + raise FederationError( + "peer declares projectionTtl 0 (live query only), which is not implemented") + return declared + + +def published_keys(keyset): + """Every key the peer published. + + Nothing is stripped. An earlier design dropped the key that signed the + descriptor, so a peer could not sign ordinary traffic with it -- that split + is gone, and a network now publishes ONE key which does both jobs. + """ + return list(keyset.get("keys", [])) + + +# What an entry is in relation to us. +# +# self this deployment's own entry +# source a peer we consume from -- our crawler asks it for its catalogs +# requester a peer that consumes from us -- it calls us and we verify its +# signatures, but we do not crawl it +# both a peer in both directions, which is what mutual admission produces +FEDERATION_ROLES = ("self", "source", "requester", "both") +DEFAULT_FEDERATION_ROLE = "both" + + +def admit_record(current, fresh, keyset, federation_role=DEFAULT_FEDERATION_ROLE): + """Turn a verified descriptor into the registry record to write. + + current is None on first admission; on every later call it is what the + registry already holds, which is what makes the revision check possible. + + Two rules, and each exists because of a specific way a peer could otherwise + take something back or take something over: + + - The revision must increase, so a superseded descriptor cannot be + replayed. That, plus the documents being served from the network's own + domain, is what the contract rests identity on. + - Status is carried over and never read from the descriptor, so a peer an + operator suspended cannot reinstate itself by publishing a new file. + What the peer says about itself is a claim; what we recorded is a + decision. + """ + network_id = fresh.get("networkId") + if not network_id: + raise FederationError("descriptor carries no networkId") + + ttl = projection_ttl(fresh) + status = STATUS_ACTIVE + if current is not None: + if (current.get("participantId") or "").lower() != network_id.lower(): + raise FederationError( + f"descriptor is for {network_id!r}, stored peer is " + f"{current.get('participantId')!r}") + held = current.get("revision") or 0 + if fresh.get("revision", 0) <= held: + raise StaleRevision( + f"descriptor revision did not increase: hold {held}, " + f"offered {fresh.get('revision', 0)}") + status = current.get("status", STATUS_ACTIVE) + + admitted_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return { + "participantId": network_id, + "name": fresh.get("name", ""), + # A peer is a node, not an upstream: it signs, and its signatures are + # verified against the keys recorded here. + "type": "node", + "status": status, + "baseUrl": (fresh.get("services") or {}).get("discovery", ""), + # A LIST, because role says what a participant serves and a network can + # serve several layers. We only know this peer as a network -- what it + # runs behind that is its own business and not in its descriptor. + "role": [ROLE_NETWORK], + "keys": [ + { + "alg": "ed25519", + "key": registry_key(k["x"]), + # validFrom is required, and is recorded as the moment we + # admitted the peer -- the only moment we can actually attest + # to. The peer's published documents carry no validity window, + # and inventing one would be recording a claim nobody made. + "validFrom": admitted_at, + "status": "active", + } + for k in published_keys(keyset) + ], + "revision": fresh.get("revision", 0), + "projectionTtl": ttl, + # Stored VERBATIM, not interpreted. What a peer can speak is its + # statement; rewriting it here would make our copy disagree with the + # document we read it from. The crawler derives the context URL it + # filters on from these at crawl time. + "schemaPacks": list(fresh.get("schemaPacks") or []), + # Carried over on a refresh. A later descriptor says nothing about the + # relationship -- that is OUR decision about a peer, not a claim it + # makes about itself -- so refreshing must not quietly widen it. + "federationRole": (current or {}).get("federationRole") or federation_role, + } + + +# --- this network's own registry ------------------------------------------ + +class ReadOnlyRegistry: + """What publish needs: searches, and no credentials. + + Separate from Registry because the privilege is different. Writing a + participant is an operator's decision and needs a Keycloak token; reading + one back to describe ourselves needs nothing, because the search endpoint is + public. Sharing one class would mean publish asking for a password it has no + use for. + """ + + def __init__(self, url, timeout=30): + self.url, self.timeout = url.rstrip("/"), timeout + + def _search(self, entity, filters): + request = urllib.request.Request( + f"{self.url}/api/v1/{entity}/search", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + found = json.load(response) + except urllib.error.HTTPError as err: + raise FederationError( + f"registry search {entity} returned {err.code}: " + f"{err.read().decode()[:200]}") from err + except Exception as err: + raise FederationError(f"reaching the registry at {self.url}: {err}") from err + return found.get("data", found) if isinstance(found, dict) else found + + def participant(self, participant_id): + rows = self._search("Participant", {"participantId": {"eq": participant_id}}) + return rows[0] if rows else None + + def schema_packs(self): + """The schema pack URLs this deployment records itself as speaking. + + Active only: a retired capability should stop being advertised without + anyone having to remember to edit a descriptor. + """ + rows = self._search("SchemaRegistry", {"status": {"eq": "active"}}) + return sorted({r["schemaUrl"] for r in rows if r.get("schemaUrl")}) + + +class Registry: + """The SunbirdRC registry this network runs. + + The same API surface setup.py uses: a Keycloak token, then POST to create + and PUT to update. Kept as its own class only so the token is minted once + per command rather than per call. + """ + + def __init__(self, args): + self.url = args.registry_url.rstrip("/") + self.keycloak = args.keycloak_url.rstrip("/") + self.issuer, self.realm = args.keycloak_issuer, args.realm + self.client_id, self.username, self.password = args.client_id, args.username, args.password + self.timeout = args.registry_timeout + self._bearer = None + + def bearer(self): + if self._bearer: + return self._bearer + body = urllib.parse.urlencode({ + "grant_type": "password", "client_id": self.client_id, + "username": self.username, "password": self.password, + }).encode() + # Keycloak sits behind PROXY_ADDRESS_FORWARDING, so it builds the + # token's issuer from the Host header. The registry validates that + # issuer against its own configured address -- the CONTAINER-INTERNAL + # one -- so a token minted without this header is rejected with a 401 + # that says nothing useful. + endpoint = (f"{self.keycloak}/auth/realms/{self.realm}" + f"/protocol/openid-connect/token") + request = urllib.request.Request( + endpoint, data=body, + headers={"X-Forwarded-Host": self.issuer, "X-Forwarded-Proto": "http", + "Content-Type": "application/x-www-form-urlencoded"}) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = json.load(response) + except urllib.error.HTTPError as err: + raise FederationError( + f"keycloak {endpoint} returned {err.code}: {err.read().decode()[:200]}") from err + except Exception as err: + raise FederationError(f"reaching keycloak at {endpoint}: {err}") from err + if not payload.get("access_token"): + # Keycloak answers 200 with an empty body when the forwarded host + # does not match what the registry expects, so a missing token is + # its own case rather than an HTTP error. + raise FederationError( + "keycloak issued no token -- check the credentials and --keycloak-issuer") + self._bearer = payload["access_token"] + return self._bearer + + def _call(self, method, path, body=None, authenticated=True): + headers = {"Content-Type": "application/json"} + if authenticated: + headers["Authorization"] = f"Bearer {self.bearer()}" + request = urllib.request.Request( + f"{self.url}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers=headers) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + raw = response.read() + except urllib.error.HTTPError as err: + raise FederationError( + f"registry {method} {path} returned {err.code}: {err.read().decode()[:300]}" + ) from err + return json.loads(raw) if raw else None + + def find(self, participant_id): + """The stored record for a participant, or None. + + Only participantId is filtered on: it is the schema's uniqueIndexFields, + so the registry already guarantees at most one match. + """ + found = self._call("POST", "/api/v1/Participant/search", + {"filters": {"participantId": {"eq": participant_id}}}, + authenticated=False) + records = found.get("data", found) if isinstance(found, dict) else found + return records[0] if records else None + + def write(self, record, existing): + """Create a record, or update the one already there.""" + if existing and existing.get("osid"): + return self._call("PUT", f"/api/v1/Participant/{existing['osid']}", record) + return self._call("POST", "/api/v1/Participant", record) + + def set_status(self, participant_id, status): + """Change only the status of an existing record. + + Nothing is fetched from the peer. Suspension is a decision about a + network we have stopped trusting, and asking that network's own server + to take part in it would be absurd. + """ + existing = self.find(participant_id) + if not existing: + raise FederationError(f"{participant_id} is not in this registry") + if not has_role(existing, ROLE_NETWORK): + raise FederationError(f"{participant_id} is not a peer network") + updated = {k: v for k, v in existing.items() if not k.startswith("os")} + updated["status"] = status + # Normalised on the way out. A record written before role became a list + # carries a plain string, and sending it back unchanged is refused -- + # the schema now wants an array. Reading either shape and writing one is + # what lets both sit in the registry at once. + updated["role"] = as_role_list(existing.get("role")) + self.write(updated, existing) + return updated + + +# --- commands ------------------------------------------------------------- + +def cmd_publish(args): + """Generate and sign this network's two published documents FROM THE REGISTRY. + + The registry entry is the source of truth, not these flags. What a network + says about itself in public is then the same thing it holds about itself + internally, by construction -- rather than two descriptions of one network + that happen to agree on the day someone typed them. + + Reading it needs no credentials: the registry's search endpoint is public. + Writing an entry is the privileged act, and that is setup.py's job, not this + one's. + """ + out = pathlib.Path(args.out) / ".well-known" + out.mkdir(parents=True, exist_ok=True) + + registry = ReadOnlyRegistry(args.registry_url, args.registry_timeout) + entry = registry.participant(args.network_id) + if entry is None: + raise FederationError( + f"{args.network_id} is not in this registry -- seed it first, " + f"then publish from it") + + network_key = load_or_create_key(args.key) + published = registry_key(b64url(public_of(network_key))) + held = [k.get("key", "").removeprefix("base64:") for k in entry.get("keys", [])] + if published not in held: + # Publishing a key the registry does not hold would hand peers something + # that verifies our descriptor but nothing else -- our signed traffic + # would be checked against the registry's key and fail. Worth refusing + # loudly rather than discovering on the first cross-network call. + raise FederationError( + f"{args.key} is not the key {args.network_id} is registered with; " + f"publishing it would sign descriptors with one key and traffic with another") + + descriptor = { + "version": "0.1", + "networkId": entry["participantId"], + "name": entry.get("name") or entry["participantId"], + "operator": {"name": args.operator or entry.get("name") or entry["participantId"]}, + "revision": args.revision, + "status": args.status, + "issuedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "operatingProfile": [p for p in args.operating_profile.split(",") if p], + # From the schema registry, which is where this deployment records what + # it can speak. Typing them again here would let the two disagree. + "schemaPacks": registry.schema_packs(), + # The entry's baseUrl IS the federated discovery endpoint -- the same + # field admission reads back out of a peer's descriptor, so what we + # publish and what a peer stores about us are one string. + "services": {"discovery": entry.get("baseUrl", "")}, + } + if args.projection_ttl is not None: + descriptor["projectionTtl"] = args.projection_ttl + + # ONE key. It signs this descriptor and it signs cross-network traffic. + # + # An earlier design split these -- a governance key for the descriptor, an + # operational key for everything else -- so that holding the traffic key + # could not let you rewrite who the network is. That is a real property, and + # it is given up deliberately: two keys meant two identities to configure, + # two to rotate, and a rule ("never sign traffic with the governance key") + # that nothing downstream could enforce, because a signature validator sees + # only that a key is published and active. + signed = sign(descriptor, KEY_ID, network_key) + keyset = {"keys": [ + {"kty": "OKP", "crv": "Ed25519", "kid": KEY_ID, "x": b64url(public_of(network_key))}, + ]} + + (out / "openagrinet").write_text(json.dumps(signed, indent=2) + "\n") + (out / "jwks.json").write_text(json.dumps(keyset, indent=2) + "\n") + + print(f"{args.network_id} now hosts its descriptor at revision {args.revision}") + print(f" descriptor {out / 'openagrinet'}") + print(f" keys {out / 'jwks.json'}") + return 0 + + +def cmd_verify(args): + """Fetch another network's documents and check them.""" + descriptor, keyset = fetcher_from(args).fetch(args.network_id) + print(f"verified {descriptor['networkId']} at revision {descriptor.get('revision')}, " + f"status {descriptor.get('status')}") + print(f" discovery {(descriptor.get('services') or {}).get('discovery')}") + print(f" keys {len(keyset.get('keys', []))} published") + print(" this proves the documents are self-consistent, NOT that the network " + "should be trusted -- admission is that decision") + return 0 + + +def cmd_admit(args): + """Record the decision to deal with a peer. Also the refresh job.""" + descriptor, keyset = fetcher_from(args).fetch(args.network_id) + + registry = Registry(args) + existing = registry.find(args.network_id) + try: + record = admit_record(existing, descriptor, keyset, args.federation_role) + except StaleRevision: + # Nothing written, and not a failure: this command is also the refresh + # job, and a refresh that finds nothing new is its normal outcome. + print(f"{args.network_id} is unchanged at revision " + f"{existing.get('revision')}; nothing written") + return 0 + registry.write(record, existing) + + verb = "refreshed" if existing else "admitted" + print(f"{verb} {record['participantId']} at revision {record['revision']}, " + f"status {record['status']}") + print(f" discovery {record['baseUrl']}") + print(f" projectionTtl {record['projectionTtl']}s") + print(f" keys {len(record['keys'])} published") + print(f" relationship {record['federationRole']}") + return 0 + + +def cmd_suspend(args): + """Stop dealing with a peer.""" + Registry(args).set_status(args.network_id, STATUS_INACTIVE) + print(f"suspended {args.network_id}; it is no longer crawled and no longer answered") + return 0 + + +def cmd_reinstate(args): + """Undo a suspension. A peer cannot do this for itself.""" + Registry(args).set_status(args.network_id, STATUS_ACTIVE) + print(f"reinstated {args.network_id}") + return 0 + + +def fetcher_from(args): + return Fetcher(scheme=args.scheme, ca_file=args.ca_file or None, + resolve=args.resolve or None, timeout=args.timeout) + + +# --- flags ---------------------------------------------------------------- + +def add_peer_access(parser): + """How an operator reaches a peer's published documents. + + Shared by verify and admit so the two cannot disagree about what + verification means -- the difference between them is what they do with the + result, not how they obtain it. + """ + parser.add_argument("--scheme", default="https", + help="http only for a stack with no certificates at all") + parser.add_argument("--ca-file", default="", + help="verify the peer's certificate against the CAs in this PEM file") + parser.add_argument("--resolve", default="", + help="reach the peer at this host:port instead of resolving its domain") + parser.add_argument("--timeout", type=float, default=10, help="per-request timeout") + + +def add_registry(parser): + parser.add_argument("--registry-url", required=True, + help="this network's own registry, e.g. http://localhost:10081") + parser.add_argument("--keycloak-url", required=True, + help="the registry's Keycloak, e.g. http://localhost:10080") + parser.add_argument("--keycloak-issuer", default="sunbird-registry-keycloak:8080", + help="the Host the registry expects the token's issuer to carry") + parser.add_argument("--realm", default="sunbird-rc", help="Keycloak realm") + parser.add_argument("--client-id", default="registry-frontend", help="Keycloak client id") + parser.add_argument("--username", required=True, help="registry admin user") + parser.add_argument("--password", required=True, help="registry admin password") + parser.add_argument("--registry-timeout", type=float, default=30, + help="per-request timeout for our own registry") + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="oanfed.py", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + publish = commands.add_parser("publish", help="generate and sign this network's documents") + publish.add_argument("--network-id", required=True, + help="this network's DNS domain, which is its identity") + publish.add_argument("--registry-url", required=True, + help="this network's own registry, which holds the entry " + "this descriptor is generated FROM") + publish.add_argument("--registry-timeout", type=float, default=30, + help="per-request timeout for the registry") + publish.add_argument("--operator", default="", + help="organisation answering for this network; " + "defaults to the registered name") + publish.add_argument("--revision", type=int, required=True, + help="descriptor revision; must increase on every republish") + publish.add_argument("--operating-profile", default="OAN/v0.1,Beckn/v2", + help="comma-separated version stack") + publish.add_argument("--status", default="Active", help="Active, Suspended or Retired") + publish.add_argument("--projection-ttl", type=int, default=None, + help="seconds a peer may keep what it crawls; omit to let peers decide") + publish.add_argument("--out", required=True, help="directory to write .well-known/ into") + publish.add_argument("--key", required=True, + help="this network's private key file; it signs the descriptor " + "AND cross-network traffic. Generated if absent") + publish.set_defaults(run=cmd_publish) + + verify_cmd = commands.add_parser("verify", help="fetch and check another network's documents") + verify_cmd.add_argument("--network-id", required=True, help="the peer to check") + add_peer_access(verify_cmd) + verify_cmd.set_defaults(run=cmd_verify) + + admit = commands.add_parser("admit", help="record the decision to deal with a peer") + admit.add_argument("--network-id", required=True, + help="the peer to admit, which is its domain") + admit.add_argument("--federation-role", default=DEFAULT_FEDERATION_ROLE, + choices=[r for r in FEDERATION_ROLES if r != "self"], + help="what this peer is to us: source (we crawl it), " + "requester (it calls us), or both. Only set on first " + "admission -- a refresh keeps what was decided") + add_peer_access(admit) + add_registry(admit) + admit.set_defaults(run=cmd_admit) + + suspend = commands.add_parser("suspend", help="stop dealing with a peer") + suspend.add_argument("--network-id", required=True, help="the peer to suspend") + add_registry(suspend) + suspend.set_defaults(run=cmd_suspend) + + reinstate = commands.add_parser("reinstate", help="undo a suspension") + reinstate.add_argument("--network-id", required=True, help="the peer to reinstate") + add_registry(reinstate) + reinstate.set_defaults(run=cmd_reinstate) + + args = parser.parse_args(argv) + try: + return args.run(args) + except FederationError as err: + print(f"oanfed {args.command}: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/networkofnetwork/bin/render-adapter.py b/networkofnetwork/bin/render-adapter.py new file mode 100755 index 00000000..f8502968 --- /dev/null +++ b/networkofnetwork/bin/render-adapter.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Render this network's ONE adapter config. + +quick-start's setup.py does everything that is hard and shared -- generating the +keypair, seeding the registry, registering the schemas -- and is reused as-is. +What it cannot do is render a config it has never heard of, so this does that +one step and nothing else. + +It reads what setup.py already wrote: the network's keys.json and its .env. No +value is invented here. + +Run: render-adapter.py +""" +import json +import os +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent +TEMPLATE = ROOT / "config" / "adapter.yaml.tmpl" +STATE = ROOT / "networks" + + +def read_env(path): + """The network's .env, as a dict. Same shape setup.py reads.""" + values = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip() + return values + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: render-adapter.py ") + network = argv[1].strip() + directory = STATE / network + + keys_file = directory / "keys" / "keys.json" + env_file = directory / ".env" + for required in (keys_file, env_file): + if not required.exists(): + sys.exit(f"render-adapter: {required} is missing -- run setup first") + + env = read_env(env_file) + # One identity for the whole network, so any role's entry carries it. The + # file is still keyed by role because setup.py writes it that way. + identity = json.loads(keys_file.read_text())["network"] + + def value(name, default=None): + got = env.get(name, default) + if got is None: + sys.exit(f"render-adapter: {name} is not set in {env_file}") + return got + + def binding(prefix): + return f"{value(prefix + '_PARTICIPANT_ID')}|{value(prefix + '_CAPABILITY')}" + + substitutions = ( + ("__BECKN_NETWORK_ID__", identity["participantId"]), + ("__KEY_ID__", identity["keyOsid"]), + ("__SIGNING_PRIVATE__", identity["signingPrivate"]), + ("__SIGNING_PUBLIC__", identity["signingPublic"]), + ("__ENCR_PRIVATE__", identity["encrPrivate"]), + ("__ENCR_PUBLIC__", identity["encrPublic"]), + ("__NETWORK_BASE_URL__", value("NETWORK_BASE_URL")), + ("__MAUSAMGRAM_BINDING_KEY__", binding("MAUSAMGRAM")), + ("__AGMARKNET_BINDING_KEY__", binding("AGMARKNET")), + ("__VISTAAR_BINDING_KEY__", binding("VISTAAR")), + ("__POCRA_BINDING_KEY__", binding("POCRA")), + ("__MAUSAMGRAM_PARTICIPANT_ID__", value("MAUSAMGRAM_PARTICIPANT_ID")), + ("__AGMARKNET_PARTICIPANT_ID__", value("AGMARKNET_PARTICIPANT_ID")), + ("__VISTAAR_PARTICIPANT_ID__", value("VISTAAR_PARTICIPANT_ID")), + ("__POCRA_PARTICIPANT_ID__", value("POCRA_PARTICIPANT_ID")), + ("__AGMARKNET_TOKEN_URL__", value("AGMARKNET_TOKEN_URL")), + ("__VISTAAR_TOKEN_URL__", value("VISTAAR_TOKEN_URL")), + # Empty disables peer crawling: the crawler builds no peer half without + # a network id, so /crawl/peers refuses and the refresh loop never + # starts. Answering another network's crawl is unaffected. + ("__FEDERATION_NETWORK_ID__", + "" if value("FEDERATION_CRAWL_PEERS", "on").lower() == "off" + else value("BECKN_NETWORK_ID")), + ("__FEDERATION_REFRESH_SECONDS__", value("FEDERATION_REFRESH_SECONDS", "900")), + ("__FEDERATION_PAGE_SIZE__", value("FEDERATION_PAGE_SIZE", "0")), + ("__FEDERATION_UPDATE_MODE__", value("FEDERATION_UPDATE_MODE", "MERGE")), + ("__FEDERATION_INTENT__", value("FEDERATION_INTENT", "")), + ("__FEDERATION_PUBLISH_RETRY_SECONDS__", value("FEDERATION_PUBLISH_RETRY_SECONDS", "30")), + ("__OTEL_ENABLED__", value("ADAPTER_OTEL_ENABLED", "false")), + ("__OTLP_ENDPOINT__", value("OTLP_ENDPOINT", "hyperdx:4317")), + ("__OTEL_ENVIRONMENT__", value("OTEL_ENVIRONMENT", "dev")), + ) + + rendered = TEMPLATE.read_text() + for placeholder, replacement in substitutions: + rendered = rendered.replace(placeholder, replacement) + + # A placeholder left behind is a value nobody noticed was missing, and the + # adapter would start with "__SIGNING_PRIVATE__" as a key. + if "__" in rendered: + leftover = sorted({w for w in rendered.split() if w.startswith("__")}) + sys.exit(f"render-adapter: unrendered placeholders remain: {leftover}") + + out_dir = directory / "config" + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / "adapter.yaml" + if out.is_dir(): + sys.exit(f"render-adapter: {out} is a directory -- docker created it by " + f"starting the adapter before this ran. Remove it and retry.") + out.write_text(rendered) + out.chmod(0o600) # holds a private key + print(f"render-adapter: wrote {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/setup.py b/networkofnetwork/bin/setup.py new file mode 100755 index 00000000..901c8439 --- /dev/null +++ b/networkofnetwork/bin/setup.py @@ -0,0 +1,680 @@ +#!/usr/bin/env python3 +"""Prepare the stack: generate the adapter keypairs, seed the registry, render +the adapter configs. + + python3 bin/setup.py + +`make up` runs this as step 2, which is where it belongs -- the adapter configs +it renders are bind-mounted files, and an adapter started before they exist +leaves a directory in their place. + +WHAT IT WRITES. Seven participants, four capability bindings, four schemas: + + 3 x node one per adapter -- consumer, network, provider -- each with the + public halves of a keypair. The private halves stay in + keys/keys.json and never reach the registry. + 4 x upstream the APIs this deployment calls. ALL FOUR are real external + hosts now, whose base URLs come from .env -- no mock is in + use by any capability. An upstream signs nothing, so it needs + no role and no keys. + 4 x binding a ProviderSchema row per capability: which upstream answers + it, the method and path, timeouts, and the mapping URL. + 4 x schema a SchemaRegistry row per capability, saying what the + capability MEANS rather than who serves it: the pack in + network-specs that defines its attributes. One per + capability, so two providers of one capability share a row. + +This is all of it. Nothing has to be created by hand afterwards, and nothing +can be from outside the VM -- the registry has no route through the gateway and +publishes on loopback only, which is why seeding lives here rather than in a +Postman request. + +The binding keys are the load-bearing part. A provider step answers only when +the key it was configured with matches the one built from the incoming payload, +and both sides come from the same .env values in this one run -- which is what +keeps them from disagreeing. To point a capability somewhere else: edit .env, +re-run this, recreate the provider adapter. + +Safe to re-run. Keys are generated once and reused from keys/keys.json, so the +identities already in the registry stay valid; participants and bindings that +exist are left alone rather than recreated, because this registry's delete is +soft and holds the unique index -- a deleted participantId cannot be reused. + +Needs python3 and the cryptography package: + + pip install cryptography +""" +import base64, json, os, pathlib, sys, time, urllib.error, urllib.parse, urllib.request + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from cryptography.hazmat.primitives import serialization as ser + +# Where this run's .env, keys and rendered adapter configs live. +# +# Defaults to the quick-start directory itself, which is every existing use. +# QUICKSTART_ROOT overrides it so one checkout can set up more than one network: +# each network gets its own directory, because keys.json and the rendered +# adapter configs carry that network's identity and a second setup run in the +# same place would overwrite the first one's. +ROOT = pathlib.Path(os.environ.get("QUICKSTART_ROOT") or pathlib.Path(__file__).resolve().parent.parent) +KEYS = ROOT / "keys" / "keys.json" +# No ADAPTERS directory here: this deployment runs ONE adapter, and its config +# is rendered by bin/render-adapter.py from config/adapter.yaml.tmpl. This script +# does the part that is the same either way -- keys, the registry, the schemas. + +REPO = pathlib.Path(__file__).resolve().parent.parent + + +def load_dotenv(): + """Read .env into the environment. + + Nothing sources .env before this runs -- `make up` shells straight out to + python3 -- and a real environment variable wins so a one-off override still + works: + + REGISTRY_PORT=9081 python3 bin/setup.py + """ + path = ROOT / ".env" + if not path.exists(): + sys.exit("setup: no .env -- copy .env.example to .env first") + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, value = line.split("=", 1) + os.environ.setdefault(name.strip(), value.strip()) + + +def env(name, default=None): + v = os.environ.get(name, default) + if v is None: + sys.exit(f"setup: {name} is not set -- is it missing from .env?") + if is_placeholder(v): + sys.exit(f"setup: {name} is still the .env.example placeholder {v!r}.\n" + f"{PLACEHOLDER_WHY}") + return v + + +def is_placeholder(value): + """True for a value still in .env.example's `` form. + + Every placeholder in the template is angle-bracketed, and no real base URL, + path, credential or id can be -- so this is an exact test rather than a + guess.""" + v = value.strip() + return v.startswith("<") and v.endswith(">") + + +PLACEHOLDER_WHY = """\ + Fill it in in .env, then re-run. Nothing has been written yet. + + This is refused rather than passed through because the registry is + APPEND-ONLY: it cannot update a record, and its delete is soft and keeps the + unique index, so an id is never reusable. Seeding a participant with a + placeholder base URL would burn that id permanently -- the capability could + only be recovered under a NEW id, which is the one thing the settled naming + was meant to avoid.""" + + +# Everything that has to be a real value before the first registry write. +# +# Checked together, up front, for two reasons. seed() runs BEFORE render(), so a +# placeholder in a render-only key like VISTAAR_TOKEN_URL would otherwise be +# found only after seven rows already existed -- unfixable, per the note above. +# And one list beats being sent back to .env ten times in a row. +REQUIRED = [ + "BECKN_NETWORK_ID", + "CONSUMER_BASE_URL", "NETWORK_BASE_URL", "PROVIDER_BASE_URL", + "MAUSAMGRAM_PARTICIPANT_ID", "MAUSAMGRAM_CAPABILITY", "MAUSAMGRAM_BASE_URL", + "MAUSAMGRAM_AUTH", "MAUSAMGRAM_MAPPING_URL", + "AGMARKNET_PARTICIPANT_ID", "AGMARKNET_CAPABILITY", "AGMARKNET_BASE_URL", + "AGMARKNET_TOKEN_URL", "AGMARKNET_ACCESS_NAME", "AGMARKNET_PASSWORD", + "AGMARKNET_MAPPING_URL", + "VISTAAR_PARTICIPANT_ID", "VISTAAR_CAPABILITY", "VISTAAR_BASE_URL", + "VISTAAR_PATH", "VISTAAR_TOKEN_URL", "VISTAAR_CLIENT_ID", + "VISTAAR_CLIENT_SECRET", "VISTAAR_MAPPING_URL", + "POCRA_PARTICIPANT_ID", "POCRA_CAPABILITY", "POCRA_BASE_URL", + "POCRA_PATH", "POCRA_MAPPING_URL", +] + + +def preflight(): + """Refuse to touch the registry while any required value is unfilled.""" + missing = [k for k in REQUIRED if os.environ.get(k) is None] + unfilled = [k for k in REQUIRED + if os.environ.get(k) is not None + and is_placeholder(os.environ[k])] + if not missing and not unfilled: + return + lines = ["setup: .env is not ready, so nothing was written.\n"] + if unfilled: + lines.append(" Still the .env.example placeholder:") + lines += [f" {k}={os.environ[k]}" for k in unfilled] + if missing: + lines.append(" Absent from .env entirely:") + lines += [f" {k}" for k in missing] + lines.append(PLACEHOLDER_WHY) + sys.exit("\n".join(lines)) + + +# --------------------------------------------------------------------- keys + +def b64(raw): + return base64.b64encode(raw).decode() + + +def ed25519_pair(): + k = Ed25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def x25519_pair(): + k = X25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def load_or_generate_keys(): + """One identity for the whole network, shared by all three adapters. + + The network IS the participant: networkId and participantId are the same + string, and there is ONE registry entry and ONE keypair behind it. + + Two identities -- an adapter's and the network's -- reintroduce the question + of which key signs what, and they make the three-adapter and consolidated + deployments diverge: in one shape the consumer layer is a separate + registered party, in the other it is the same process. One identity means a + peer sees the same network either way, and the layer split stays a local + deployment detail. + + The dict is still keyed by role because the config templates are per-role + and each needs the same values substituted. All three entries carry the same + participantId and the same keypair. + """ + network = env("BECKN_NETWORK_ID") + if not network: + sys.exit("setup: BECKN_NETWORK_ID is required -- it is this network's identity") + + if KEYS.exists(): + print("keys: reusing keys/keys.json") + identities = json.loads(KEYS.read_text()) + # The keypair persists; the id is read from .env each run. This registry + # cannot update a record, so changing an id in .env seeds a NEW + # participant rather than editing one -- and the same keypair moving to + # the new id is what makes that a rename rather than a rekey. + for role in ROLES: + if identities[role]["participantId"] != network: + print(f" {role}: id is now {network}, keeping the keypair") + identities[role]["participantId"] = network + KEYS.write_text(json.dumps(identities, indent=2)) + write_network_key(identities) + return identities + + print(f"keys: generating one identity for {network}") + sign_private, sign_public = ed25519_pair() + encr_private, encr_public = x25519_pair() + identities = {} + for role in ROLES: + identities[role] = {"participantId": network, + "signingPrivate": sign_private, "signingPublic": sign_public, + "encrPrivate": encr_private, "encrPublic": encr_public} + # parents=True because under QUICKSTART_ROOT the network's directory itself + # may be new, not just the keys/ inside it. + KEYS.parent.mkdir(parents=True, exist_ok=True) + KEYS.write_text(json.dumps(identities, indent=2)) + KEYS.chmod(0o600) + write_network_key(identities) + return identities + + +def write_network_key(identities): + """The same private key, in the one-line form oanfed.py reads. + + oanfed publishes this network's descriptor and signs it; the adapters sign + traffic with it; the registry holds its public half. One key, so one file -- + written here rather than generated there, so the two cannot drift into + publishing one key and signing with another. + + base64url unpadded, which is what a JWKS carries, so the file and the + published `x` are the same alphabet. + """ + raw = base64.b64decode(identities["network"]["signingPrivate"]) + path = KEYS.parent / "network.key" + path.write_text(base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + "\n") + path.chmod(0o600) + + +# ----------------------------------------------------------------- registry + +def registry_url(): + return f"http://localhost:{env('REGISTRY_PORT', '8081')}" + + +def token(): + body = urllib.parse.urlencode({ + "client_id": env("KEYCLOAK_CLIENT_ID", "registry-frontend"), + "grant_type": "password", + "username": env("REGISTRY_ADMIN_USER", "no-user"), + "password": env("REGISTRY_ADMIN_PASSWORD", "no-user-password"), + }).encode() + # Keycloak sits behind PROXY_ADDRESS_FORWARDING, so it builds the token's + # issuer from these headers. Without them it answers with an empty body. + # + # registry-keycloak:8080 is the CONTAINER-INTERNAL address, and is + # deliberately not KEYCLOAK_PORT. The registry validates the issuer against + # OAUTH2_RESOURCES_0_URI, which names that internal address -- so a token + # minted with the host port in its issuer is rejected with a 401 and an + # empty body, however the port is published. + # + # It must match this compose file's keycloak SERVICE NAME. It does not match + # quick-start's, which is why this copy differs from that one by this line. + req = urllib.request.Request( + f"http://localhost:{env('KEYCLOAK_PORT', '8080')}/auth/realms/" + f"{env('KEYCLOAK_REALM', 'sunbird-rc')}/protocol/openid-connect/token", + data=body, headers={"X-Forwarded-Host": "registry-keycloak:8080", + "X-Forwarded-Proto": "http"}) + with urllib.request.urlopen(req, timeout=30) as r: + payload = json.load(r) + if not payload.get("access_token"): + sys.exit("setup: keycloak issued no token -- check the KEYCLOAK_* values in .env") + return payload["access_token"] + + +def post(entity, payload, bearer): + # No {"EntityName": {...}} wrapper: this registry takes the record itself, + # and a wrapper comes back as "extraneous key [...] is not permitted". + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}", method="POST", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {bearer}"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + except urllib.error.HTTPError as e: + # The registry answers a rejected write with a JSON envelope, but not + # always: a 401 comes back with an empty body. Decoding blindly turns + # that into a JSONDecodeError traceback that says nothing about what + # went wrong, so report the status instead. + raw = e.read() + try: + return json.loads(raw) + except ValueError: + detail = raw.decode(errors="replace").strip()[:200] or "(empty body)" + sys.exit(f"setup: the registry refused a write -- HTTP {e.code}: {detail}\n" + f" A 401 here usually means the token was minted for a different\n" + f" issuer than the registry validates against.") + + +def search(entity, filters): + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}/search", method="POST", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r).get("data", []) + + +def wait_for_registry(): + for _ in range(60): + try: + search("Participant", {}) + return + except Exception: + time.sleep(2) + sys.exit("setup: the registry did not come up -- check `docker compose ps`\n" + " and that REGISTRY_PORT in .env matches the published port") + + +def signing_key_block(public_key): + """The published half of the signing keypair. + + Bare base64, with no encoding label in front of it: what a verifier hands + to a base64 decoder is the value as published, and a label left on fails + every verification with a decode error pointing nowhere near the registry. + + No friendly key id, because nothing could look one up: the registry assigns + an osid on write and that is what a sender names in the Authorization + header. No use either -- alg carries the purpose, ed25519 signs.""" + return [{"alg": "ed25519", "key": public_key, "status": "active", + "validFrom": "2026-01-01T00:00:00Z", "validUntil": "2030-01-01T00:00:00Z"}] + + +def node(participant_id, name, role, base_url, public_key, federation_role=None): + """A participant that speaks Beckn. + + One level, no wrapper object: type decides which fields apply. baseUrl must + be https for a node, and the id must be hostname-shaped -- it is the + identity a signature is checked against. + + baseUrl IS PUBLISHED FOR PEERS, and is the one field here that a real + network reads rather than this stack. Nothing inside this stack resolves it + -- routing between the adapters is the router plugin's config, which uses + the compose service names -- so a wrong value costs nothing locally and + everything on a shared network, where it is the address a peer POSTs to. + That is why it comes from .env rather than being built from the id: this + registry cannot update a record, so a fabricated URL is permanent.""" + record = {"participantId": participant_id, "name": name, "type": "node", + "status": "active", "baseUrl": base_url, + "role": role, "keys": signing_key_block(public_key)} + if federation_role: + record["federationRole"] = federation_role + return record + + +def upstream(participant_id, name, base_url): + """An ordinary API the provider adapter calls. + + No role and no keys: it has never heard of Beckn, so it signs nothing and + nothing verifies it. Both are permitted by the schema but neither is read -- + a signature is checked against the node identity that signed it. + + No credential either. The adapter presents one from its own config, naming + the environment variable it comes from, so nothing secret is held here.""" + return {"participantId": participant_id, "name": name, "type": "upstream", + "status": "active", "baseUrl": base_url} + + +def ensure_binding(bearer, participant_id, capability, path, mapping_url, + method="GET"): + """Create a capability binding only when absent. + + actions is a list, not a map: the registry treats every nested object as an + entity and injects an osid into it, which a map cannot carry. It is also + what lets one action be retired without touching the others. + + mappings is one reference carrying both directions, because the response + mapping reads what the request mapping resolved. + + method is a parameter because it is the provider's contract, not a network + convention: Mausamgram and Agmarknet answer a GET whose query the mapping + builds, while the knowledge provider and POCRA take a POST with a JSON body. + The adapter reads it from this row, so nothing about it is compiled in.""" + binding = f"{participant_id}|{capability}" + if search("ProviderSchema", {"bindingKey": {"eq": binding}}): + print(f" {binding}: already present") + return + result = post("ProviderSchema", { + "bindingKey": binding, "participantId": participant_id, + "capabilityCode": capability, "status": "active", + "actions": [{"action": "select", "method": method, "path": path, + "mappings": mapping_url, + "timeoutMs": 15000, "retryMax": 2, + "status": "active"}]}, bearer) + print(f" {binding}: {result['params']['status']} " + f"{result['params'].get('errmsg', '')[:160]}") + + +# The four capability schemas, keyed by the .env variable each capability code +# comes from so the code itself is never written twice. +# +# The display name mirrors the pack's own info.title, minus its "OpenAgriNet -" +# and "Attributes" framing. The version is per capability rather than one +# setting for all four, because a capability can move to v0.2 while the others +# stay -- that is the point of versioning the packs separately. +CAPABILITY_SCHEMAS = ( + ("MAUSAMGRAM_CAPABILITY", "Weather Observation", "v0.1"), + ("AGMARKNET_CAPABILITY", "Mandi Price", "v0.1"), + ("VISTAAR_CAPABILITY", "Knowledge Advisory", "v0.1"), + ("POCRA_CAPABILITY", "Agriculture Facility", "v0.1"), +) + + +def schema_url(capability, version, ref): + """Build a pack URL from the capability code, which already contains the + pack directory's name: openagrinet:WeatherObservation is served out of + schema/WeatherObservation. Deriving it means the two cannot drift. + + The registry pattern-checks the result against exactly this shape, so a + capability whose pack is laid out differently is refused rather than + stored -- which is the behaviour to want, since all eight packs in + network-specs are //attributes.yaml today.""" + # openagrinet.github.io, NOT raw.githubusercontent.com: the latter 404s for + # every path under network-specs, so a crawl asking with schemaContext + # derived from these URLs matched nothing and mirrored zero catalogs. The + # published pages are what capability examples carry in their @context. + # + # ref is unused here -- the pages site serves one published version and has + # no per-ref path -- but is kept so the caller's SCHEMA_PACK_REF still reads + # as the thing that would select one if the layout gains it. + del ref + return ("https://openagrinet.github.io/network-specs/" + f"schema/{capability.partition(':')[2]}/{version}" + "/attributes.yaml") + + +def ensure_capability_schema(bearer, capability, name, version, ref): + """Create a capability's schema row only when absent. + + THE REGISTRY NEVER FETCHES schemaUrl -- it only checks its shape. So a row + pointing at a ref that does not serve the pack stores perfectly happily and + fails later, at whoever resolves it. Getting SCHEMA_PACK_REF right matters + more than the write succeeding. + + And it cannot be corrected afterwards: SchemaRegistry.json sets + additionalProperties false, so an update is rejected over the system fields + the registry itself merges in, while delete is soft and keeps the unique + index on capabilityCode. A wrong URL here is permanent for that code.""" + if search("SchemaRegistry", {"capabilityCode": {"eq": capability}}): + print(f" {capability}: already present") + return + result = post("SchemaRegistry", { + "capabilityCode": capability, "name": name, "version": version, + "schemaUrl": schema_url(capability, version, ref), + "status": "active"}, bearer) + print(f" {capability}: {result['params']['status']} " + f"{result['params'].get('errmsg', '')[:160]}") + + +def ensure_participant(bearer, participant_id, payload): + """Create only when absent. Delete here is soft and keeps the unique index, + so a recreate would fail on a duplicate key rather than replacing.""" + if search("Participant", {"participantId": {"eq": participant_id}}): + print(f" {participant_id}: already present") + return + result = post("Participant", payload, bearer) + status = result["params"]["status"] + print(f" {participant_id}: {status} {result['params'].get('errmsg', '')[:160]}") + + +def seed(identities): + wait_for_registry() + bearer = token() + + # Seven participants and four capability bindings, all of it from here. + # + # The registry is not reachable from outside this stack -- no published + # port beyond loopback and no proxy host in front of it -- so there is no + # second way to create these. Everything the network needs to answer a + # request has to exist by the time this returns. + # ONE entry. The network IS the participant, and `role` says what the + # NETWORK serves rather than which container exists -- a split + # three-adapter deployment and a consolidated one carry the same value. + # + # baseUrl is the network's own published address, which is what a peer + # reaches us at. Not any individual adapter's: routing between the layers is + # the routing config's job and never appears in the registry. + print("registry: one network identity") + identity = identities["network"] + ensure_participant(bearer, identity["participantId"], + node(identity["participantId"], + env("BECKN_NETWORK_NAME", identity["participantId"]), + list(ROLES), + env("NETWORK_BASE_URL"), + identity["signingPublic"], + # OURS. Stored rather than worked out: deciding + # whether an entry is this deployment's own by + # comparing ids against config has been got wrong + # twice, once badly enough that the crawler tried to + # crawl itself. + federation_role="self")) + + # All four upstreams are real external providers now -- no mock is in use + # by any capability, so every base URL comes from .env and the stack needs + # egress to all of them. + # + # No default base URL for weather any more: falling back to the mock's + # service name would seed a row that cannot serve the basic auth the + # capability is configured with, and the registry cannot update it after. + print("registry: four upstream providers") + weather = env("MAUSAMGRAM_PARTICIPANT_ID") + ensure_participant(bearer, weather, + upstream(weather, "IMD Mausamgram NWP", + env("MAUSAMGRAM_BASE_URL"))) + + # Mandi is the REAL Agmarknet now, because authScheme tokenQuery needs a + # token endpoint and mock-agmarknet has none. No default base URL for the + # same reason: falling back to the mock's service name would seed a row + # that cannot serve the configured scheme, and the registry cannot update + # it afterwards. + mandi = env("AGMARKNET_PARTICIPANT_ID") + ensure_participant(bearer, mandi, + upstream(mandi, "Agmarknet Vistaar", + env("AGMARKNET_BASE_URL"))) + + # The knowledge capability. Also a REAL external provider, so the stack + # needs egress to it and its base URL comes from .env rather than a service + # name. It is the only one whose action is a POST. + knowledge = env("VISTAAR_PARTICIPANT_ID") + ensure_participant(bearer, knowledge, + upstream(knowledge, "Bharat Vistaar knowledge retrieval", + env("VISTAAR_BASE_URL"))) + + # POCRA, serving AgricultureFacility. A real external provider too, and the + # only one needing no credential at all. + pocra = env("POCRA_PARTICIPANT_ID") + ensure_participant(bearer, pocra, + upstream(pocra, "POCRA agriculture facilities", + env("POCRA_BASE_URL"))) + + # And what each of them answers. The binding key is participantId piped to + # capabilityCode, and it has to match what the provider adapter was + # rendered with -- both come from the same .env, which is what keeps them + # from disagreeing. + print("registry: four capability bindings") + ensure_binding(bearer, weather, env("MAUSAMGRAM_CAPABILITY"), + env("MAUSAMGRAM_PATH", "/get-daily"), env("MAUSAMGRAM_MAPPING_URL")) + ensure_binding(bearer, mandi, env("AGMARKNET_CAPABILITY"), + env("AGMARKNET_PATH", "/v1/fetch-agmarknet-vistaar"), + env("AGMARKNET_MAPPING_URL")) + ensure_binding(bearer, knowledge, env("VISTAAR_CAPABILITY"), + env("VISTAAR_PATH"), env("VISTAAR_MAPPING_URL"), + method="POST") + ensure_binding(bearer, pocra, env("POCRA_CAPABILITY"), + env("POCRA_PATH"), env("POCRA_MAPPING_URL"), + method="POST") + + # And what each capability MEANS, which is a different question from who + # serves it. A binding above says mausamgram answers WeatherObservation + # over this path with this mapping; a row here says what a + # WeatherObservation is, by pointing at the schema pack that defines its + # attributes. One row per capability, not per binding -- a second provider + # for the same capability shares this row rather than adding one. + # + # SEPARATE FROM THE MAPPING URLS ABOVE, which live in helmcharts and + # describe one upstream's wire format. These are network-specs and describe + # the network's own vocabulary, so they move on their own ref. + print("registry: four capability schemas") + ref = env("SCHEMA_PACK_REF", "main") + for var, name, version in CAPABILITY_SCHEMAS: + ensure_capability_schema(bearer, env(var), name, version, ref) + + +def key_osids(identities): + """Read back each key's osid, and check the registry still holds the public + half we have the private half for. + + The Authorization header names a key by its osid rather than by the friendly + id, so the adapters have to be configured with the value the registry + assigned. + + The mismatch check matters because this registry cannot update a record and + its delete is soft: a participant seeded against an earlier keys.json keeps + that public key forever. Signing with a new private half would then produce + signatures nobody can verify -- and the failure would surface much later, as + an authentication error with no obvious cause.""" + # All three entries name the same participant, so this resolves one record + # three times. Left as a loop rather than special-cased: each role's config + # still needs the osid substituted, and one extra cached search is cheaper + # than a second code path. + for role, identity in identities.items(): + records = search("Participant", {"participantId": {"eq": identity["participantId"]}}) + keys = (records[0].get("keys") or []) if records else [] + if not keys: + sys.exit(f"setup: {identity['participantId']} has no published key") + + # Bare base64 now, but an older row may still carry the label, and a + # confusing mismatch error is worse than one tolerant line. + published = keys[0]["key"].removeprefix("base64:") + if published != identity["signingPublic"]: + sys.exit( + f"setup: {identity['participantId']} is registered with a different key.\n" + f" This registry cannot update a record, and its delete is soft and keeps\n" + f" the unique index, so the id cannot be reused. Either restore the\n" + f" matching keys/keys.json, or pick a new BECKN_NETWORK_ID\n" + f" in .env and re-run.") + identity["keyOsid"] = keys[0]["osid"] + return identities + + +# ------------------------------------------------------------------ configs + +# The role key is three things at once: the entry in keys/keys.json, the +# __ROLE_* placeholder prefix, and the config filename. They were spelled +# differently once -- role "exp" writing experience.yaml -- and a lookup table +# kept them apart. They are the same word now, so the table is gone. +# +# CHANGING A ROLE KEY IS NOT A RENAME. It makes this script generate a FRESH +# keypair for a participant the registry has already published a public key +# for, which it cannot update and whose delete is soft -- so the id cannot be +# reused either. The adapter then signs with a key nobody can verify, and it +# surfaces much later as an authentication error with no obvious cause. Renaming +# exp -> consumer therefore came with `make destroy` and a new keys.json. +ROLES = ("consumer", "network", "provider") + + +if __name__ == "__main__": + load_dotenv() + preflight() + identities = load_or_generate_keys() + seed(identities) + identities = key_osids(identities) + KEYS.write_text(json.dumps(identities, indent=2)) + print(f""" +ready. The registry holds seven participants and four capability bindings, and +the adapter configs are rendered, so nothing further has to be created by hand. + + {env('MAUSAMGRAM_PARTICIPANT_ID')}|{env('MAUSAMGRAM_CAPABILITY')} + {env('AGMARKNET_PARTICIPANT_ID')}|{env('AGMARKNET_CAPABILITY')} + {env('VISTAAR_PARTICIPANT_ID')}|{env('VISTAAR_CAPABILITY')} + {env('POCRA_PARTICIPANT_ID')}|{env('POCRA_CAPABILITY')} + +Those are the binding keys the provider adapter answers to. They were rendered +into its config from the same .env this seeded the registry from, which is what +keeps the two from disagreeing. A payload naming anything else is answered 404 +"this module serves no capability matching the request" -- explicit, but it +names the request rather than the mismatch, so compare it against these four. + +The left half of each key is the participant id, and it names the ENTITY that +operates the upstream -- not the capability, which is already the right half, +and not the deployment, which is only a base URL. It is also the string a +catalog publishes as offer.provider.id, so capability-examples/ can be run +against this stack unedited. + +All four upstreams are real external providers reached over the internet, so +this stack needs egress to each of them and no capability depends on a mock. +Repointing one is an .env edit and a re-run of this, but it takes a NEW +participant id: the registry cannot update a row and its delete is soft and +keeps the unique index, so an id seeded against one base URL keeps that base +URL for good. A re-run with the same id prints "already present" and changes +nothing. The registry is not reachable from outside this stack, so that write +happens from here. + +Next: `make up` continues to step 3 and starts the adapters. If you ran this +on its own, the adapters need recreating to pick up the rendered configs: + + docker compose up -d --force-recreate provider-adapter network-adapter consumer-adapter + +Then import ../api-collection/ -- both files, the collection and the +environment -- and run it. Eighteen requests across the four capabilities, +publish then discover then select in each, with nothing to fill in.""") diff --git a/networkofnetwork/config/adapter.yaml.tmpl b/networkofnetwork/config/adapter.yaml.tmpl new file mode 100644 index 00000000..4aa76a7d --- /dev/null +++ b/networkofnetwork/config/adapter.yaml.tmpl @@ -0,0 +1,505 @@ +# ONE OAN adapter, serving every layer this network has. +# +# The quick-start runs three adapters -- consumer, network, provider -- as three +# containers. That is a deployment choice, not what a network IS: a network is +# one participant with one identity, and the three layers are modules of it. +# This config is the consolidated shape, which is the one to reason about when +# there are two whole networks on the machine already. +# +# Each layer is a module on its own path of one port: +# +# /consumer/ what the experience layer posts to +# /network/ publish and discover, answered from this network's discovery +# /provider/ select, init, confirm, status -- answered by a capability, or +# routed to another network when the request is not ours +# /publish this network's catalogue system publishing inward +# /crawl/ operator endpoints: trigger a crawl, read its status +# +# Every module shares one identity, one keypair and one registry -- see the +# anchor below. Under one identity they are not three participants that happen +# to agree; they are one participant, and the config says so once. +appName: "__BECKN_NETWORK_ID__" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 9200 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +plugins: + # Ships metrics, traces and logs over OTLP/gRPC. All three flags false builds + # no exporter at all, which is the right setting for a stack with no collector. + otelsetup: + id: otelsetup + config: + serviceName: "__BECKN_NETWORK_ID__" + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + + # The crawler's OWN registry, separate from the modules' below. It is key + # distribution for verifying what the crawl fetches, not request handling, and + # the crawler refuses to start without it. + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + + # The crawler. One per deployment, app-level, because it is a background pass + # rather than something a request reaches. + crawler: + id: catalogcrawler + config: + dbDsn: "postgres://discovery:discovery@discovery-postgres:5432/catalogcrawler?sslmode=disable" + discoveryPushUrl: "http://discovery-service:8080/publish" + participantId: "__BECKN_NETWORK_ID__" + bppUri: "__NETWORK_BASE_URL__" + + # The federated half. Present means this network crawls what it has + # admitted; EMPTY means it crawls only its own providers -- it still + # ANSWERS another network's crawl, because that is the network module's + # job and nothing to do with this block. + # + # Rendered empty when FEDERATION_CRAWL_PEERS=off, which is how a + # deployment opts out of crawling without having to un-admit anyone. + federationNetworkId: "__FEDERATION_NETWORK_ID__" + federationSigningKey: "__SIGNING_PRIVATE__" + federationKeyId: "__KEY_ID__" + federationRefreshSeconds: "__FEDERATION_REFRESH_SECONDS__" + federationPageSize: "__FEDERATION_PAGE_SIZE__" + federationUpdateMode: "__FEDERATION_UPDATE_MODE__" + # What breadth of another network's catalogue to mirror. + # + # REQUIRED IN PRACTICE, though the code defaults it to an empty intent. + # Discovery refuses one that narrows nothing -- "an intent needs at least + # one of spatial or filters" for an empty one, and "this expression + # narrows nothing the index can serve" for a filter built to match all -- + # so a crawl with no intent configured silently mirrors zero catalogs. + # That is the whole of the long-standing "a published catalog never comes + # back from discover": the catalog was there, nothing ever asked for it. + # + # So a deployment must say what it wants. This asks for weather, which is + # what the mocks here publish; widen it per capability as more are + # published. + federationIntent: '__FEDERATION_INTENT__' + federationPublishRetrySeconds: "__FEDERATION_PUBLISH_RETRY_SECONDS__" + +modules: + + # ---------------------------------------------------------------- consumer + # + # What the experience layer posts to, and the only address it knows. It + # decides nothing: discovery goes to the network layer, everything + # transactional to the provider layer, and context.bppUri travels onward in + # the body untouched. + - name: consumer + path: /consumer/ + handler: + type: std + # bap because this module SENDS. role sets direction, not participant type. + role: bap + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + router: + id: router + config: + routingConfig: /app/config/routing-consumer.yaml + # No validateSign: the experience app is inside the trust boundary, so + # there is no network signature to check inbound. + steps: + - validateSchema + - addRoute + + # ----------------------------------------------------------------- network + # + # publish and discover. Routes both to this network's discovery service and + # enforces no payload shape of its own. + - name: network + path: /network/ + handler: + type: std + # bpp because this module RECEIVES. + role: bpp + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + router: + id: router + config: + routingConfig: /app/config/routing-network.yaml + steps: + - addRoute + + # ---------------------------------------------------------------- provider + # + # Where a select is answered -- or sent on, when it was addressed to another + # network. + - name: provider + path: /provider/ + handler: + type: std + role: bpp + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + + # Only this module has one: a capability step turns an upstream's answer + # back into Beckn through a mapping file, and no other module calls an + # upstream. + mapper: + id: jsonmapper + config: + fetchTimeout: "10s" + cacheTTL: "1m" + negativeTTL: "1m" + maxMappingBytes: 262144 + maxCacheEntries: 200 + + providerSteps: + - id: WeatherObservation + config: + bindingKeys: "__MAUSAMGRAM_BINDING_KEY__" + # header, not basic: IMD issues one opaque token, and basic would + # build the header itself from a username and password pair. + __MAUSAMGRAM_PARTICIPANT_ID__: + authScheme: header + headerName: Authorization + headerValueEnv: MAUSAMGRAM_AUTH + + - id: MandiPrice + config: + bindingKeys: "__AGMARKNET_BINDING_KEY__" + # Agmarknet issues short-lived tokens. The field names are this + # provider's spelling, not a standard. tokenTtl is an estimate -- + # the response carries no expiry -- and a 401 drops the held token. + __AGMARKNET_PARTICIPANT_ID__: + authScheme: tokenQuery + tokenUrl: "__AGMARKNET_TOKEN_URL__" + tokenUserField: access_name + tokenUserEnv: AGMARKNET_ACCESS_NAME + tokenSecretField: password + tokenSecretEnv: AGMARKNET_PASSWORD + tokenResponseField: token + queryName: token + tokenTtl: 10m + + - id: KnowledgeAdvisory + config: + bindingKeys: "__VISTAAR_BINDING_KEY__" + __VISTAAR_PARTICIPANT_ID__: + authScheme: oauth2 + tokenUrl: "__VISTAAR_TOKEN_URL__" + clientIdEnv: VISTAAR_CLIENT_ID + clientSecretEnv: VISTAAR_CLIENT_SECRET + # A retrieval answer is larger than a price row: four passages + # measured 15.7 KB against the live corpus. + maxResponseBytes: 1048576 + + - id: AgricultureFacility + config: + bindingKeys: "__POCRA_BINDING_KEY__" + __POCRA_PARTICIPANT_ID__: + authScheme: none + # Must stay 1. POCRA answers 200 with an empty catalogue after + # roughly twenty calls in a few minutes. That is indistinguishable + # from a genuine "nothing nearby", so raising this risks telling a + # farmer there is no facility when there is. + searchConcurrency: 1 + + # Inbound signature validation is OFF: validateSign is absent, so this + # tier accepts unsigned traffic, and it answers unsigned because signAck + # is absent too. + # + # OUTBOUND signing is on: sign is listed, so a request forwarded to + # another network leaves with this network's signature. + steps: + - validateSchema + # There is no step here for "is this request ours to answer?". Each + # capability step asks it first, before its own binding check, so a + # deployment gets it by serving capabilities at all -- there is no list + # to leave it out of, and a fifth capability added below cannot miss it. + # + # Who we are comes from this module's subscriberId above; no step + # repeats it. A request for us is served exactly as before. One for + # anybody else is sent to its own context.bppUri, which the consumer + # copied from the catalog it chose. + # + - WeatherObservation + - MandiPrice + - KnowledgeAdvisory + - AgricultureFacility + # Signing is config, not code: a cross-network call leaves signed + # because the step is listed here. + # + # AFTER the capability steps, and that ordering matters. reqpreprocessor + # resolves this request's subscriber id FROM THE BODY, so on a request + # addressed to another network it resolves to THEM. The capability step + # above corrects it to us when it decides the request is not ours. + # Signing before that ran would look up the other network's keyset and + # fail with "failed to get signing key: keyset not found". + # + # It runs on every request, as every step does. On one that stays here + # that costs a signature nobody reads -- the header is set on the + # outbound request, and a request answered locally is never sent. + - sign + + # --------------------------------------------------------- provider publish + # + # This network's catalogue system publishing inward. A module of its own, not + # another action above: the routing step fails any action missing from its + # config, and listing select there would proxy it to the network layer instead + # of answering it locally. + - name: publish + # A TRAILING SLASH, so the action survives. Routing takes the action from + # what is left of the URL after the mount path is stripped, so an exact + # /publish mount leaves nothing and addRoute fails with "endpoint '' is not + # supported". Published to as POST /publish/publish. + path: /publish/ + handler: + type: std + # bap because this module SENDS. + role: bap + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + router: + id: router + config: + routingConfig: /app/config/routing-publish.yaml + steps: + - validateSchema + - addRoute + + # ------------------------------------------------------------------- crawl + # + # Operator endpoints: /crawl/trigger, /crawl/peers, /crawl/status. + # + # Not published through the front door, which is what keeps them off the + # network -- the adapter has no signed-operator identity to check them against. + - name: crawl + path: /crawl/ + handler: + type: catalogCrawl + authDisabled: true + basePath: /crawl diff --git a/networkofnetwork/config/default.conf.template b/networkofnetwork/config/default.conf.template new file mode 100644 index 00000000..2108711b --- /dev/null +++ b/networkofnetwork/config/default.conf.template @@ -0,0 +1,152 @@ +# The network's front door. +# +# A network is known by its networkId, which is a domain. Everything another +# network needs therefore has to answer on that one name: the two published +# documents, and the Beckn endpoints. This server is what makes that true +# locally -- it is aliased to the networkId on the shared bridge, and it is the +# only container on it. +# +# It is also the whole of this network's public surface. The registry, the +# discovery service and the crawl endpoints are not reachable through it, which +# is what makes the boundary real rather than decorative. +# +# nginx:alpine runs envsubst over this file at start-up, so nothing below names +# a host or a port that is not supplied by the environment. + +server { + # TLS, because the contract says a network is reached over https and the + # registry's Participant schema pins baseUrl to ^https://. The certificate + # is signed by the local CA in networks/ca, and another network verifies + # against it. + # + # What this still cannot demonstrate is domain OWNERSHIP -- a laptop cannot + # -- which is why the CA is trusted by path rather than installed anywhere. + listen 443 ssl; + server_name ${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + # The two published documents, served as files. Static on purpose: the + # contract calls these publication points, not APIs, and a file is the + # smallest thing that satisfies it. + location /.well-known/ { + root /usr/share/nginx/html; + default_type application/json; + add_header Cache-Control "no-cache"; + } + + # Discovery: where another network's crawl sends its signed discover. + # + # This is the path the registry entry's baseUrl carries, so it is what a + # descriptor publishes as services.discovery and what admission records. + # + # A PROXY, not a second implementation. A signed discover from elsewhere + # reaches the same module a local caller reaches, or the two drift and only + # the federated path goes untested. + # + # Two locations because baseUrl cannot end in a slash -- the Participant + # schema's pattern refuses it -- so the exact form has to be matched as well + # as the subtree below it. + location = /network { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/network/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + location /network/ { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/network/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + + # The provider endpoints: where another network's select, init, confirm or + # status arrives, having been addressed here by a catalog's bppUri. + # + # The other direction of the same contract -- /network is how another + # network FINDS what we publish, this is how it then ACTS on it. + location = /provider { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/provider/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + location /provider/ { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/provider/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + + # Anything else is not part of the contract. 404 rather than a default page, + # so a misconfigured caller sees a refusal it can act on. + # + # /consumer, /publish and /crawl are deliberately among the "anything else": + # the first two are this deployment's own callers and the third is an + # operator endpoint, and none of them is another network's business. + location / { + return 404; + } +} + +# This network's mock upstream providers. +# +# SEPARATE server blocks, not more locations on the one above: these are +# different hosts, and the block above is this network's public surface -- +# another network's business. These are our own providers, reached only from +# inside. +# +# They are https on a hostname rather than http://mock-imd:9100 because the +# registry's Participant schema pins a provider's baseUrl to ^https:// exactly +# as it does a network's. The front door terminates TLS with the local CA the +# adapter already trusts, so the seeded URL is one the stack can actually call. +# +# One block each with a literal proxy_pass, rather than one block picking the +# upstream from $host: a proxy_pass carrying a variable is resolved per request +# and needs an explicit resolver, which is three more moving parts for two +# static destinations. +# +# Only the two providers that HAVE a mock. vistaar and pocra are in the +# certificate but have no container, so a call to them fails at the front door +# rather than at DNS. +server { + listen 443 ssl; + server_name mausamgram.${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + proxy_pass http://${MOCKIMD_HOST}:${MOCKIMD_INTERNAL_PORT}; + include /etc/nginx/conf.d/proxy-headers.inc; + } +} + +server { + listen 443 ssl; + server_name agmarknet.${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + # The token endpoint the mock does not have. + # + # mockagmarknet checks for a FIXED token in the query string and issues + # none -- the real Agmarknet has a separate endpoint for that, and the mock + # reproduces only the checking half. So the adapter's tokenQuery scheme had + # a tokenUrl with nothing behind it and every mandi select failed at the + # exchange, before the provider was ever called. + # + # Answered here rather than by changing the mock: this is local scaffolding + # for a credential that is not real, and keeping it in the front door means + # the adapter still performs a genuine exchange over TLS rather than having + # the auth step stubbed out. + # + # local-mandi-token is mockagmarknet's own default, and the field name is + # what tokenResponseField in the adapter config asks for. + location = /token { + default_type application/json; + return 200 '{"token":"local-mandi-token","expires_in":600}'; + } + + location / { + proxy_pass http://${MOCKAGMARKNET_HOST}:${MOCKAGMARKNET_INTERNAL_PORT}; + include /etc/nginx/conf.d/proxy-headers.inc; + } +} diff --git a/networkofnetwork/config/init-crawler-db.sh b/networkofnetwork/config/init-crawler-db.sh new file mode 100755 index 00000000..f0427cd1 --- /dev/null +++ b/networkofnetwork/config/init-crawler-db.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# The crawler keeps its bookkeeping -- queues, cursors, staged catalogs -- in a +# database of its own on the discovery server. +# +# Created here, by postgres' own init hook, rather than by hand after the fact. +# Destroying a stack takes the volume with it, and without this the adapter +# restart-loops on "applying 0001_init.sql" against a database that is not there. +set -e +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE DATABASE catalogcrawler; +EOSQL diff --git a/networkofnetwork/config/proxy-headers.inc b/networkofnetwork/config/proxy-headers.inc new file mode 100644 index 00000000..4f354511 --- /dev/null +++ b/networkofnetwork/config/proxy-headers.inc @@ -0,0 +1,7 @@ +# Shared by every proxied location. +# +# The signature is over the body and the created/expires pair, so the body must +# arrive byte-identical. Buffering is fine; rewriting is not. +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_request_buffering on; diff --git a/networkofnetwork/config/registry/imports/realm-export.json b/networkofnetwork/config/registry/imports/realm-export.json new file mode 100644 index 00000000..0f5efa16 --- /dev/null +++ b/networkofnetwork/config/registry/imports/realm-export.json @@ -0,0 +1,2321 @@ +{ + "id": "sunbird-rc", + "realm": "sunbird-rc", + "displayName": "Sunbird Rc Core", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "a772a1cd-7904-4e5c-a864-5041fa69d491", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "42dba8cf-f483-4668-a087-cba46ed86ad2", + "name": "admin", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "5fa4077d-1686-4506-97a6-5bce1bce59dc", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "network_operator", + "description": "Network Operator - onboards and governs Providers", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "registryOperator", + "description": "Registry Operator - manages SchemaRegistry, Participant and ProviderSchema records", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "270ecc82-3249-475c-a851-d3ea162059b8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "0de39ec0-7602-4aa2-b54d-ab12e9bdb76f", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4259031b-736e-49eb-9e70-4a312a48e211", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9887e071-49b0-464b-b6fe-a1c585a709c7", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7d61f967-1dce-482f-96e5-9eff79eb4851", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-events", + "manage-identity-providers", + "manage-clients", + "view-identity-providers", + "manage-authorization", + "view-users", + "manage-users", + "manage-events", + "manage-realm", + "impersonation", + "view-authorization", + "query-clients", + "create-client", + "view-clients", + "query-users", + "query-realms", + "view-realm", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "e93b1761-fb32-46c5-bfa2-4b853c7b5573", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "bc32a137-07a1-40f0-b9fd-a6e64e27f99b", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4b5abd90-d6a2-4981-a50f-520292496f0b", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "99d2ed5f-00a9-44ed-8b9f-bdd7ba3facb8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "3fbd2cd5-0698-490e-a52f-ef528d001a62", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9b8b4f1c-5ed6-49ca-bec3-0a9a4867ad26", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7e6341ff-a1d8-4400-af94-3a007a06706a", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ad0c87da-9f34-4992-a83a-f6b924f1944d", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "badb0d4d-06da-45e8-a777-ef47f712d3ed", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "f8f48f0f-bd2a-4cb7-9b77-af69b9805c25", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ca0b1e94-6578-4295-abf4-ae99f7df7595", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ff7230eb-7dae-44a5-8f68-f68747f35589", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "890d054b-86f9-49f5-8dd9-14f62aa956de", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "627f3f4c-58e3-49f3-9989-a05d4d0a8752", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-api": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "2e9bfeff-129e-4072-9617-5847644aac24", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "attributes": {} + } + ], + "account": [ + { + "id": "5694c2d0-6d02-4182-bb09-78f4f5f1ec2d", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0612622f-dae7-48f8-8985-fe7e5ab8acc7", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "eeefbd57-94b8-4d7d-bf2f-075c39ccb746", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "9e9165b9-1170-47ab-802a-aecffefb3ab7", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "a8d0a100-e382-49ba-ac42-48dbf815a2de", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "08772792-146d-4676-ba2d-ce56b0104263", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0a2e7893-784e-47ef-ba35-4a26901350c0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + } + ], + "registry-frontend": [] + } + }, + "groups": [], + "defaultRole": { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "sunbird-rc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "3cc9ac60-b67d-4c57-8005-acd4d236b2dc", + "createdTimestamp": 1634296700339, + "username": "service-account-admin-api", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "admin-api", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-sunbird-rc", + "admin" + ], + "clientRoles": { + "realm-management": [ + "manage-users", + "manage-realm" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "username": "no-user", + "enabled": true, + "emailVerified": false, + "credentials": [ + { + "type": "password", + "value": "no-user-password", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-sunbird-rc", + "network_operator", + "registryOperator" + ] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "01326f76-7838-47fc-ae62-399a75c5ae38", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "f871d6fc-d997-4ac6-99fe-d797955bc9f0", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "8ab32c51-9aa0-4e28-80bf-0d6b53151354", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "25962708-6d45-47d9-8935-5db159234aac", + "clientId": "admin-api", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "*", + "http://localhost:4200/", + "http://localhost:4200/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "84ae9d6c-424f-47f0-9d4d-f2e98fed7339", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "98406938-b8db-4992-8519-917054f6ed0e", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "90d6b17a-5a06-4546-8091-960301f8147e", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b245b10b-606c-417c-bbc0-8f81a7a992a6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "961a8a92-1598-48ff-adee-1e5fee0df757", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "69c04ae8-6669-48e7-8234-08986a7f490d", + "clientId": "registry-frontend", + "name": "Registry Frontend", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "login_theme": "sunbird-rc", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b777b14f-b0e8-4da5-a802-092803319cbe", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/sunbird-rc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/sunbird-rc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "7160f35d-97d3-4730-9769-4b03b32e5191", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "b4695333-f842-4ef7-874e-99260e77b9cb", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "fc23d4b8-76c5-4e59-9305-10846b8bcefe", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "08f06ba5-3e60-4a0a-aaf9-f70bfc7ae99e", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e28227ee-cb54-4557-8908-01864f80055f", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "364a632f-b66a-4ca4-8bbe-ec2ce1af9df8", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "42cad815-4de0-4b67-abca-7f7aaf55e589", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "e3302def-d387-465c-a420-7ab01570e94a", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "63a3cb24-b124-428e-ac0f-253eb1fe485d", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2eb041ca-970a-45fd-a167-2a497579bc8c", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "2c02e9ce-7d86-4a5b-84b8-cf93114ddf26", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "2d39b55e-46c4-4dec-bd83-f081c708f544", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "e869fffd-f801-492d-a6c7-d6c6143817e5", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "daeb863b-4773-4668-98fb-403e93414eb2", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a98b9f93-ec39-4f3c-acb7-cd92161e3717", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c59a379a-3934-4e6f-be20-1803b0786d97", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0a299a91-277c-4f38-95e7-6c520f892b63", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "d51531b3-a8ea-44e2-a48f-69991f9166cc", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "b4b33a89-01db-468e-9a4e-c5ac58304fed", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "d1727e08-fb90-49ce-bb7e-d7a55a50ee64", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "564ae79d-e505-416c-b794-ddd3a3c21fde", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "986d2d9e-0d0d-4317-92b3-a7a8d9bec4de", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "e14ec2e9-0d24-4960-8779-00f769ccc01b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "8f044609-b615-4522-b9e7-8361cb08b0b3", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "341b838d-ba26-4280-b0af-3e5d3403c938", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "26a17e7e-1a6e-439f-a54a-05a63d1c91fb", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "2489b2c0-5b3a-4404-8428-be4ce653da72", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "8c7e1d96-bf79-42e6-9360-b5e7b8dddc8d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "01f959d8-123f-4263-ad6e-386e8b4d0e05", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "b47a30d6-3c49-4bbe-b15e-b0eb6cffc0f3", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "2cc8166d-6d77-4a85-9945-bc22b0f550e3", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "1d87a800-cc11-4d85-aa76-8a6d828e2269", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "42cc1538-f83a-4a94-b5a5-d16b80824a02", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "13ff6325-822e-4087-9e74-086de77fe89e", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e501527e-dec8-4fde-a539-8e77d86b5081", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "b4b519d8-070d-4dab-854e-d6e3b2b36205", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "ed42958b-6e78-42a9-9f40-2e40bd6c8dd0", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "saml-user-property-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "saml-role-list-mapper", + "oidc-full-name-mapper" + ] + } + }, + { + "id": "572219a7-3053-4940-87c5-ad94a6fb6dd3", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "8704a420-bf90-4e12-9e33-d21f39a2385b", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "8cf98455-916b-487a-8322-3f5d283400c2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "0b43488d-108b-41f5-ab6d-56a4ac8ff63c", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "6f0ebf9b-900a-4ca9-8fea-90719f218689", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b5a486b3-abf9-49a1-8dc6-dc5e20776681", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "be43420e-8ffc-4f53-b745-f2f0cd88f000", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f749bd77-72f2-4dc4-a65e-dd89b255f12f", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "a541cbb1-8a27-4061-a389-9f24ba1c2eb1", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "18504bf7-63f1-4848-b565-6348fa6b0048", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "497d8386-9a74-4b7b-a4e6-78bbbbb5d795", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "d964973c-2106-4db3-a814-f7a34ae7a1ce", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ff1b250-85b1-4709-8719-3eabcb34493f", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "43f683be-52e7-43cd-aa9e-6318b8079ad0", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "5195e46e-b2c3-49e3-8987-db8b19c45fc5", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "aca5480d-842c-4fa9-aff1-b8af8d51d82a", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "66a00ea9-7ec1-4450-905a-14b7f3f8e4bf", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "a24445a4-1988-4b5b-bde6-fa36dbd07e03", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "31c2bc3b-6eb1-4c4f-8464-3528f7445ef7", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4ccc8da9-0e1e-4f30-99c0-e2139f671a80", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "fa1183d1-af7a-40dd-ba85-d7d37867639c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "f5b5e49b-7cc9-4011-b9fa-60f0ef65e735", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5cf727d2-e25a-4c88-a55b-4eea9134adb1", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ec4f009-2f16-464b-8feb-a0bdc0dad195", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "a695a5e0-326f-4658-8518-a1769d97ad5f", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4611e0a9-a6a4-4e32-8500-e68877b464b1", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "55be727b-a17b-40c5-a5c3-c2d72c7f54cb", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "00ec3b72-3abb-4db3-ad2f-595bc2f7e086", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "82e4f3a3-744b-4d8a-8785-6eabaf9e05c9", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "0c0312b2-db7c-433c-ab15-20b18bfb5f4a", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ee0faa63-999c-42e6-8189-c22a5cc14dc5", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "506eed8f-88c9-4978-b13a-886f1efc45c0", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "14.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} diff --git a/networkofnetwork/config/registry/schemas/Participant.json b/networkofnetwork/config/registry/schemas/Participant.json new file mode 100644 index 00000000..fd8e5e93 --- /dev/null +++ b/networkofnetwork/config/registry/schemas/Participant.json @@ -0,0 +1,217 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Participant", + "type": "object", + "required": [ + "Participant" + ], + "properties": { + "Participant": { + "$ref": "#/definitions/Participant" + } + }, + "definitions": { + "Participant": { + "description": "Someone the network deals with. `type` says which kind, and decides which of the remaining fields apply: a node speaks Beckn and is addressed by its participantId; an upstream is an ordinary API our adapter calls. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": [ + "participantId", + "name", + "type", + "status", + "baseUrl" + ], + "properties": { + "participantId": { + "$ref": "#/definitions/ParticipantId" + }, + "name": { + "description": "Human label.", + "type": "string", + "minLength": 1, + "maxLength": 200, + "pattern": "\\S" + }, + "type": { + "$ref": "#/definitions/ParticipantType" + }, + "status": { + "$ref": "#/definitions/Status" + }, + "baseUrl": { + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https for a node, since that is its wire identity; an upstream may be plaintext, because nothing in the registry is sent to it.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + }, + "role": { + "description": "What this party does on the network, as a list. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs. A network is ONE participant however many adapters it runs, so this says what the network serves rather than which process exists -- a split three-adapter deployment and a consolidated one carry the same value. Required on a node. Permitted on an upstream, where nothing reads it. Search it with `contains`, not `eq`: an exact match against a list finds nothing, while `contains` matches both a list and an older single-string value.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "consumer", + "provider", + "network" + ] + } + }, + "keys": { + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header. Required on a node, since a node's signatures are verified against them. Permitted on an upstream, which does not sign anything this stack verifies.", + "type": "array", + "minItems": 1, + "maxItems": 8, + "uniqueItems": true, + "items": { + "$ref": "#/definitions/PublicKey" + } + }, + "revision": { + "description": "The revision of the peer's descriptor this record was built from. Only ever increases: a descriptor offering a revision at or below the stored one is a replay and is refused, which is what stops a withdrawn or superseded declaration being presented as current. Applies to role network.", + "type": "integer", + "minimum": 1 + }, + "federationRole": { + "description": "What this entry is, in relation to us. self is this deployment's own entry -- the network reading it. source is a peer we consume from: our crawler asks it for its catalogs. requester is a peer that consumes from us: it calls us and we verify its signatures, but we do not crawl it. both is a peer in both directions, which is what mutual admission produces. STORED rather than worked out: deciding whether an entry is our own by comparing its id against configuration has been got wrong twice, most memorably when a crawler tried to crawl itself. Absent on an entry written before this field existed, which is read as both. Applies to role network.", + "type": "string", + "enum": [ + "self", + "source", + "requester", + "both" + ] + }, + "schemaPacks": { + "description": "The schema packs a peer network declared in its descriptor, stored verbatim. Recorded rather than interpreted: what a peer can speak is its statement, and rewriting it here would make our copy disagree with the document it was read from. The crawler derives the JSON-LD context URL it filters on from these, because a pack URL names the pack's attributes.yaml while a message is interpreted against the matching context.jsonld. Applies to role network.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "projectionTtl": { + "description": "How long, in seconds, this peer permits us to keep what we crawled from it. Declared by the peer in its descriptor and recorded here verbatim: the network that owns the data decides how stale a copy of it may get, and a crawler that chose its own value would be deciding that for someone else. Always positive on a stored record -- a peer declaring 0 forbids caching entirely and is refused at admission rather than cached against its wishes. Applies to role network.", + "type": "integer", + "minimum": 1 + } + }, + "allOf": [ + { + "description": "A node speaks Beckn: it needs a role and keys, and its id is its wire identity so it must be a hostname.", + "if": { + "properties": { + "type": { + "const": "node" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "role", + "keys" + ], + "properties": { + "participantId": { + "maxLength": 253, + "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$" + }, + "baseUrl": { + "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + } + } + } + } + ] + }, + "PublicKey": { + "description": "A public key, held as material: it is public, so there is nothing to protect. Identified by the osid the registry assigns on write -- which is what a sender names in the Authorization header, and so the only id a verifier can look up. alg carries the purpose: ed25519 signs, x25519 encrypts.", + "type": "object", + "additionalProperties": true, + "required": [ + "alg", + "key", + "validFrom", + "status" + ], + "properties": { + "alg": { + "description": "The purpose, and the only thing that carries it now: ed25519 signs, x25519 encrypts. Both curves are 32-byte keys.", + "type": "string", + "enum": [ + "ed25519", + "x25519" + ] + }, + "key": { + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. Bare -- no encoding label in front of it. A truncated or wrong-curve key fails at write time.", + "type": "string", + "pattern": "^[A-Za-z0-9+/]{43}=$" + }, + "validFrom": { + "type": "string", + "format": "date-time" + }, + "validUntil": { + "description": "Absent means open-ended. Overlap with the next key's validFrom is the rotation window.", + "type": "string", + "format": "date-time" + }, + "status": { + "description": "revoked withdraws the key early. Effective only once the reader refreshes.", + "type": "string", + "enum": [ + "active", + "revoked" + ] + } + } + }, + "ParticipantId": { + "description": "Stable id, and the only id. For a node this is its network identity -- what field 1 of the Authorization header names, and what a signature is verified against; for an upstream it is the Beckn offer.provider.id.", + "type": "string", + "maxLength": 253, + "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" + }, + "Status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "ParticipantType": { + "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", + "type": "string", + "enum": [ + "node", + "upstream" + ] + } + }, + "_osConfig": { + "uniqueIndexFields": [ + "participantId" + ], + "indexFields": [ + "status", + "type", + "baseUrl" + ], + "privateFields": [], + "roles": [ + "registryOperator" + ], + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ] + } +} diff --git a/networkofnetwork/config/registry/schemas/ProviderSchema.json b/networkofnetwork/config/registry/schemas/ProviderSchema.json new file mode 100644 index 00000000..5744c682 --- /dev/null +++ b/networkofnetwork/config/registry/schemas/ProviderSchema.json @@ -0,0 +1,81 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ProviderSchema", + "type": "object", + "required": ["ProviderSchema"], + "properties": { "ProviderSchema": { "$ref": "#/definitions/ProviderSchema" } }, + + "definitions": { + + "ProviderSchema": { + "description": "One row is one provider and one capability. What varies per Beckn action — the URL, the method, the mapping, the timeout — varies inside actions[]. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": ["bindingKey", "participantId", "capabilityCode", "status", "actions"], + "properties": { + "bindingKey": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{2,63}\\|openagrinet:[A-Z][A-Za-z0-9]*$" + }, + "participantId": { "$ref": "#/definitions/ParticipantId" }, + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "status": { "$ref": "#/definitions/Status" }, + + "actions": { + "description": "At least one. uniqueItems compares whole objects, so it cannot pin one entry per action — verify/records.py does.", + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { "$ref": "#/definitions/ActionBinding" } + } + } + }, + + "ActionBinding": { + "description": "How to call this provider for one Beckn action: where, how, and with which mapping. Anything the upstream needs that the Beckn body cannot express is the adapter plugin's work, not a field here. status is per action, so one can be retired without touching the others. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": ["action", "method", "path", "mappings", "status"], + "properties": { + "action": { "$ref": "#/definitions/Action" }, + "method": { "type": "string", "enum": ["GET", "POST"] }, + "path": { "$ref": "#/definitions/Path" }, + "mappings": { "$ref": "#/definitions/MappingPath" }, + + "timeoutMs": { "type": "integer", "minimum": 1000, "maximum": 120000, "default": 15000 }, + "retryMax": { "type": "integer", "minimum": 0, "maximum": 5, "default": 0 }, + + "status": { "$ref": "#/definitions/Status" } + } + }, + + "Action": { + "description": "beckn v2.0.0 request actions. An on_* callback is not one: it is the response half of the action that made the call.", + "type": "string", + "enum": ["discover", "select", "init", "confirm", "status", + "track", "cancel", "update", "rate", "support"] + }, + + "MappingPath": { + "description": "Fully-qualified URL of one published mapping file, holding request: and response: as YAML block scalars. The action segment must equal the action it sits under — verify/records.py.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*/[A-Za-z0-9._~%-]*\\.(discover|select|init|confirm|status|track|cancel|update|rate|support)\\.ya?ml$" + }, + + "ParticipantId": { "type": "string", "maxLength": 253, "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" }, + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, + "Status": { "type": "string", "enum": ["active", "inactive"] }, + "Path": { "type": "string", "maxLength": 512, + "description": "Appended to that upstream's baseUrl. Single slashes: an empty segment is never deliberate, and many servers answer //a differently from /a. A trailing slash is allowed, because /api/ and /api are a distinction some APIs make.", + "pattern": "^(?!.*//)/[A-Za-z0-9._~%/-]*$" } + }, + + "_osConfig": { + "uniqueIndexFields": ["bindingKey"], + "indexFields": ["participantId", "capabilityCode", "status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/networkofnetwork/config/registry/schemas/SchemaRegistry.json b/networkofnetwork/config/registry/schemas/SchemaRegistry.json new file mode 100644 index 00000000..0efd0f1b --- /dev/null +++ b/networkofnetwork/config/registry/schemas/SchemaRegistry.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SchemaRegistry", + "type": "object", + "required": ["SchemaRegistry"], + "properties": { "SchemaRegistry": { "$ref": "#/definitions/SchemaRegistry" } }, + + "definitions": { + + "SchemaRegistry": { + "type": "object", + "additionalProperties": false, + "required": ["capabilityCode", "name", "version", "schemaUrl", "status"], + "properties": { + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "name": { "type": "string", "minLength": 1, "maxLength": 200, "pattern": "\\S" }, + "version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+$" }, + "schemaUrl": { "type": "string", "maxLength": 2000, + "pattern": "^https://openagrinet\\.github\\.io/network-specs/schema/[A-Za-z0-9]+/v[0-9]+\\.[0-9]+/[A-Za-z0-9_-]+\\.yaml$" }, + "status": { "$ref": "#/definitions/Status" } + } + }, + + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, + "Status": { "type": "string", "enum": ["active", "inactive"] } + }, + + "_osConfig": { + "uniqueIndexFields": ["capabilityCode"], + "indexFields": ["status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/networkofnetwork/config/routing-consumer.yaml b/networkofnetwork/config/routing-consumer.yaml new file mode 100644 index 00000000..64ebfb41 --- /dev/null +++ b/networkofnetwork/config/routing-consumer.yaml @@ -0,0 +1,37 @@ +# What the consumer layer does with each action. +# +# Both targets are THIS SAME adapter on another path. In the three-adapter +# quick-start they were two other containers; consolidated, the hop is a loopback +# call from one module to another. It stays a real HTTP hop rather than an +# in-process call because that is what makes the two shapes the same config: +# split these across containers and only the host in these URLs changes. +# +# targetType "url" appends the action to the path, so the base here is the module +# mount point and /discover or /select is added to it. +routingRules: + # Discovery is answered from this network's own discovery service, which the + # network module fronts. That includes catalogs crawled from other networks -- + # they were mirrored in, so a local query returns both. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network" + endpoints: + - discover + + # Everything transactional goes to the provider layer, and this layer decides + # nothing about where it ends up. Whether the request is ours to answer or + # belongs to another network is decided in ONE place -- routeByReceiver on the + # provider module, which is also where the capability steps that would + # otherwise wrongly answer it live. + # + # context.bppUri travels onward in the body untouched. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/provider" + endpoints: + - select + - init + - confirm + - status diff --git a/networkofnetwork/config/routing-network.yaml b/networkofnetwork/config/routing-network.yaml new file mode 100644 index 00000000..4b8f5be3 --- /dev/null +++ b/networkofnetwork/config/routing-network.yaml @@ -0,0 +1,10 @@ +# The network layer: publish and discover both go to this network's discovery +# service, which is the only thing behind this module. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://discovery-service:8080" + endpoints: + - discover + - publish diff --git a/networkofnetwork/config/routing-publish.yaml b/networkofnetwork/config/routing-publish.yaml new file mode 100644 index 00000000..0472bf44 --- /dev/null +++ b/networkofnetwork/config/routing-publish.yaml @@ -0,0 +1,12 @@ +# This network's catalogue system publishing inward. +# +# It goes to the NETWORK module rather than straight to discovery, so a publish +# takes the same path a publish from anywhere else takes -- one place that +# decides what reaches discovery, rather than two. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network" + endpoints: + - publish diff --git a/networkofnetwork/demo/README.md b/networkofnetwork/demo/README.md new file mode 100644 index 00000000..76676953 --- /dev/null +++ b/networkofnetwork/demo/README.md @@ -0,0 +1,267 @@ +# Network of networks — demo runbook + +Two whole OAN networks on one laptop, able to find and call each other. + +Every command and every response below was run end to end on 2026-10-07, from +an empty machine. Timings are what it actually took. + +Ports: `bharatvistar` gets 9300-9306, `mahavistara` 9400-9406. + +**Postman:** import `demo/oan-network-of-networks.postman_collection.json`. Two +folders, Bharat and Maha, with every request below already filled in. + +**B does not crawl.** `FEDERATION_CRAWL_PEERS=off` in its `.env`. With both +networks crawling, each mirrors the other, and B ends up re-importing its own +catalogue through A's copy of it -- and could overwrite its own with a stale +one. B still ANSWERS A's crawl; that is the network module, nothing to do with +this switch. And A's mandi is published `visibleTo` A alone, so there is +nothing of A's for B to take even if it did crawl. The script +allocates the next free block per network, so a third network just works. + +--- + +## Before you start + +Docker, Python 3, `openssl`, `curl`. The adapter image must exist locally: + +```bash +cd ~/Workspace/OAN/network-adapter +docker build --network=host -f Dockerfile.adapter-with-plugins -t oan/network-adapter:local . +``` + +`--network=host` is not optional on this machine — IPv6 is broken here and +`go mod download` hangs without it. Takes about 4 minutes cold. + +Everything else runs from `helmcharts/networkofnetwork`. + +--- + +## 1. Stand up two networks (~75s each) + +```bash +cd ~/Workspace/OAN/helmcharts/networkofnetwork + +make network NETWORK_ID=bharatvistar.oan.local +make network NETWORK_ID=mahavistara.oan.local +``` + +Each one brings up its own registry, Keycloak, discovery service, Postgres, two +mock providers, an adapter and a front door — then seeds the registry and +renders the adapter config. Re-running it is also how you apply a config +change. + +Check what is deployed: + +```bash +make ls +``` + +``` +bharatvistar.oan.local 9300-9306 +mahavistara.oan.local 9400-9406 +``` + +--- + +## 2. Each network hosts its own identity (instant) + +```bash +make host-descriptor NETWORK_ID=bharatvistar.oan.local +make host-descriptor NETWORK_ID=mahavistara.oan.local +``` + +``` +bharatvistar.oan.local now hosts its descriptor at revision 1 + descriptor networks/bharatvistar.oan.local/.well-known/openagrinet + keys networks/bharatvistar.oan.local/.well-known/jwks.json +``` + +Two signed documents per network, served over TLS at the network's own domain. +One identity, one keypair: the `networkId` IS the `participantId`, the key that +signs the descriptor is the key that signs traffic, and its public half is the +registry entry's key. + +--- + +## 3. Each network admits the other (instant) + +```bash +make admit NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +make admit NETWORK_ID=mahavistara.oan.local PEER=bharatvistar.oan.local +``` + +``` +admitted mahavistara.oan.local at revision 1, status active + discovery https://mahavistara.oan.local/network/discover + projectionTtl 3600s + keys 1 published + relationship both +``` + +This fetches the other network's descriptor over TLS, verifies its signature +against the key it publishes, and records the decision to deal with it. Nothing +is trusted because it is on the same machine. + +--- + +## 4. Give each network different data + +This is the story: **A has only mandi prices, B has only weather.** Neither is +missing a capability — both adapters serve all four — the difference is what +each one has PUBLISHED. + +```bash +# A publishes mandi +curl -s -X POST localhost:9300/publish/publish \ + -H 'Content-Type: application/json' -d @demo/publish-catalog-mandi.json | jq . + +# B publishes weather +curl -s -X POST localhost:9400/publish/publish \ + -H 'Content-Type: application/json' -d @demo/publish-catalog.json | jq . +``` + +Both answer `ACCEPTED`. Each catalog is published `visibleTo` both networks, and +carries its own `bppId`/`bppUri` — catalog fields, which is how a consumer later knows where to +send a select. + +--- + +## 5. A crawls B + +```bash +curl -s -X POST localhost:9300/crawl/peers +``` + +No arguments. The crawler reads the registry: it lists the ACTIVE participants +with role `network`, drops its own entry, and takes each peer's discovery URL, +`projectionTtl` and `schemaPacks` from the record `make admit` wrote. So "who do +I crawl" is the set of networks you have admitted, and nothing else. + +It also re-crawls on its own every `federationRefreshSeconds` (900s here), so in +a long demo this happens again without being asked. + +``` +catalogcrawler: crawled a network networkId=mahavistara.oan.local catalogs=1 +``` + +--- + +## 6. The farmer asks ONE network and gets both + +```bash +curl -s -X POST localhost:9301/discover \ + -H 'Content-Type: application/json' -d @demo/discover-farmer.json | jq . +``` + +``` +catalog:mandi-price:agmarknet.bharatvistar -> select at bharatvistar.oan.local +catalog:weather-observation:mausamgram.mahavistara -> select at mahavistara.oan.local +``` + +One query, against network A only. A's own mandi catalogue, and B's weather +catalogue which A crawled. The farmer never named a network and never knew there +were two. + +The payload carries no `context.networkId`. An omitted audience means no +audience filter at all, so the farmer sees everything A holds — its own and its +mirrored copies alike, without having to know that either exists. + +**Use the no-networkId form for the demo.** Naming an audience filters to +catalogs published `visibleTo` that audience, and a crawled copy is stamped +with the audience the crawl asked with — which is not reliably ours: + +``` +ask A with no networkId -> mandi-price, weather-observation +ask A as mahavistara.oan.local -> mandi-price, weather-observation +ask A as bharatvistar.oan.local -> mandi-price (the mirror is missing) +``` + +The crawl asks the peer twice — once as the peer, once as us — and ours goes +last so our copy should win. Both calls are made (the peer's nginx log shows +two POSTs), but the second comes back empty while the same request sent by hand +returns the catalog. Unresolved; see section 3 of the todo. It does not affect +the farmer path, which names no audience. + +--- + +## 7. Select each one — one local, one across the network + +The consumer picks a catalog and copies its `bppId` into the select's +`context.receiverId`, and its `bppUri` into `context.bppUri`. Nothing else +decides where the request goes. + +(The context names the two ends by DIRECTION — `senderId` and `receiverId` — +not by Beckn role. `bapId`/`bppId` are the older spelling and appear in no +payload here. `bppUri` stays, because it addresses the PROVIDER a catalog is +served from rather than saying who sent the envelope.) + +**Mandi — `receiverId` is A, so A answers it itself:** + +```bash +curl -s -X POST localhost:9300/provider/select \ + -H 'Content-Type: application/json' -d @demo/select-mandi-local.json | jq . +``` + +Answers `on_select` from `agmarknet.bharatvistar`, with prices from A's own mock. + +**Weather — `receiverId` is B, so A sends it to B:** + +```bash +curl -s -X POST localhost:9300/provider/select \ + -H 'Content-Type: application/json' -d @demo/select-cross-network.json | jq . +``` + +Answers `on_select` from `mausamgram.mahavistara` — the OTHER network's provider, +with three days of forecast. + +Both requests arrive at the SAME endpoint on the SAME network. The only +difference is the `receiverId` the farmer's consumer copied out of the catalog. +In +A's log: + +``` +mahavistara.oan.local is not bharatvistar.oan.local; sending it to http://... +Signature generated: ... +Routing to ctx.Route to &model.Route{TargetType:"url", ...} +``` + +A recognised the request was not its own to answer, signed it as **itself**, and +sent it to the address the catalog carried. Nothing in A is configured with who +its peers are. + +--- + +## Tear down + +```bash +make down NETWORK_ID=bharatvistar.oan.local # stop, keep the data +make destroy NETWORK_ID=bharatvistar.oan.local # delete the data, free the ports + +make teardown # destroy EVERY network, asks first +``` + +--- + +## Known rough edges — say these before someone finds them + +| | | +|---|---| +| `federationIntent` decides what breadth of a peer's catalogue is mirrored, and is set to weather here. Discovery refuses an intent that narrows nothing, so there is no "mirror everything". | | +| Only `mausamgram` and `agmarknet` have mock containers. `vistaar` and `pocra` are in the certificate but have nothing behind them. | | +| `validateSign` is in no step chain, so a cross-network select is answered whether or not it is signed. | | +| Nothing re-reads a peer's descriptor on a schedule — `admit` is also the refresh, and a person has to run it. | | +| Nothing removes a mirrored catalog. Refresh happens; withdrawal does not, because discovery has no delete endpoint. | | +| A select routes on `receiverId` alone. Any id a caller invents is enough, and the address comes from the same request. | | + +--- + +## If something breaks + +| symptom | cause | +|---|---| +| adapter restart-loops on `applying 0001_init.sql` | a `destroy` took the crawler database with it. `CREATE DATABASE catalogcrawler` by hand, or destroy and bring it back up. | +| a rebuilt image seems to have no effect | `docker restart` reuses the container's existing layer. Re-run `make network NETWORK_ID=...`, which recreates. | +| `failed to get signing key: keyset not found` on a cross-network select | `sign` is running before the capability steps. It must come after them. | +| `endpoint '' is not supported` | a module mounted at an exact path with no trailing slash — routing reads the action from what is left of the URL. | +| a crawl returns `catalogs=0` | the intent, or the schema packs. Both are silent: discovery answers 200 with nothing. | +| a peer's front door answers as yourself | two stacks on one bridge share a DNS alias. Address containers by their project-prefixed names. | diff --git a/networkofnetwork/demo/discover-farmer.json b/networkofnetwork/demo/discover-farmer.json new file mode 100644 index 00000000..d2644706 --- /dev/null +++ b/networkofnetwork/demo/discover-farmer.json @@ -0,0 +1,19 @@ +{ + "context": { + "action": "discover", + "domain": "agriculture", + "messageId": "5e511275-d9ed-442a-964c-b4f4cc574c1b", + "timestamp": "2026-10-07T03:23:01Z", + "transactionId": "df14c457-d091-48ad-8bbc-6214001f0578", + "version": "2.0.0", + "senderId": "bharatvistar.oan.local" + }, + "message": { + "intent": { + "filters": { + "type": "jsonpath", + "expression": "$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \"Market\" || @.resourceAttributes.subjectCategories[*] == \"Weather\")" + } + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/discover.json b/networkofnetwork/demo/discover.json new file mode 100644 index 00000000..a7a1468a --- /dev/null +++ b/networkofnetwork/demo/discover.json @@ -0,0 +1,20 @@ +{ + "context": { + "action": "discover", + "domain": "agriculture", + "messageId": "5e511275-d9ed-442a-964c-b4f4cc574c1b", + "networkId": "mahavistara.oan.local", + "timestamp": "2026-10-07T03:23:01Z", + "transactionId": "df14c457-d091-48ad-8bbc-6214001f0578", + "version": "2.0.0", + "senderId": "bharatvistar.oan.local" + }, + "message": { + "intent": { + "filters": { + "expression": "$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \"Weather\")", + "type": "jsonpath" + } + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/oan-network-of-networks.postman_collection.json b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json new file mode 100644 index 00000000..5beee5d9 --- /dev/null +++ b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json @@ -0,0 +1,258 @@ +{ + "info": { + "_postman_id": "d90ecfa3-df76-4844-a51c-32d73de9a2f9", + "name": "OAN \u2014 Network of Networks", + "description": "Two OAN networks side by side, each with its own registry, discovery and adapter.\n\n**The demo, in order:**\n1. Bharat 1 \u2014 publish mandi into A\n2. Bharat 2 \u2014 search A, returns mandi only\n3. Maha 1 \u2014 publish weather into B\n4. wait ~30s, no manual crawl\n5. Bharat 2 again \u2014 now returns mandi AND weather\n6. Bharat 3 \u2014 select mandi, answered locally\n7. Bharat 4 \u2014 select weather, crosses to B\n\nBring the stack up with `make network NETWORK_ID=...` in helmcharts/networkofnetwork. Full runbook in demo/README.md.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "bharat_adapter", + "value": "http://localhost:9300" + }, + { + "key": "bharat_discovery", + "value": "http://localhost:9301" + }, + { + "key": "maha_adapter", + "value": "http://localhost:9400" + }, + { + "key": "maha_discovery", + "value": "http://localhost:9401" + } + ], + "item": [ + { + "name": "Bharat (network A \u2014 has mandi)", + "description": "Network A. Publishes its OWN mandi catalogue, and crawls B every 30s so B's weather also answers here.\n\nPorts: adapter 9300, discovery 9301.", + "item": [ + { + "name": "1. Publish mandi catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/publish/publish", + "host": [ + "{{bharat_adapter}}/publish/publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"messageId\": \"bd83dc21-38f3-48fe-9be5-b82d4a213666\",\n \"transactionId\": \"794abc60-4d9b-463f-ba1d-926724ade7f8\",\n \"timestamp\": \"2026-10-07T03:50:03Z\",\n \"version\": \"2.0.0\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\"\n ],\n \"senderId\": \"bharatvistar.oan.local\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:mandi-price:agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MH\",\n \"name\": \"Bharatvistar mandi prices\",\n \"shortDesc\": \"On-demand commodity prices for agricultural markets.\",\n \"longDesc\": \"Each Resource is one commodity at one market. Consumers discover by location or commodity and invoke the Provider for current prices.\"\n },\n \"isActive\": true,\n \"bppId\": \"bharatvistar.oan.local\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"provider\": {\n \"id\": \"agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\",\n \"shortDesc\": \"Live agricultural market price information.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:nashik:onion\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_NASHIK_ONION\",\n \"name\": \"Onion prices at Nashik\",\n \"shortDesc\": \"Current onion prices for Nashik, Maharashtra.\"\n },\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"informationMode\": \"OnDemand\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n }\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:mandi-price:agmarknet.bharatvistar\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"FULL\",\n \"visibleTo\": [\n \"bharatvistar.oan.local\"\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A's own catalogue. visibleTo names ONLY A: there is no reason for another network to mirror it, and leaving B out is what stops B crawling it back." + }, + "response": [] + }, + { + "name": "2. Discover (the farmer's search)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_discovery}}/discover", + "host": [ + "{{bharat_discovery}}/discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"discover\",\n \"domain\": \"agriculture\",\n \"messageId\": \"5e511275-d9ed-442a-964c-b4f4cc574c1b\",\n \"timestamp\": \"2026-10-07T03:23:01Z\",\n \"transactionId\": \"df14c457-d091-48ad-8bbc-6214001f0578\",\n \"version\": \"2.0.0\",\n \"senderId\": \"bharatvistar.oan.local\"\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" || @.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "No context.networkId. An omitted audience means no audience filter at all, so this returns everything A holds -- its own mandi AND the weather it crawled from B. The farmer never names a network." + }, + "response": [] + }, + { + "name": "3. Select mandi (answered locally)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/provider/select", + "host": [ + "{{bharat_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d23\",\n \"timestamp\": \"2026-10-07T06:12:01.330Z\",\n \"senderId\": \"seeker.bharatvistar.oan.local\",\n \"receiverId\": \"bharatvistar.oan.local\",\n \"senderUri\": \"https://seeker.bharatvistar.oan.local/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"provider\": {\n \"id\": \"agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"resourceIds\": [\n \"res:agmarknet:nashik:onion\"\n ]\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:nashik:onion\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"informationMode\": \"OnDemand\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n },\n \"supportedCommodities\": [\n {\n \"code\": \"149\",\n \"name\": \"Onion\"\n }\n ],\n \"market\": {\n \"district\": \"96\",\n \"state\": \"MH\",\n \"marketCode\": \"2056\",\n \"name\": \"Nashik\"\n }\n }\n }\n ]\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "context.receiverId is A, so A answers it from its own provider. Returns on_select with real prices from the mock." + }, + "response": [] + }, + { + "name": "4. Select weather (crosses to B)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/provider/select", + "host": [ + "{{bharat_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-26T06:12:01.330Z\",\n \"bppUri\": \"http://oan-mahavistara-oan-local-adapter:9200/provider\",\n \"senderId\": \"seeker-network-vistaar.da.gov.in\",\n \"receiverId\": \"mahavistara.oan.local\",\n \"senderUri\": \"https://seeker-network-vistaar.da.gov.in/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-08-26T00:00:00+05:30\",\n \"endsAt\": \"2026-08-30T23:59:59+05:30\"\n }\n },\n \"quantity\": {\n \"selected\": {\n \"measure\": {\n \"value\": \"1\",\n \"unit\": \"unit\"\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "SAME endpoint as 3. The only difference is context.receiverId, which the consumer copied out of the catalog: it names B, so A signs as itself and sends it to context.bppUri. The answer comes from B's provider." + }, + "response": [] + }, + { + "name": "5. Crawl B now (optional)", + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{bharat_adapter}}/crawl/peers", + "host": [ + "{{bharat_adapter}}/crawl/peers" + ] + }, + "description": "Not needed for the demo -- A crawls every 30s on its own. Use it only to skip the wait. Takes no body: the crawler reads who to crawl from its registry." + }, + "response": [] + } + ] + }, + { + "name": "Maha (network B \u2014 has weather)", + "description": "Network B. Publishes its OWN weather catalogue and shares it with A.\n\nB does NOT crawl: FEDERATION_CRAWL_PEERS=off. Otherwise each network mirrors the other and B ends up re-importing its own catalogue through A's copy of it.\n\nPorts: adapter 9400, discovery 9401.", + "item": [ + { + "name": "1. Publish weather catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_adapter}}/publish/publish", + "host": [ + "{{maha_adapter}}/publish/publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"bppUri\": \"https://mahavistara.oan.local/provider\",\n \"messageId\": \"faa5ae21-67ac-45c7-a2f7-92bea0a2587e\",\n \"transactionId\": \"0292de98-363b-4b6b-b61a-98ed5d7a4af5\",\n \"timestamp\": \"2026-10-07T03:19:08Z\",\n \"version\": \"2.0.0\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\"\n ],\n \"senderId\": \"mahavistara.oan.local\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:weather-observation:mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM_WX\",\n \"name\": \"Mahavistara weather forecasts\",\n \"shortDesc\": \"Daily weather forecasts for locations in Mahavistara.\",\n \"longDesc\": \"Each Resource is a point forecast. Consumers discover by location and invoke the Provider for a three day forecast.\"\n },\n \"isActive\": true,\n \"bppId\": \"mahavistara.oan.local\",\n \"bppUri\": \"https://mahavistara.oan.local/provider\",\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\",\n \"shortDesc\": \"Numerical weather prediction from IMD.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX_POINT_FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Three day forecast for a point location.\"\n },\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n }\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:weather-observation:mausamgram.mahavistara\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"FULL\",\n \"visibleTo\": [\n \"mahavistara.oan.local\",\n \"bharatvistar.oan.local\"\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "THE IMPORTANT FIELD IS visibleTo. It names A as well as B, which is the whole permission for A to mirror this. Drop A from it and A's crawl returns nothing -- no error, just nothing." + }, + "response": [] + }, + { + "name": "2. Discover (what B holds)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_discovery}}/discover", + "host": [ + "{{maha_discovery}}/discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"discover\",\n \"domain\": \"agriculture\",\n \"messageId\": \"5e511275-d9ed-442a-964c-b4f4cc574c1b\",\n \"timestamp\": \"2026-10-07T03:23:01Z\",\n \"transactionId\": \"df14c457-d091-48ad-8bbc-6214001f0578\",\n \"version\": \"2.0.0\",\n \"senderId\": \"mahavistara.oan.local\"\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" || @.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "B holds only its own weather. Run it after A has crawled to show B did NOT import A's mandi back." + }, + "response": [] + }, + { + "name": "3. Select weather (direct, no crossing)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_adapter}}/provider/select", + "host": [ + "{{maha_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-26T06:12:01.330Z\",\n \"senderId\": \"mahavistara.oan.local\",\n \"receiverId\": \"mahavistara.oan.local\",\n \"senderUri\": \"https://seeker-network-vistaar.da.gov.in/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-08-26T00:00:00+05:30\",\n \"endsAt\": \"2026-08-30T23:59:59+05:30\"\n }\n },\n \"quantity\": {\n \"selected\": {\n \"measure\": {\n \"value\": \"1\",\n \"unit\": \"unit\"\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "The same select as Bharat's #4, but sent straight to B. Same answer -- which is the point: A adds no special handling, it just forwards." + }, + "response": [] + } + ] + } + ] +} diff --git a/networkofnetwork/demo/publish-catalog-mandi.json b/networkofnetwork/demo/publish-catalog-mandi.json new file mode 100644 index 00000000..ac76b059 --- /dev/null +++ b/networkofnetwork/demo/publish-catalog-mandi.json @@ -0,0 +1,82 @@ +{ + "context": { + "action": "catalog/publish", + "bppUri": "https://bharatvistar.oan.local/provider", + "messageId": "bd83dc21-38f3-48fe-9be5-b82d4a213666", + "transactionId": "794abc60-4d9b-463f-ba1d-926724ade7f8", + "timestamp": "2026-10-07T03:50:03Z", + "version": "2.0.0", + "schemaContext": [ + "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld" + ], + "senderId": "bharatvistar.oan.local" + }, + "message": { + "catalogs": [ + { + "id": "catalog:mandi-price:agmarknet.bharatvistar", + "descriptor": { + "code": "MANDI_PRICE_MH", + "name": "Bharatvistar mandi prices", + "shortDesc": "On-demand commodity prices for agricultural markets.", + "longDesc": "Each Resource is one commodity at one market. Consumers discover by location or commodity and invoke the Provider for current prices." + }, + "isActive": true, + "bppId": "bharatvistar.oan.local", + "bppUri": "https://bharatvistar.oan.local/provider", + "provider": { + "id": "agmarknet.bharatvistar", + "descriptor": { + "code": "AGMARKNET", + "name": "Agmarknet Vistaar", + "shortDesc": "Live agricultural market price information." + } + }, + "resources": [ + { + "id": "res:agmarknet:nashik:onion", + "descriptor": { + "code": "MANDI_PRICE_NASHIK_ONION", + "name": "Onion prices at Nashik", + "shortDesc": "Current onion prices for Nashik, Maharashtra." + }, + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "subjectCategories": [ + "Market" + ], + "informationMode": "OnDemand", + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + } + } + } + ] + } + ], + "publishDirectives": [ + { + "catalogId": "catalog:mandi-price:agmarknet.bharatvistar", + "catalogType": "REGULAR", + "updateMode": "FULL", + "visibleTo": [ + "bharatvistar.oan.local" + ] + } + ] + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/publish-catalog.json b/networkofnetwork/demo/publish-catalog.json new file mode 100644 index 00000000..d844bde1 --- /dev/null +++ b/networkofnetwork/demo/publish-catalog.json @@ -0,0 +1,82 @@ +{ + "context": { + "action": "catalog/publish", + "bppUri": "https://mahavistara.oan.local/provider", + "messageId": "faa5ae21-67ac-45c7-a2f7-92bea0a2587e", + "transactionId": "0292de98-363b-4b6b-b61a-98ed5d7a4af5", + "timestamp": "2026-10-07T03:19:08Z", + "version": "2.0.0", + "schemaContext": [ + "https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld" + ], + "senderId": "mahavistara.oan.local" + }, + "message": { + "catalogs": [ + { + "id": "catalog:weather-observation:mausamgram.mahavistara", + "descriptor": { + "code": "MAUSAMGRAM_WX", + "name": "Mahavistara weather forecasts", + "shortDesc": "Daily weather forecasts for locations in Mahavistara.", + "longDesc": "Each Resource is a point forecast. Consumers discover by location and invoke the Provider for a three day forecast." + }, + "isActive": true, + "bppId": "mahavistara.oan.local", + "bppUri": "https://mahavistara.oan.local/provider", + "provider": { + "id": "mausamgram.mahavistara", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP", + "shortDesc": "Numerical weather prediction from IMD." + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "descriptor": { + "code": "WX_POINT_FORECAST", + "name": "Point weather forecast", + "shortDesc": "Three day forecast for a point location." + }, + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + } + } + } + ] + } + ], + "publishDirectives": [ + { + "catalogId": "catalog:weather-observation:mausamgram.mahavistara", + "catalogType": "REGULAR", + "updateMode": "FULL", + "visibleTo": [ + "mahavistara.oan.local", + "bharatvistar.oan.local" + ] + } + ] + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-cross-network.json b/networkofnetwork/demo/select-cross-network.json new file mode 100644 index 00000000..b3a1797d --- /dev/null +++ b/networkofnetwork/demo/select-cross-network.json @@ -0,0 +1,74 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z", + "bppUri": "http://oan-mahavistara-oan-local-adapter:9200/provider", + "senderId": "seeker-network-vistaar.da.gov.in", + "receiverId": "mahavistara.oan.local", + "senderUri": "https://seeker-network-vistaar.da.gov.in/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-08-26T00:00:00+05:30", + "endsAt": "2026-08-30T23:59:59+05:30" + } + }, + "quantity": { + "selected": { + "measure": { + "value": "1", + "unit": "unit" + } + } + } + } + ], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": [ + "res:mausamgram:point-forecast" + ], + "provider": { + "id": "mausamgram.mahavistara", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP" + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-local.json b/networkofnetwork/demo/select-local.json new file mode 100644 index 00000000..d661a2dc --- /dev/null +++ b/networkofnetwork/demo/select-local.json @@ -0,0 +1,73 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z", + "senderId": "seeker-network-vistaar.da.gov.in", + "receiverId": "bharatvistar.oan.local", + "senderUri": "https://seeker-network-vistaar.da.gov.in/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-08-26T00:00:00+05:30", + "endsAt": "2026-08-30T23:59:59+05:30" + } + }, + "quantity": { + "selected": { + "measure": { + "value": "1", + "unit": "unit" + } + } + } + } + ], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": [ + "res:mausamgram:point-forecast" + ], + "provider": { + "id": "mausamgram.bharatvistar", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP" + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-mandi-local.json b/networkofnetwork/demo/select-mandi-local.json new file mode 100644 index 00000000..8452866c --- /dev/null +++ b/networkofnetwork/demo/select-mandi-local.json @@ -0,0 +1,82 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "bppUri": "https://bharatvistar.oan.local/provider", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d23", + "timestamp": "2026-10-07T06:12:01.330Z", + "senderId": "seeker.bharatvistar.oan.local", + "receiverId": "bharatvistar.oan.local", + "senderUri": "https://seeker.bharatvistar.oan.local/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "offer": { + "id": "offer:agmarknet:open-data", + "provider": { + "id": "agmarknet.bharatvistar", + "descriptor": { + "code": "AGMARKNET", + "name": "Agmarknet Vistaar" + } + }, + "resourceIds": [ + "res:agmarknet:nashik:onion" + ] + }, + "resources": [ + { + "id": "res:agmarknet:nashik:onion", + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "subjectCategories": [ + "Market" + ], + "informationMode": "OnDemand", + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + }, + "supportedCommodities": [ + { + "code": "149", + "name": "Onion" + } + ], + "market": { + "district": "96", + "state": "MH", + "marketCode": "2056", + "name": "Nashik" + } + } + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/docker-compose.yml b/networkofnetwork/docker-compose.yml new file mode 100644 index 00000000..e70fc1aa --- /dev/null +++ b/networkofnetwork/docker-compose.yml @@ -0,0 +1,326 @@ +# One OAN network, whole, on one machine. +# +# Separate from docker-compose.yml and deliberately so. That file is the +# three-adapter quick-start: one stack, fixed names, fixed ports, and an +# operator's own deployment. This one exists to be run MORE THAN ONCE on the +# same machine, as two or three networks that can see each other -- which is the +# only way to demonstrate federation without three laptops. +# +# What differs from the quick-start: +# +# ONE adapter, not three. A network is one participant; running the consumer, +# network and provider layers as three containers is a deployment choice, and +# the consolidated shape is the simpler one to reason about when there are two +# of everything already. +# +# A FRONT DOOR. A network id is a DNS domain, and a peer reaches a network at +# that domain over https. nginx terminates TLS, serves the two published +# documents, and proxies the rest inward. +# +# EVERYTHING PREFIXED. Container names and docker network names are global to +# the daemon rather than scoped to a compose project, so two networks running +# the same file would collide on both without this. +# +# Every value comes from the network's own .env, written by bin/network-env.py. +# Nothing here names a network. + +name: ${COMPOSE_PROJECT_NAME} + +networks: + # This network's own internals. Nothing outside the project reaches them. + internal: + name: ${COMPOSE_PROJECT_NAME}-internal + # The bridge every network on this machine shares. The ONLY crossing point, + # and only the front door and the adapter are on it -- which is what makes the + # boundary real rather than decorative. + federation: + name: oan-federation + external: true + +volumes: + registry-postgres-data: + discovery-postgres-data: + +x-logging: &logging + driver: json-file + options: { max-size: "10m", max-file: "3" } + +services: + + # ---------------------------------------------------------------- registry + + registry-postgres: + image: postgres:14 + container_name: ${COMPOSE_PROJECT_NAME}-registry-postgres + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + POSTGRES_DB: registry + POSTGRES_USER: ${REGISTRY_DB_USER} + POSTGRES_PASSWORD: ${REGISTRY_DB_PASSWORD} + volumes: + - registry-postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${REGISTRY_DB_USER} -d registry"] + interval: 5s + timeout: 5s + retries: 20 + + registry-keycloak: + image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest + container_name: ${COMPOSE_PROJECT_NAME}-registry-keycloak + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + DB_VENDOR: postgres + DB_ADDR: registry-postgres + DB_PORT: "5432" + DB_DATABASE: registry + DB_USER: ${REGISTRY_DB_USER} + DB_PASSWORD: ${REGISTRY_DB_PASSWORD} + KEYCLOAK_USER: ${KEYCLOAK_ADMIN_USER} + KEYCLOAK_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + KEYCLOAK_IMPORT: /opt/jboss/keycloak/imports/realm-export.json + PROXY_ADDRESS_FORWARDING: "true" + volumes: + # The realm this registry runs on. KEYCLOAK_IMPORT names it, and without + # the file keycloak restarts in a loop saying only that it added the admin + # user again. + - ./config/registry/imports:/opt/jboss/keycloak/imports + # Waited on rather than assumed: the registry service resolves its OAuth + # issuer at start-up, and a keycloak that is not listening yet fails it + # outright rather than retrying. + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9990/ || exit 1"] + interval: 10s + timeout: 10s + retries: 30 + depends_on: + registry-postgres: + condition: service_healthy + ports: + - "127.0.0.1:${KEYCLOAK_PORT}:8080" + + registry-service: + image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_IMAGE_TAG} + container_name: ${COMPOSE_PROJECT_NAME}-registry-service + restart: unless-stopped + logging: *logging + # internal ONLY, and that is a decision rather than an omission: the + # registry is this network's own record and nothing outside it has business + # reading it directly. + networks: [internal] + environment: + connectionInfo_uri: jdbc:postgresql://registry-postgres:5432/registry + connectionInfo_username: ${REGISTRY_DB_USER} + connectionInfo_password: ${REGISTRY_DB_PASSWORD} + search_providerName: dev.sunbirdrc.registry.service.NativeSearchService + authentication_enabled: "true" + sunbird_sso_realm: ${KEYCLOAK_REALM} + sunbird_sso_url: http://registry-keycloak:8080/auth + OAUTH2_RESOURCES_0_URI: http://registry-keycloak:8080/auth/realms/${KEYCLOAK_REALM} + OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH: realm_access.roles + identity_provider: dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + sunbird_sso_admin_client_id: ${KEYCLOAK_ADMIN_CLIENT_ID} + sunbird_sso_client_id: ${KEYCLOAK_CLIENT_ID} + sunbird_sso_admin_client_secret: ${KEYCLOAK_ADMIN_CLIENT_SECRET} + sunbird_keycloak_user_set_password: "true" + sunbird_keycloak_user_password: ${KEYCLOAK_DEFAULT_USER_PASSWORD} + encryption_enabled: "false" + event_enabled: "false" + idgen_enabled: "false" + claims_enabled: "false" + did_enabled: "false" + signature_enabled: "false" + certificate_enabled: "false" + filestorage_enabled: "false" + notification_enabled: "false" + notification_async_enabled: "false" + async_enabled: "false" + webhook_enabled: "false" + registry_base_apis_enable: "false" + manager_type: DefinitionsManager + expand_reference: "false" + swagger_title: OAN Registry + logging.level.root: INFO + volumes: + - ./config/registry/schemas:/home/sunbirdrc/config/public/_schemas:ro + depends_on: + registry-keycloak: + condition: service_healthy + ports: + - "127.0.0.1:${REGISTRY_PORT}:8081" + + # --------------------------------------------------------------- discovery + + discovery-postgres: + # pgvector rather than plain postgres: the discovery service's HNSW index + # options arrived in 0.8. + image: pgvector/pgvector:0.8.0-pg16 + container_name: ${COMPOSE_PROJECT_NAME}-discovery-postgres + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + # Literals, matching quick-start: these never varied per deployment, and a + # variable that is always the same value is a question nobody needs asked. + POSTGRES_DB: discovery + POSTGRES_USER: discovery + POSTGRES_PASSWORD: discovery + volumes: + - discovery-postgres-data:/var/lib/postgresql/data + # The crawler keeps its bookkeeping in a database of its own on this same + # server. Created here because a destroyed stack takes the volume with it, + # and the adapter restart-loops on a missing database. + - ./config/init-crawler-db.sh:/docker-entrypoint-initdb.d/10-crawler.sh:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U discovery -d discovery"] + interval: 5s + timeout: 5s + retries: 20 + + discovery-service: + image: ${DISCOVERY_IMAGE} + container_name: ${COMPOSE_PROJECT_NAME}-discovery-service + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + APP_DOMAIN: ${DISCOVERY_DOMAIN:-Agriculture} + APP_NETWORK_ID: ${BECKN_NETWORK_ID} + APP_SUBSCRIBER_ID: ${DISCOVERY_SUBSCRIBER_ID} + DATABASE_AUTO_MIGRATE: "true" + DATABASE_URL: postgres://discovery:discovery@discovery-postgres:5432/discovery?sslmode=disable + GEO_MAX_GEOMETRIES_PER_CATALOG: ${DISCOVERY_MAX_GEOMETRIES_PER_CATALOG:-256} + OTEL_EXPORTER: ${DISCOVERY_OTEL_EXPORTER:-none} + RATE_LIMIT_BURST: ${DISCOVERY_RATE_LIMIT_BURST:-100000} + RATE_LIMIT_RPS: ${DISCOVERY_RATE_LIMIT_RPS:-100000} + SEARCH_ENABLE_TEXT_SEARCH: "false" + SERVER_PORT: "8080" + VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} + depends_on: + discovery-postgres: + condition: service_healthy + ports: + - "127.0.0.1:${DISCOVERY_PORT}:8080" + + # ------------------------------------------------------------------- mocks + + mock-imd: + image: ${MOCKIMD_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-mock-imd + restart: unless-stopped + logging: *logging + networks: [internal] + ports: + - "127.0.0.1:${MOCKIMD_PORT}:9100" + + mock-agmarknet: + image: ${MOCKAGMARKNET_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-mock-agmarknet + restart: unless-stopped + logging: *logging + networks: [internal] + ports: + - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" + + # ----------------------------------------------------------------- adapter + + # ONE adapter, serving every layer this network has. Its modules are mounted + # on separate paths of one port -- see config/network/adapter.yaml.tmpl. + # + # On the federation bridge as well as the internal one, because IT is what + # reaches another network: the crawl asking for catalogs, and a cross-network + # select going out. The front door is how others reach US; this is the other + # direction over the same bridge. + adapter: + image: ${ADAPTER_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-adapter + restart: unless-stopped + logging: *logging + networks: [internal, federation] + environment: + BECKN_NETWORK_ID: ${BECKN_NETWORK_ID} + # Go reads SSL_CERT_FILE for the system roots on Linux, so an outbound + # call verifies another network's certificate against the local CA with no + # code that knows anything about local certificates. + # + # Real verification against a different root, NOT verification turned off: + # a certificate this CA did not sign, or one whose name does not match, is + # still refused. + SSL_CERT_FILE: /etc/oan/ca.crt + CONFIG_FILE: /app/config/adapter.yaml + MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} + AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} + AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} + VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} + VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} + volumes: + - ${NETWORK_DIR}/config/adapter.yaml:/app/config/adapter.yaml:ro + # Routing is NOT rendered per network -- nothing in it differs between + # networks, because every target is either this same adapter on another + # path or a service name inside this project. + - ./config/routing-consumer.yaml:/app/config/routing-consumer.yaml:ro + - ./config/routing-network.yaml:/app/config/routing-network.yaml:ro + - ./config/routing-publish.yaml:/app/config/routing-publish.yaml:ro + - ${NETWORK_DIR}/tls/ca.crt:/etc/oan/ca.crt:ro + depends_on: + registry-service: + condition: service_started + discovery-service: + condition: service_started + ports: + - "127.0.0.1:${ADAPTER_PORT}:9200" + + # --------------------------------------------------------------- front door + + # What another network talks to. It is the whole of this network's public + # surface: the two published documents, and a proxy inward for the Beckn + # actions. Nothing else here is reachable from the bridge. + wellknown: + image: nginx:1.27-alpine + container_name: ${COMPOSE_PROJECT_NAME}-wellknown + restart: unless-stopped + logging: *logging + environment: + # Read by envsubst into default.conf.template at start-up. + NETWORK_ID: ${BECKN_NETWORK_ID} + ADAPTER_HOST: adapter + ADAPTER_PORT: "9200" + MOCKIMD_HOST: mock-imd + MOCKIMD_INTERNAL_PORT: "9100" + MOCKAGMARKNET_HOST: mock-agmarknet + MOCKAGMARKNET_INTERNAL_PORT: "9101" + volumes: + - ${NETWORK_DIR}/.well-known:/usr/share/nginx/html/.well-known:ro + - ${NETWORK_DIR}/tls:/etc/nginx/tls:ro + - ./config/default.conf.template:/etc/nginx/templates/default.conf.template:ro + - ./config/proxy-headers.inc:/etc/nginx/conf.d/proxy-headers.inc:ro + networks: + internal: + # The upstream provider names, on the INTERNAL bridge because it is our + # own adapter that calls them. Without these the adapter's lookup of + # mausamgram. fails at DNS and the seeded baseUrl is a URL + # nothing can reach. + aliases: + - mausamgram.${BECKN_NETWORK_ID} + - agmarknet.${BECKN_NETWORK_ID} + federation: + # This is what makes the network id resolve. Another network's adapter + # looking up bharatvistar.oan.local reaches this container and nothing + # else behind it. + aliases: + - ${BECKN_NETWORK_ID} + depends_on: + adapter: + condition: service_started + ports: + # Published so the operator's own machine can fetch the documents. The + # network id only resolves inside the bridge, so a CLI reaches this port + # and carries the id as its Host header. + - "127.0.0.1:${FRONT_DOOR_PORT}:443" From 78c405545d0db3b8453ff9ad69f57e52442f8735 Mon Sep 17 00:00:00 2001 From: Prajwal TD <157502711+tdprajwal@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:23:49 +0530 Subject: [PATCH 2/5] feat: split governance and operational keys, strict JCS, validate the descriptor [OpenAgriNet/network-adapter#74] - Governance key (keys/governance.key, mode 600) signs only the Network Descriptor. Operational keys come from the registry entry and are delegated by kid in the signed operationalKeys field; the JWKS publishes both. admit stores only delegated keys, so the governance key can never verify traffic. - canonical() is RFC 8785: UTF-16 key order; floats, unsafe integers and lone surrogates are refused. Signature algorithm Ed25519 (RFC 9864). - kid is the RFC 7638 thumbprint, so rotation needs no configuration. - Descriptor and JWKS validated against the contract before signing and on verify. publish refuses a non-increasing revision, a governance key that is also a traffic key, or a readable key file. It writes the JWKS before the descriptor, each atomically. - test_oanfed.py: 21 checks. Co-Authored-By: Claude Opus 5.5 (1M context) --- networkofnetwork/bin/network.sh | 6 +- networkofnetwork/bin/oanfed.py | 452 +++++++++++++++++++++------- networkofnetwork/bin/setup.py | 19 -- networkofnetwork/bin/test_oanfed.py | 234 ++++++++++++++ networkofnetwork/demo/README.md | 30 +- 5 files changed, 608 insertions(+), 133 deletions(-) create mode 100644 networkofnetwork/bin/test_oanfed.py diff --git a/networkofnetwork/bin/network.sh b/networkofnetwork/bin/network.sh index f5ad2d91..249ee966 100755 --- a/networkofnetwork/bin/network.sh +++ b/networkofnetwork/bin/network.sh @@ -122,10 +122,14 @@ cmd_host_descriptor() { [ -n "$ttl" ] && ttl_arg=(--projection-ttl "$ttl") # The BASE registry url: oanfed appends /api/v1 to it itself. + # + # The governance key signs the descriptor and nothing else, so it is its own + # file -- generated on first publish -- and never the adapters' traffic key, + # which oanfed reads from the registry and delegates in the descriptor. python3 "$HERE/oanfed.py" publish \ --network-id "$network" \ --registry-url "http://127.0.0.1:$(port_of "$network" REGISTRY_PORT)" \ - --key "$dir/keys/network.key" \ + --governance-key "$dir/keys/governance.key" \ --revision "$revision" \ "${ttl_arg[@]}" \ --out "$dir" diff --git a/networkofnetwork/bin/oanfed.py b/networkofnetwork/bin/oanfed.py index 0d563c49..53f88354 100644 --- a/networkofnetwork/bin/oanfed.py +++ b/networkofnetwork/bin/oanfed.py @@ -27,7 +27,7 @@ Run a command with -h for its flags. """ -import argparse, base64, http.client, json, pathlib, ssl, sys, urllib.error, urllib.parse, urllib.request +import argparse, base64, hashlib, http.client, json, os, pathlib, re, ssl, sys, urllib.error, urllib.parse, urllib.request from datetime import datetime, timezone from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey @@ -45,14 +45,21 @@ # run out of memory. MAX_DOCUMENT_BYTES = 1 << 20 -# The only algorithm this implementation signs or verifies with. The contract -# leaves the choice open; narrowing it here means an unexpected value is refused -# rather than quietly treated as this one. +# The only signature algorithm this implementation signs or verifies with, by +# its fully-specified JOSE name (RFC 9864, which deprecates the polymorphic +# "EdDSA"). The contract leaves the choice open; narrowing it here means an +# unexpected value is refused rather than quietly treated as this one. ALGORITHM = "Ed25519" -# The kid on this network's one published key. Stable, because a descriptor's -# signature points at it and a peer resolves the key by it. -KEY_ID = "network-1" +# The Network Descriptor schema version this implementation writes and reads. A +# version it does not know is refused rather than half-understood. +DESCRIPTOR_VERSION = "1.0" +DESCRIPTOR_STATUSES = ("Active", "Suspended", "Retired") + +# How far ahead of our clock a peer's issuedAt may be before it is refused. A +# document from the future is a clock fault or a pre-signed replay waiting to +# happen; a few minutes absorbs ordinary skew. +MAX_CLOCK_SKEW_SECONDS = 300 ROLE_NETWORK = "network" @@ -73,35 +80,65 @@ def has_role(record, role): STATUS_ACTIVE, STATUS_INACTIVE = "active", "inactive" +class FederationError(Exception): + """Anything a caller is expected to act on rather than crash over.""" + + # --- canonical form ------------------------------------------------------- +# JCS serialises numbers the way ECMAScript does, which is exact for integers +# only up to 2**53. Beyond it two implementations can disagree on the digits. +MAX_SAFE_INTEGER = 2 ** 53 - 1 + + def canonical(descriptor): """The exact bytes that get signed: RFC 8785 (JCS). The signature is excluded, because it cannot cover itself. Everything else is included: narrowing the covered set would let an unsigned field be changed in flight, and the fields most worth tampering with -- status, - revision, the discovery URL -- are exactly the ones a smaller set would be - tempted to omit. + revision, the discovery URL, which keys may sign traffic -- are exactly the + ones a smaller set would be tempted to omit. JCS, and not "whatever this language's JSON encoder happens to emit", because the point of a canonical form is that a DIFFERENT implementation - rebuilds the same bytes. An encoder-specific form works perfectly until - someone else writes the second implementation, and then fails every - signature check on documents nobody tampered with. - - Our descriptors contain only strings, integers, arrays and objects, so the - parts of JCS that are genuinely hard -- number formatting -- do not arise. - What matters here is: keys sorted, no insignificant whitespace, UTF-8, and - no escaping beyond what JSON requires. + rebuilds the same bytes. """ without_signature = {k: v for k, v in descriptor.items() if k != "signature"} - return json.dumps( - without_signature, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ).encode("utf-8") + return _jcs(without_signature).encode("utf-8") + + +def _jcs(value): + """One JCS value. Floats are refused rather than formatted. + + A descriptor has no use for a fraction, and number formatting is the one + part of JCS where implementations genuinely diverge -- so the safe rule is + that a non-integer never reaches a signature. + """ + if value is None or isinstance(value, bool): + return json.dumps(value) + if isinstance(value, int): + if abs(value) > MAX_SAFE_INTEGER: + raise FederationError(f"integer {value} is outside the range JCS signs exactly") + return str(value) + if isinstance(value, float): + raise FederationError(f"non-integer number {value!r} cannot be signed; use an integer") + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError as err: + raise FederationError("string is not valid Unicode (lone surrogate)") from err + # Python's escaping is JCS's: \b \f \n \r \t \" \\ as short forms, other + # controls as lowercase \u00xx, everything else (including '/') raw. + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list): + return "[" + ",".join(_jcs(v) for v in value) + "]" + if isinstance(value, dict): + # JCS sorts by UTF-16 code units, not code points. The two differ only + # above U+FFFF, which is exactly where a naive sort_keys goes wrong. + keys = sorted(value, key=lambda k: k.encode("utf-16-be")) + return "{" + ",".join(f"{_jcs(k)}:{_jcs(value[k])}" for k in keys) + "}" + raise FederationError(f"{type(value).__name__} cannot be signed") # --- keys ----------------------------------------------------------------- @@ -112,6 +149,8 @@ def b64url(raw): def unb64url(encoded): + if not isinstance(encoded, str) or not re.fullmatch(r"[A-Za-z0-9_-]*", encoded): + raise FederationError("value is not base64url") return base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) @@ -128,21 +167,39 @@ def registry_key(jwk_x): return base64.b64encode(unb64url(jwk_x)).decode() +def jwk(public): + """The JWK for a raw Ed25519 public key. + + The kid is the key's RFC 7638 thumbprint: derived from the key itself, so it + never needs configuring, can never point at the wrong key, and changes on + its own when a key is rotated -- which is what lets old and new overlap in + one JWKS without colliding. + """ + x = b64url(public) + members = {"crv": "Ed25519", "kty": "OKP", "x": x} + thumbprint = hashlib.sha256(_jcs(members).encode()).digest() + return {"kty": "OKP", "crv": "Ed25519", "alg": ALGORITHM, "use": "sig", + "kid": b64url(thumbprint), "x": x} + + def load_or_create_key(path): - """Read a private key, generating one if the file is absent. + """Read the governance private key, generating one if the file is absent. Stored as the raw 32-byte seed, base64url-unpadded -- the same encoding the public half uses in the JWKS, so an operator reading either file sees one format rather than two. + + A key file anyone but its owner can read is refused, not used. This key + decides what the network says about itself, including which keys may sign + as it; a copy of it is the network. """ path = pathlib.Path(path) if path.exists(): + if path.stat().st_mode & 0o077: + raise FederationError( + f"{path} is readable by group or others; chmod 600 it before signing with it") raw = unb64url(path.read_text().strip()) - # 32 is the seed, which is what this tool writes. 64 is Go's - # ed25519.PrivateKey -- the seed followed by the public half -- which is - # what the Go implementation this replaces wrote, and what every key - # already on disk is. Both are read so an existing network does not have - # to regenerate its identity to move tools. + # 64 is Go's ed25519.PrivateKey -- the seed followed by the public half. if len(raw) == 64: raw = raw[:32] if len(raw) != 32: @@ -158,8 +215,10 @@ def load_or_create_key(path): encryption_algorithm=ser.NoEncryption(), ) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(b64url(raw) + "\n") - path.chmod(0o600) + # Created 0600 rather than chmod-ed after: no window where it is readable. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as f: + f.write(b64url(raw) + "\n") return private @@ -169,11 +228,119 @@ def public_of(private): ) -# --- sign and verify ------------------------------------------------------ +# --- the published documents: shape --------------------------------------- + +def _is_https_url(value): + if not isinstance(value, str): + return False + parsed = urllib.parse.urlsplit(value) + return parsed.scheme == "https" and bool(parsed.hostname) + + +def _parse_time(value): + if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", value): + raise FederationError(f"issuedAt {value!r} is not a UTC timestamp like 2026-09-17T10:00:00Z") + try: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + except ValueError as err: + raise FederationError(f"issuedAt {value!r} is not a real date") from err + + +def _nonempty_str(value): + return isinstance(value, str) and value.strip() != "" + + +def validate_keyset(keyset): + """A JWKS, checked; returns {kid: raw public key} for its Ed25519 keys. + + Keys of other types are allowed and skipped -- a network may publish more + than this implementation verifies with. A duplicate kid is refused outright: + with two keys under one name, "the key the signature names" means nothing. + """ + if not isinstance(keyset, dict) or not isinstance(keyset.get("keys"), list): + raise FederationError("JWKS is not an object with a keys array") + found, seen = {}, set() + for entry in keyset["keys"]: + if not isinstance(entry, dict) or not _nonempty_str(entry.get("kid")): + raise FederationError("JWKS holds a key without a kid") + kid = entry["kid"] + if kid in seen: + raise FederationError(f"JWKS holds two keys with kid {kid!r}") + seen.add(kid) + if entry.get("kty") != "OKP" or entry.get("crv") != "Ed25519": + continue + if entry.get("alg", ALGORITHM) != ALGORITHM: + raise FederationError(f"key {kid!r} declares alg {entry.get('alg')!r}") + if entry.get("use", "sig") != "sig": + raise FederationError(f"key {kid!r} is not a signing key") + raw = unb64url(entry.get("x", "")) + if len(raw) != 32: + raise FederationError(f"key {kid!r} is {len(raw)} bytes, want 32") + found[kid] = raw + return found + + +def validate_descriptor(descriptor, now=None): + """The Network Descriptor contract, field by field. + + Run before signing, so this network never publishes something a peer must + refuse, and after fetching, so a peer's document is checked against the + same rules rather than a looser reading of them. + """ + if not isinstance(descriptor, dict): + raise FederationError("descriptor is not a JSON object") + + def need(ok, message): + if not ok: + raise FederationError(message) + + need(descriptor.get("version") == DESCRIPTOR_VERSION, + f"descriptor version {descriptor.get('version')!r} is not {DESCRIPTOR_VERSION!r}") + network_id = descriptor.get("networkId") + need(isinstance(network_id, str) + and re.fullmatch(r"(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}", + network_id) is not None, + f"networkId {network_id!r} is not a lowercase DNS domain") + need(_nonempty_str(descriptor.get("name")), "name is missing") + operator = descriptor.get("operator") + need(isinstance(operator, dict) and _nonempty_str(operator.get("name")), + "operator.name is missing") + need("operatorId" not in operator or _nonempty_str(operator["operatorId"]), + "operator.operatorId is empty") + revision = descriptor.get("revision") + need(isinstance(revision, int) and not isinstance(revision, bool) + and 1 <= revision <= MAX_SAFE_INTEGER, + f"revision {revision!r} is not a positive integer") + need(descriptor.get("status") in DESCRIPTOR_STATUSES, + f"status {descriptor.get('status')!r} is not one of {', '.join(DESCRIPTOR_STATUSES)}") + issued = _parse_time(descriptor.get("issuedAt")) + now = now or datetime.now(timezone.utc) + need((issued - now).total_seconds() <= MAX_CLOCK_SKEW_SECONDS, + f"issuedAt {descriptor['issuedAt']} is in the future") + profile = descriptor.get("operatingProfile") + need(isinstance(profile, list) and profile and all(_nonempty_str(p) for p in profile), + "operatingProfile is not a non-empty list of versions") + packs = descriptor.get("schemaPacks") + need(isinstance(packs, list) and all(_is_https_url(p) for p in packs) + and len(set(packs)) == len(packs), + "schemaPacks is not a list of distinct https URLs") + services = descriptor.get("services") + need(isinstance(services, dict) and _is_https_url(services.get("discovery")), + "services.discovery is not an https URL") + contacts = descriptor.get("contacts") + need(contacts is None or (isinstance(contacts, dict) + and all(_nonempty_str(v) for v in contacts.values())), + "contacts is not an object of strings") + ttl = descriptor.get("projectionTtl") + need(ttl is None or (isinstance(ttl, int) and not isinstance(ttl, bool) and ttl >= 0), + f"projectionTtl {ttl!r} is not a non-negative integer") + delegated = descriptor.get("operationalKeys") + need(isinstance(delegated, list) and delegated + and all(_nonempty_str(k) for k in delegated) and len(set(delegated)) == len(delegated), + "operationalKeys is not a non-empty list of distinct kids") -class FederationError(Exception): - """Anything a caller is expected to act on rather than crash over.""" +# --- sign and verify ------------------------------------------------------ class StaleRevision(FederationError): """The descriptor does not advance the revision. @@ -189,55 +356,60 @@ class StaleRevision(FederationError): """ -def sign(descriptor, key_id, private): - """Return a copy of the descriptor carrying its signature. +def sign(descriptor, private): + """Return a copy of the descriptor carrying the governance signature. - The governance key signs this and nothing else. The contract forbids it from - signing ordinary traffic, which is why admission deliberately keeps it out - of the registry record a peer's calls are verified against. + The governance key signs this and nothing else. It is named by its own + thumbprint, so the signature cannot point at a different key than the one + that made it. """ - if not descriptor.get("networkId"): - raise FederationError("descriptor carries no networkId") - signed = dict(descriptor) + signed = {k: v for k, v in descriptor.items() if k != "signature"} signed["signature"] = { - "keyId": key_id, + "keyId": jwk(public_of(private))["kid"], "algorithm": ALGORITHM, - "value": b64url(private.sign(canonical(descriptor))), + "value": b64url(private.sign(canonical(signed))), } return signed -def verify(descriptor, keyset): - """Check a descriptor against the keys its network published. +def verify(descriptor, keyset, now=None): + """Check a descriptor, its shape, and its signature against the JWKS. + + It answers ONE question -- did the holder of a published governance key + write this document -- and deliberately not whether the network should be + trusted. The keys came from the same domain as the document, so a network + can always verify its own lies. Admission is what decides trust. - It answers ONE question -- did the holder of a published key write this - document -- and deliberately not whether the network should be trusted. The - keys came from the same domain as the document, so a network can always - verify its own lies. Admission is what decides trust. + The key that signed the descriptor must not also be one it delegates for + traffic: the contract says the governance key never signs normal traffic, + and this is the one place that can be checked. """ + validate_descriptor(descriptor, now) + keys = validate_keyset(keyset) + signature = descriptor.get("signature") - if not signature: + if not isinstance(signature, dict): raise FederationError("descriptor carries no signature") if signature.get("algorithm") != ALGORITHM: raise FederationError(f"unsupported signature algorithm: {signature.get('algorithm')!r}") - key_id = signature.get("keyId") - jwk = next((k for k in keyset.get("keys", []) if k.get("kid") == key_id), None) - if jwk is None: - raise FederationError(f"no published key matches the signature's keyId {key_id!r}") - - raw = unb64url(jwk["x"]) - if len(raw) != 32: - raise FederationError(f"key {key_id!r} is {len(raw)} bytes, want 32") + if key_id not in keys: + raise FederationError(f"no published Ed25519 key matches the signature's keyId {key_id!r}") + delegated = descriptor["operationalKeys"] + if key_id in delegated: + raise FederationError( + f"governance key {key_id!r} is also delegated for traffic; the two must differ") + missing = [k for k in delegated if k not in keys] + if missing: + raise FederationError(f"operationalKeys not in the JWKS: {', '.join(missing)}") try: - Ed25519PublicKey.from_public_bytes(raw).verify( - unb64url(signature["value"]), canonical(descriptor) + Ed25519PublicKey.from_public_bytes(keys[key_id]).verify( + unb64url(signature.get("value", "")), canonical(descriptor) ) except Exception as err: raise FederationError("signature does not verify") from err - # --- fetching a peer's documents ------------------------------------------ class ResolvingHTTPSConnection(http.client.HTTPSConnection): @@ -362,14 +534,16 @@ def projection_ttl(descriptor): return declared -def published_keys(keyset): - """Every key the peer published. +def operational_keys(descriptor, keyset): + """The keys a peer delegated for traffic, and only those. - Nothing is stripped. An earlier design dropped the key that signed the - descriptor, so a peer could not sign ordinary traffic with it -- that split - is gone, and a network now publishes ONE key which does both jobs. + Its governance key is published in the same JWKS but deliberately left out: + whatever is recorded here is what the peer's requests are verified against, + and the governance key must never pass that check. Which keys are delegated + comes from the SIGNED descriptor, so a JWKS edit alone cannot add one. """ - return list(keyset.get("keys", [])) + delegated = set(descriptor.get("operationalKeys") or []) + return [k for k in keyset.get("keys", []) if k.get("kid") in delegated] # What an entry is in relation to us. @@ -442,7 +616,7 @@ def admit_record(current, fresh, keyset, federation_role=DEFAULT_FEDERATION_ROLE "validFrom": admitted_at, "status": "active", } - for k in published_keys(keyset) + for k in operational_keys(fresh, keyset) ], "revision": fresh.get("revision", 0), "projectionTtl": ttl, @@ -614,6 +788,51 @@ def set_status(self, participant_id, status): # --- commands ------------------------------------------------------------- +def registered_signing_keys(entry, now=None): + """The entry's Ed25519 keys that are usable now or about to be. + + These are the operational keys: the adapters sign traffic with them and the + registry already holds them, plural, for rotation overlap. A key not yet + valid IS included -- a peer has to know a key before it starts signing with + it. A revoked or expired one is not: publishing it would delegate a key this + network has already withdrawn. + """ + now = now or datetime.now(timezone.utc) + usable = [] + for k in entry.get("keys") or []: + if str(k.get("alg", "")).lower() != "ed25519" or k.get("status", "active") != "active": + continue + until = k.get("validUntil") + if until and datetime.fromisoformat(until.replace("Z", "+00:00")) <= now: + continue + raw = base64.b64decode(str(k.get("key", "")).removeprefix("base64:")) + if len(raw) != 32: + raise FederationError(f"registry key {k.get('osid', '?')} is not a 32-byte Ed25519 key") + usable.append(raw) + return usable + + +def last_published_revision(path): + """The revision already on disk, or 0. Unreadable is an error, not 0. + + Treating a corrupt file as "nothing published" would let a revision go + backwards, and every peer holding the higher one would refuse ours. + """ + if not path.exists(): + return 0 + try: + return int(json.loads(path.read_text())["revision"]) + except Exception as err: + raise FederationError(f"cannot read the revision in {path}: {err}") from err + + +def write_atomically(path, document): + """Replace a published file in one step, so a reader never sees half of it.""" + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n") + os.replace(tmp, path) + + def cmd_publish(args): """Generate and sign this network's two published documents FROM THE REGISTRY. @@ -622,12 +841,25 @@ def cmd_publish(args): internally, by construction -- rather than two descriptions of one network that happen to agree on the day someone typed them. - Reading it needs no credentials: the registry's search endpoint is public. - Writing an entry is the privileged act, and that is setup.py's job, not this - one's. + Two kinds of key, as the contract requires: + + governance signs the descriptor and nothing else. Its private half is + --governance-key; it is NOT in the registry, so no peer can + verify a request signed with it. + operational sign traffic. Read from the registry entry; the descriptor + delegates them by kid in operationalKeys, under the + governance signature. + + Reading the registry needs no credentials: its search endpoint is public. """ out = pathlib.Path(args.out) / ".well-known" - out.mkdir(parents=True, exist_ok=True) + descriptor_path, jwks_path = out / "openagrinet", out / "jwks.json" + + held = last_published_revision(descriptor_path) + if args.revision <= held: + raise FederationError( + f"revision {args.revision} does not increase on the published {held}; " + f"every peer holding {held} would refuse it") registry = ReadOnlyRegistry(args.registry_url, args.registry_timeout) entry = registry.participant(args.network_id) @@ -636,27 +868,31 @@ def cmd_publish(args): f"{args.network_id} is not in this registry -- seed it first, " f"then publish from it") - network_key = load_or_create_key(args.key) - published = registry_key(b64url(public_of(network_key))) - held = [k.get("key", "").removeprefix("base64:") for k in entry.get("keys", [])] - if published not in held: - # Publishing a key the registry does not hold would hand peers something - # that verifies our descriptor but nothing else -- our signed traffic - # would be checked against the registry's key and fail. Worth refusing - # loudly rather than discovering on the first cross-network call. + operational = registered_signing_keys(entry) + if not operational: + raise FederationError( + f"{args.network_id} has no active Ed25519 key in the registry to delegate") + + governance = load_or_create_key(args.governance_key) + if public_of(governance) in operational: + # The one configuration that silently breaks the contract: the key that + # vouches for the network would also be accepted on its traffic. raise FederationError( - f"{args.key} is not the key {args.network_id} is registered with; " - f"publishing it would sign descriptors with one key and traffic with another") + f"{args.governance_key} is also a registered traffic key; the governance " + f"key must be separate -- use a new file") + + governance_jwk = jwk(public_of(governance)) + operational_jwks = [jwk(raw) for raw in operational] descriptor = { - "version": "0.1", + "version": DESCRIPTOR_VERSION, "networkId": entry["participantId"], "name": entry.get("name") or entry["participantId"], "operator": {"name": args.operator or entry.get("name") or entry["participantId"]}, "revision": args.revision, "status": args.status, "issuedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "operatingProfile": [p for p in args.operating_profile.split(",") if p], + "operatingProfile": [p.strip() for p in args.operating_profile.split(",") if p.strip()], # From the schema registry, which is where this deployment records what # it can speak. Typing them again here would let the two disagree. "schemaPacks": registry.schema_packs(), @@ -664,30 +900,30 @@ def cmd_publish(args): # field admission reads back out of a peer's descriptor, so what we # publish and what a peer stores about us are one string. "services": {"discovery": entry.get("baseUrl", "")}, + # The delegation: which JWKS keys may sign traffic as this network. + "operationalKeys": [k["kid"] for k in operational_jwks], } if args.projection_ttl is not None: descriptor["projectionTtl"] = args.projection_ttl - # ONE key. It signs this descriptor and it signs cross-network traffic. - # - # An earlier design split these -- a governance key for the descriptor, an - # operational key for everything else -- so that holding the traffic key - # could not let you rewrite who the network is. That is a real property, and - # it is given up deliberately: two keys meant two identities to configure, - # two to rotate, and a rule ("never sign traffic with the governance key") - # that nothing downstream could enforce, because a signature validator sees - # only that a key is published and active. - signed = sign(descriptor, KEY_ID, network_key) - keyset = {"keys": [ - {"kty": "OKP", "crv": "Ed25519", "kid": KEY_ID, "x": b64url(public_of(network_key))}, - ]} - - (out / "openagrinet").write_text(json.dumps(signed, indent=2) + "\n") - (out / "jwks.json").write_text(json.dumps(keyset, indent=2) + "\n") + signed = sign(descriptor, governance) + keyset = {"keys": [governance_jwk, *operational_jwks]} + # Checked exactly as a peer will check it, before anything is written: a + # document this network would itself refuse never goes out. + verify(signed, keyset) + + # Keys first. A peer fetching between the two writes then finds the old + # descriptor with a JWKS that still verifies it -- never a descriptor naming + # a key that is not published yet. + out.mkdir(parents=True, exist_ok=True) + write_atomically(jwks_path, keyset) + write_atomically(descriptor_path, signed) print(f"{args.network_id} now hosts its descriptor at revision {args.revision}") - print(f" descriptor {out / 'openagrinet'}") - print(f" keys {out / 'jwks.json'}") + print(f" descriptor {descriptor_path}") + print(f" keys {jwks_path}") + print(f" governance {governance_jwk['kid']}") + print(f" operational {', '.join(descriptor['operationalKeys'])}") return 0 @@ -801,13 +1037,15 @@ def main(argv=None): help="descriptor revision; must increase on every republish") publish.add_argument("--operating-profile", default="OAN/v0.1,Beckn/v2", help="comma-separated version stack") - publish.add_argument("--status", default="Active", help="Active, Suspended or Retired") + publish.add_argument("--status", default="Active", choices=DESCRIPTOR_STATUSES, + help="what this network declares itself to be") publish.add_argument("--projection-ttl", type=int, default=None, help="seconds a peer may keep what it crawls; omit to let peers decide") publish.add_argument("--out", required=True, help="directory to write .well-known/ into") - publish.add_argument("--key", required=True, - help="this network's private key file; it signs the descriptor " - "AND cross-network traffic. Generated if absent") + publish.add_argument("--governance-key", required=True, + help="the governance private key file; it signs the descriptor and " + "NEVER traffic, so it must not be a registered key. " + "Generated (mode 600) if absent") publish.set_defaults(run=cmd_publish) verify_cmd = commands.add_parser("verify", help="fetch and check another network's documents") diff --git a/networkofnetwork/bin/setup.py b/networkofnetwork/bin/setup.py index 901c8439..bf720b97 100755 --- a/networkofnetwork/bin/setup.py +++ b/networkofnetwork/bin/setup.py @@ -209,7 +209,6 @@ def load_or_generate_keys(): print(f" {role}: id is now {network}, keeping the keypair") identities[role]["participantId"] = network KEYS.write_text(json.dumps(identities, indent=2)) - write_network_key(identities) return identities print(f"keys: generating one identity for {network}") @@ -225,27 +224,9 @@ def load_or_generate_keys(): KEYS.parent.mkdir(parents=True, exist_ok=True) KEYS.write_text(json.dumps(identities, indent=2)) KEYS.chmod(0o600) - write_network_key(identities) return identities -def write_network_key(identities): - """The same private key, in the one-line form oanfed.py reads. - - oanfed publishes this network's descriptor and signs it; the adapters sign - traffic with it; the registry holds its public half. One key, so one file -- - written here rather than generated there, so the two cannot drift into - publishing one key and signing with another. - - base64url unpadded, which is what a JWKS carries, so the file and the - published `x` are the same alphabet. - """ - raw = base64.b64decode(identities["network"]["signingPrivate"]) - path = KEYS.parent / "network.key" - path.write_text(base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + "\n") - path.chmod(0o600) - - # ----------------------------------------------------------------- registry def registry_url(): diff --git a/networkofnetwork/bin/test_oanfed.py b/networkofnetwork/bin/test_oanfed.py new file mode 100644 index 00000000..4ebbcc61 --- /dev/null +++ b/networkofnetwork/bin/test_oanfed.py @@ -0,0 +1,234 @@ +"""Checks for publishing and verifying the Network Descriptor and JWKS. + + python3 -m unittest networkofnetwork/bin/test_oanfed.py + +No network, no registry: the registry is a stub returning one entry. +""" + +import base64, copy, json, os, pathlib, sys, tempfile, unittest +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +import oanfed as o +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +NETWORK = "network-a.example.org" + + +def raw_public(private): + return o.public_of(private) + + +class StubRegistry: + entry = None + + def __init__(self, *_): + pass + + def participant(self, participant_id): + return self.entry if participant_id == NETWORK else None + + def schema_packs(self): + return ["https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1"] + + +class Base(unittest.TestCase): + def setUp(self): + self.dir = pathlib.Path(tempfile.mkdtemp()) + self.traffic = Ed25519PrivateKey.generate() + StubRegistry.entry = { + "participantId": NETWORK, "name": "Network A", + "baseUrl": f"https://{NETWORK}/network/discover", + "keys": [self.registry_key(self.traffic)], + } + self._real = o.ReadOnlyRegistry + o.ReadOnlyRegistry = StubRegistry + + def tearDown(self): + o.ReadOnlyRegistry = self._real + + @staticmethod + def registry_key(private, **extra): + return {"alg": "ed25519", "status": "active", "validFrom": "2026-01-01T00:00:00Z", + "key": base64.b64encode(raw_public(private)).decode(), **extra} + + def publish(self, revision=1, **overrides): + args = SimpleNamespace( + network_id=NETWORK, registry_url="http://registry", registry_timeout=1, + operator="", revision=revision, operating_profile="OAN/v0.1,Beckn/v2", + status="Active", projection_ttl=None, out=str(self.dir), + governance_key=str(self.dir / "keys" / "governance.key")) + vars(args).update(overrides) + o.cmd_publish(args) + well_known = self.dir / ".well-known" + return (json.loads((well_known / "openagrinet").read_text()), + json.loads((well_known / "jwks.json").read_text())) + + +class Canonical(unittest.TestCase): + def test_keys_sort_by_utf16_code_units(self): + # RFC 8785 section 3.2.3: U+1F600 sorts BEFORE U+FB33 in UTF-16. + keys = ["€", "\r", "דּ", "1", "\U0001F600", "\u0080", "ö"] + out = o._jcs({k: 0 for k in keys}) + order = [json.loads("{" + p + "}").popitem()[0] for p in out[1:-1].split(",")] + self.assertEqual(order, ["\r", "1", "\u0080", "ö", "€", "\U0001F600", "דּ"]) + + def test_escaping_and_no_whitespace(self): + self.assertEqual(o._jcs({"b": "a/\u001f\né", "a": [1, True, None]}), + '{"a":[1,true,null],"b":"a/\\u001f\\né"}') + + def test_floats_huge_ints_and_lone_surrogates_are_refused(self): + for bad in (1.5, 2 ** 53, "\ud800"): + with self.assertRaises(o.FederationError): + o._jcs({"x": bad}) + + def test_signature_is_excluded(self): + d = {"a": 1} + self.assertEqual(o.canonical(d), o.canonical({**d, "signature": {"value": "x"}})) + + +class Publish(Base): + def test_documents_verify_and_split_the_keys(self): + descriptor, keyset = self.publish() + o.verify(descriptor, keyset) + governance = descriptor["signature"]["keyId"] + self.assertEqual(descriptor["operationalKeys"], [o.jwk(raw_public(self.traffic))["kid"]]) + self.assertNotIn(governance, descriptor["operationalKeys"]) + self.assertEqual({k["kid"] for k in keyset["keys"]}, + {governance, *descriptor["operationalKeys"]}) + for field in ("version", "networkId", "name", "operator", "revision", "status", + "issuedAt", "operatingProfile", "schemaPacks", "services"): + self.assertIn(field, descriptor) + self.assertEqual(descriptor["operatingProfile"], ["OAN/v0.1", "Beckn/v2"]) + + def test_governance_key_is_created_private_and_reused(self): + first, _ = self.publish(1) + key = self.dir / "keys" / "governance.key" + self.assertEqual(key.stat().st_mode & 0o777, 0o600) + second, _ = self.publish(2) + self.assertEqual(first["signature"]["keyId"], second["signature"]["keyId"]) + + def test_readable_governance_key_is_refused(self): + self.publish(1) + os.chmod(self.dir / "keys" / "governance.key", 0o644) + with self.assertRaisesRegex(o.FederationError, "chmod 600"): + self.publish(2) + + def test_revision_must_increase(self): + self.publish(3) + for revision in (3, 2): + with self.assertRaisesRegex(o.FederationError, "does not increase"): + self.publish(revision) + + def test_corrupt_published_descriptor_is_not_read_as_revision_zero(self): + self.publish(5) + (self.dir / ".well-known" / "openagrinet").write_text("{") + with self.assertRaises(o.FederationError): + self.publish(1) + + def test_governance_key_that_is_also_a_traffic_key_is_refused(self): + self.publish(1) + seed = o.unb64url((self.dir / "keys" / "governance.key").read_text().strip()) + governance = Ed25519PrivateKey.from_private_bytes(seed) + StubRegistry.entry["keys"].append(self.registry_key(governance)) + with self.assertRaisesRegex(o.FederationError, "governance key must be separate"): + self.publish(2) + + def test_only_usable_registry_keys_are_delegated(self): + revoked, expired, upcoming = (Ed25519PrivateKey.generate() for _ in range(3)) + future = (datetime.now(timezone.utc) + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + StubRegistry.entry["keys"] += [ + self.registry_key(revoked, status="revoked"), + self.registry_key(expired, validUntil="2020-01-01T00:00:00Z"), + self.registry_key(upcoming, validFrom=future), + {"alg": "x25519", "status": "active", "key": base64.b64encode(b"e" * 32).decode()}, + ] + descriptor, _ = self.publish() + self.assertEqual(descriptor["operationalKeys"], + [o.jwk(raw_public(k))["kid"] for k in (self.traffic, upcoming)]) + + def test_no_usable_traffic_key_is_refused(self): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, status="revoked")] + with self.assertRaisesRegex(o.FederationError, "no active Ed25519 key"): + self.publish() + + def test_invalid_descriptor_is_never_written(self): + StubRegistry.entry["baseUrl"] = "http://insecure/discover" + with self.assertRaisesRegex(o.FederationError, "services.discovery"): + self.publish() + self.assertFalse((self.dir / ".well-known" / "openagrinet").exists()) + + def test_unregistered_network_is_refused(self): + with self.assertRaisesRegex(o.FederationError, "not in this registry"): + self.publish(network_id="other.example.org") + + +class Verify(Base): + def setUp(self): + super().setUp() + self.descriptor, self.keyset = self.publish() + + def refuses(self, descriptor=None, keyset=None, match=""): + with self.assertRaisesRegex(o.FederationError, match): + o.verify(self.descriptor if descriptor is None else descriptor, + self.keyset if keyset is None else keyset) + + def test_any_change_to_a_signed_field_is_caught(self): + for field, value in (("status", "Suspended"), ("revision", 99), + ("services", {"discovery": "https://evil.example/d"}), + ("operationalKeys", [self.keyset["keys"][1]["kid"], "extra"])): + tampered = {**self.descriptor, field: value} + with self.subTest(field=field): + self.refuses(tampered) + + def test_signer_must_not_also_be_delegated_for_traffic(self): + governance = self.descriptor["signature"]["keyId"] + d = {**self.descriptor, "operationalKeys": [governance]} + self.refuses(d, match="also delegated for traffic") + + def test_delegated_key_missing_from_jwks(self): + keyset = {"keys": [k for k in self.keyset["keys"] + if k["kid"] == self.descriptor["signature"]["keyId"]]} + self.refuses(keyset=keyset, match="operationalKeys not in the JWKS") + + def test_malformed_signatures_fail_cleanly(self): + for signature in (None, {}, {"algorithm": "EdDSA"}, + {**self.descriptor["signature"], "keyId": "nope"}, + {**self.descriptor["signature"], "value": "!!not-base64!!"}, + {**self.descriptor["signature"], "value": ""}): + with self.subTest(signature=signature): + self.refuses({**self.descriptor, "signature": signature}) + + def test_malformed_jwks(self): + dup = copy.deepcopy(self.keyset) + dup["keys"].append(dict(dup["keys"][0])) + short = copy.deepcopy(self.keyset) + short["keys"][0]["x"] = o.b64url(b"short") + for keyset in ({}, {"keys": "x"}, {"keys": [{"kty": "OKP"}]}, dup, short): + with self.subTest(keyset=keyset): + self.refuses(keyset=keyset) + + def test_contract_fields(self): + future = (datetime.now(timezone.utc) + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + for field, value in (("version", "0.1"), ("networkId", "Not A Domain"), + ("name", ""), ("operator", {}), ("revision", 0), + ("revision", True), ("status", "active"), ("issuedAt", future), + ("issuedAt", "2026-02-30T00:00:00Z"), ("operatingProfile", []), + ("schemaPacks", ["http://x/y"]), ("projectionTtl", -1)): + with self.subTest(field=field, value=value): + with self.assertRaises(o.FederationError): + o.validate_descriptor({**self.descriptor, field: value}) + + def test_admit_stores_only_delegated_keys(self): + # A key added to the JWKS without the governance signature delegating it + # must not reach the peer record either. + injected = copy.deepcopy(self.keyset) + injected["keys"].append(o.jwk(raw_public(Ed25519PrivateKey.generate()))) + record = o.admit_record(None, self.descriptor, injected) + self.assertEqual([k["key"] for k in record["keys"]], + [base64.b64encode(raw_public(self.traffic)).decode()]) + + +if __name__ == "__main__": + unittest.main() diff --git a/networkofnetwork/demo/README.md b/networkofnetwork/demo/README.md index 76676953..0f1efe82 100644 --- a/networkofnetwork/demo/README.md +++ b/networkofnetwork/demo/README.md @@ -72,14 +72,32 @@ make host-descriptor NETWORK_ID=mahavistara.oan.local ``` bharatvistar.oan.local now hosts its descriptor at revision 1 - descriptor networks/bharatvistar.oan.local/.well-known/openagrinet - keys networks/bharatvistar.oan.local/.well-known/jwks.json + descriptor networks/bharatvistar.oan.local/.well-known/openagrinet + keys networks/bharatvistar.oan.local/.well-known/jwks.json + governance + operational ``` -Two signed documents per network, served over TLS at the network's own domain. -One identity, one keypair: the `networkId` IS the `participantId`, the key that -signs the descriptor is the key that signs traffic, and its public half is the -registry entry's key. +Two signed documents per network, served over TLS at the network's own domain: +`/.well-known/openagrinet` (the Network Descriptor) and `/.well-known/jwks.json`. + +- **Signature:** Ed25519 (`"algorithm": "Ed25519"`, RFC 9864 name) over the + RFC 8785 (JCS) form of the descriptor minus its `signature`. Floats are + refused, so every implementation rebuilds the same bytes. +- **Two kinds of key**, as the contract requires. The **governance** key + (`keys/governance.key`, generated on first run, mode 600) signs the descriptor + and never traffic. The **operational** keys are the registry entry's active + keys -- the ones the adapters sign traffic with. The descriptor delegates them + by kid in `operationalKeys`, under the governance signature. +- **kid** is each key's RFC 7638 thumbprint, so a rotated key gets a new kid on + its own. To rotate an operational key, add it to the registry entry, re-run + `host-descriptor`, and revoke the old one once peers have refreshed. +- **revision** must increase: re-running bumps it, and `oanfed publish` refuses + one at or below what is already published. The JWKS is written before the + descriptor, each in one atomic step. + +Peers store only the delegated operational keys on `admit`, so the governance +key can never pass their traffic check. --- From da2391fbe1c176592e2ba2588faece22cadda27b Mon Sep 17 00:00:00 2001 From: Prajwal TD <157502711+tdprajwal@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:48:34 +0530 Subject: [PATCH 3/5] test: end-to-end check of the hosted descriptor and JWKS [OpenAgriNet/network-adapter#74] make e2e-descriptor publishes a new revision, fetches it over TLS through the front door as the peer would, tampers with the served files to confirm each forgery is refused, then restores what was served. Nothing is written to either registry. --- networkofnetwork/Makefile | 9 +- networkofnetwork/bin/e2e_descriptor.py | 244 +++++++++++++++++++++++++ networkofnetwork/demo/README.md | 11 ++ 3 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 networkofnetwork/bin/e2e_descriptor.py diff --git a/networkofnetwork/Makefile b/networkofnetwork/Makefile index 0b26a7cf..6aeabfa4 100644 --- a/networkofnetwork/Makefile +++ b/networkofnetwork/Makefile @@ -16,7 +16,7 @@ NET := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/network.sh -.PHONY: help network setup host-descriptor admit down destroy teardown ps logs ls +.PHONY: help network setup host-descriptor e2e-descriptor admit down destroy teardown ps logs ls # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -41,6 +41,13 @@ setup: ; @$(NET) setup $(NETWORK_ID) # adapter's /publish/ mount. host-descriptor: ; @$(NET) host-descriptor $(NETWORK_ID) +# End-to-end check of the hosted descriptor and jwks: publishes a new revision, +# fetches it over TLS as PEER would, tampers with it, and restores what was +# served. Writes nothing to either registry. +# +# make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +e2e-descriptor: ; @python3 $(dir $(NET))e2e_descriptor.py $(NETWORK_ID) $(PEER) + # Record our decision to deal with another network: fetch its descriptor over # TLS, verify the signature against the key it publishes, and store it. # diff --git a/networkofnetwork/bin/e2e_descriptor.py b/networkofnetwork/bin/e2e_descriptor.py new file mode 100644 index 00000000..1f5f6d29 --- /dev/null +++ b/networkofnetwork/bin/e2e_descriptor.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""End-to-end check of a network's hosted Network Descriptor and JWKS. + + make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local + +Needs both networks up (`make network`). It publishes a new revision with the +current oanfed.py, fetches it over TLS through the network's front door exactly +as a peer would, then tampers with the served files and checks each forgery is +refused. + +It leaves the network as it found it: the served documents and the governance +key are restored on exit, pass or fail. Nothing is written to either registry -- +admission is computed, not recorded. +""" + +import base64, copy, json, os, pathlib, subprocess, sys, tempfile, urllib.request + +HERE = pathlib.Path(__file__).resolve().parent +STATE = HERE.parent / "networks" +sys.path.insert(0, str(HERE)) +import oanfed as o +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +results = [] + + +def check(name, ok, detail=""): + results.append((ok, name)) + print(("PASS " if ok else "FAIL ") + name + (f" -- {detail}" if detail else "")) + + +def ports(network): + out = subprocess.run([sys.executable, str(HERE / "network-ports.py"), "ports", network], + capture_output=True, text=True, check=True).stdout + return dict(line.split("=", 1) for line in out.split()) + + +def run(*args): + p = subprocess.run([sys.executable, str(HERE / "oanfed.py"), *args], + capture_output=True, text=True) + return p.returncode, (p.stdout + p.stderr).strip() + + +def search(registry_port, network): + request = urllib.request.Request( + f"http://127.0.0.1:{registry_port}/api/v1/Participant/search", + data=json.dumps({"filters": {"participantId": {"eq": network}}}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=10) as response: + found = json.load(response) + rows = found if isinstance(found, list) else found.get("data", []) + return rows[0] if rows else None + + +def main(network, peer): + net_dir = STATE / network + well_known, gov = net_dir / ".well-known", net_dir / "keys" / "governance.key" + ca = str(STATE / "ca" / "ca.crt") + ours, theirs = ports(network), ports(peer) + door, registry = ours["FRONT_DOOR_PORT"], ours["REGISTRY_PORT"] + + files = ("openagrinet", "jwks.json") + original = {f: (well_known / f).read_bytes() if (well_known / f).exists() else None + for f in files} + gov_original = gov.read_bytes() if gov.exists() else None + + def publish(revision, *extra, key=gov, network_id=network, registry_url=None, out=net_dir): + return run("publish", "--network-id", network_id, + "--registry-url", registry_url or f"http://127.0.0.1:{registry}", + "--governance-key", str(key), "--revision", str(revision), + "--out", str(out), *extra) + + def verify(target=network, *extra): + return run("verify", "--network-id", target, "--resolve", f"127.0.0.1:{door}", *extra) + + def verify_trusted(target=network): + return verify(target, "--ca-file", ca) + + def curl(path): + p = subprocess.run(["curl", "-s", "-D", "-", "-o", "/dev/null", "--cacert", ca, + "--resolve", f"{network}:{door}:127.0.0.1", + f"https://{network}:{door}{path}"], capture_output=True, text=True) + return p.stdout.lower() + + served = lambda f: json.loads((well_known / f).read_text()) + + try: + # ---- happy path ---------------------------------------------------- + held = served("openagrinet")["revision"] if original["openagrinet"] else 0 + code, out = publish(held + 1) + check("publish the next revision from the registry", code == 0, out.splitlines()[0]) + if code != 0: + return + check("governance key is mode 600", (gov.stat().st_mode & 0o777) == 0o600) + new = {f: (well_known / f).read_bytes() for f in files} + + def restore_new(): + for f, b in new.items(): + (well_known / f).write_bytes(b) + + d, k = served("openagrinet"), served("jwks.json") + + for path in ("/.well-known/openagrinet", "/.well-known/jwks.json"): + h = curl(path) + first = h.splitlines()[0] if h else "no response" + check(f"GET {path} over TLS: 200, application/json, no-cache", + " 200" in first and "content-type: application/json" in h and "no-cache" in h, + first) + h = curl("/.well-known/nothing") + check("an unknown well-known path is 404", " 404" in (h.splitlines() or [""])[0]) + + code, out = verify_trusted() + check("a peer verifies it over TLS", code == 0 and f"revision {held + 1}" in out, + out.splitlines()[0]) + + entry = search(registry, network) + traffic = base64.b64decode(entry["keys"][0]["key"].removeprefix("base64:")) + identity = json.loads((net_dir / "keys/keys.json").read_text())["network"] + check("the delegated key is the registry key and the adapter's signing key", + d["operationalKeys"] == [o.jwk(traffic)["kid"]] + and base64.b64decode(identity["signingPublic"]) == traffic) + gov_kid = d["signature"]["keyId"] + check("the governance key is published, not delegated, not in the registry", + gov_kid in {x["kid"] for x in k["keys"]} and gov_kid not in d["operationalKeys"] + and o.public_of(o.load_or_create_key(gov)) != traffic) + check("Ed25519, version 1.0, every required field", + d["signature"]["algorithm"] == "Ed25519" and d["version"] == "1.0" + and all(f in d for f in ("networkId", "name", "operator", "revision", "status", + "issuedAt", "operatingProfile", "schemaPacks", "services"))) + + current = search(theirs["REGISTRY_PORT"], network) + fetched_d, fetched_k = o.Fetcher(ca_file=ca, resolve=f"127.0.0.1:{door}").fetch(network) + record = o.admit_record(current, fetched_d, fetched_k) + check(f"{peer} would store only the traffic key at the new revision (not written)", + [x["key"] for x in record["keys"]] == [base64.b64encode(traffic).decode()] + and record["revision"] == held + 1, + f"{peer} holds revision {current.get('revision') if current else None}") + try: + o.admit_record({**(current or {"participantId": network}), "revision": held + 1}, + fetched_d, fetched_k) + check("re-reading the same revision changes nothing", False) + except o.StaleRevision: + check("re-reading the same revision changes nothing", True) + + # ---- publish refuses ----------------------------------------------- + unchanged = lambda: (well_known / "openagrinet").read_bytes() == new["openagrinet"] + for revision in (held + 1, held, 0): + code, out = publish(revision) + check(f"publish refuses revision {revision} (not increasing)", + code != 0 and unchanged(), out) + code, out = publish(held + 2, "--status", "Paused") + check("publish refuses an unknown status", code != 0, out.splitlines()[-1]) + code, out = publish(held + 2, "--projection-ttl", "-5") + check("publish refuses a negative projectionTtl, writes nothing", + code != 0 and unchanged(), out) + os.chmod(gov, 0o644) + code, out = publish(held + 2) + os.chmod(gov, 0o600) + check("publish refuses a governance key others can read", code != 0, out) + traffic_key = pathlib.Path(tempfile.mkdtemp()) / "traffic.key" + traffic_key.write_text(o.b64url(base64.b64decode(identity["signingPrivate"])) + "\n") + traffic_key.chmod(0o600) + code, out = publish(held + 2, key=traffic_key) + check("publish refuses the traffic key as the governance key", code != 0, out) + code, out = publish(held + 2, network_id="nobody.example.org", + out=pathlib.Path(tempfile.mkdtemp())) + check("publish refuses a network not in the registry", code != 0, out) + code, out = publish(held + 2, "--registry-timeout", "2", + registry_url="http://127.0.0.1:1") + check("publish fails cleanly when the registry is down", + code != 0 and "Traceback" not in out, out) + + # ---- a peer refuses tampered documents ----------------------------- + def tampered(name, expect, mutate_d=None, mutate_k=None, raw=None): + dd, kk = copy.deepcopy(d), copy.deepcopy(k) + if mutate_d: + mutate_d(dd) + if mutate_k: + mutate_k(kk) + (well_known / "openagrinet").write_text(raw if raw is not None else json.dumps(dd)) + (well_known / "jwks.json").write_text(json.dumps(kk)) + code, out = verify_trusted() + check(f"peer refuses: {name}", code != 0 and "Traceback" not in out and expect in out, out) + restore_new() + + bad = "signature does not verify" + tampered("status changed after signing", bad, lambda x: x.update(status="Retired")) + tampered("discovery URL redirected", bad, + lambda x: x["services"].update(discovery="https://evil.example/d")) + tampered("revision bumped without re-signing", bad, + lambda x: x.update(revision=x["revision"] + 100)) + tampered("governance key also delegated for traffic", "also delegated", + lambda x: x["operationalKeys"].append(gov_kid)) + tampered("signature removed", "no signature", lambda x: x.pop("signature")) + tampered("algorithm set to none", "unsupported", + lambda x: x["signature"].update(algorithm="none")) + tampered("governance key dropped from the JWKS", "no published", + mutate_k=lambda x: x.update(keys=[y for y in x["keys"] if y["kid"] != gov_kid])) + tampered("delegated key dropped from the JWKS", "not in the JWKS", + mutate_k=lambda x: x.update(keys=[y for y in x["keys"] if y["kid"] == gov_kid])) + tampered("duplicate kid in the JWKS", "two keys", + mutate_k=lambda x: x["keys"].append(dict(x["keys"][0]))) + tampered("governance key swapped for an attacker's", bad, + mutate_k=lambda x: x["keys"].__setitem__(0, { + **o.jwk(o.public_of(Ed25519PrivateKey.generate())), "kid": gov_kid})) + tampered("descriptor names another network", "publishes a descriptor for", + lambda x: x.update(networkId=peer)) + tampered("unknown version", "version", lambda x: x.update(version="9.9")) + tampered("not JSON", "parsing", raw="oops") + tampered("larger than 1 MiB", "parsing", raw='{"pad":"' + "x" * (1 << 20) + '"}') + (well_known / "openagrinet").unlink() + code, out = verify_trusted() + check("peer refuses: descriptor missing (404)", code != 0 and "404" in out, out) + restore_new() + code, out = verify_trusted(peer) + check(f"peer refuses: {peer}'s name at {network}'s door (TLS name mismatch)", + code != 0 and "Traceback" not in out, out[:160]) + code, out = verify() + check("peer refuses: certificate from an untrusted CA", code != 0, out[:160]) + + finally: + for f, b in original.items(): + if b is None: + (well_known / f).unlink(missing_ok=True) + else: + (well_known / f).write_bytes(b) + if gov_original is None: + gov.unlink(missing_ok=True) + else: + gov.write_bytes(gov_original) + gov.chmod(0o600) + + restored = all(((well_known / f).read_bytes() if (well_known / f).exists() else None) == b + for f, b in original.items()) + check(f"{network} restored to what it served before", restored) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit("usage: e2e_descriptor.py NETWORK_ID PEER") + main(sys.argv[1], sys.argv[2]) + failed = [name for ok, name in results if not ok] + print(f"\n{len(results) - len(failed)}/{len(results)} passed") + sys.exit(1 if failed else 0) diff --git a/networkofnetwork/demo/README.md b/networkofnetwork/demo/README.md index 0f1efe82..6c6e6391 100644 --- a/networkofnetwork/demo/README.md +++ b/networkofnetwork/demo/README.md @@ -99,6 +99,17 @@ Two signed documents per network, served over TLS at the network's own domain: Peers store only the delegated operational keys on `admit`, so the governance key can never pass their traffic check. +To check it end to end -- publish, fetch over TLS as the peer would, then +tamper with the served files and confirm each forgery is refused: + +```bash +python3 -m unittest bin/test_oanfed.py +make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +``` + +The second restores whatever the network served before and writes nothing to +either registry. + --- ## 3. Each network admits the other (instant) From b4fe11d667ab26b4e77937aa42f78dffe40f4df6 Mon Sep 17 00:00:00 2001 From: Prajwal TD <157502711+tdprajwal@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:59:21 +0530 Subject: [PATCH 4/5] test: Postman checks for the hosted descriptor and JWKS [OpenAgriNet/network-adapter#74] Adds an "Identity documents" folder: GET each network's /.well-known/openagrinet and /.well-known/jwks.json from its front door and check the Descriptor contract fields, the Ed25519 keys, and that the governance key is published but never delegated for traffic. --- ...etwork-of-networks.postman_collection.json | 320 +++++++++++++++++- 1 file changed, 316 insertions(+), 4 deletions(-) diff --git a/networkofnetwork/demo/oan-network-of-networks.postman_collection.json b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json index 5beee5d9..3d8fed2f 100644 --- a/networkofnetwork/demo/oan-network-of-networks.postman_collection.json +++ b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json @@ -1,8 +1,8 @@ { "info": { "_postman_id": "d90ecfa3-df76-4844-a51c-32d73de9a2f9", - "name": "OAN \u2014 Network of Networks", - "description": "Two OAN networks side by side, each with its own registry, discovery and adapter.\n\n**The demo, in order:**\n1. Bharat 1 \u2014 publish mandi into A\n2. Bharat 2 \u2014 search A, returns mandi only\n3. Maha 1 \u2014 publish weather into B\n4. wait ~30s, no manual crawl\n5. Bharat 2 again \u2014 now returns mandi AND weather\n6. Bharat 3 \u2014 select mandi, answered locally\n7. Bharat 4 \u2014 select weather, crosses to B\n\nBring the stack up with `make network NETWORK_ID=...` in helmcharts/networkofnetwork. Full runbook in demo/README.md.", + "name": "OAN — Network of Networks", + "description": "Two OAN networks side by side, each with its own registry, discovery and adapter.\n\n**Identity documents (#74):** run the *Identity documents* folder first -- each network's signed Descriptor and JWKS.\n\n**The demo, in order:**\n1. Bharat 1 — publish mandi into A\n2. Bharat 2 — search A, returns mandi only\n3. Maha 1 — publish weather into B\n4. wait ~30s, no manual crawl\n5. Bharat 2 again — now returns mandi AND weather\n6. Bharat 3 — select mandi, answered locally\n7. Bharat 4 — select weather, crosses to B\n\nBring the stack up with `make network NETWORK_ID=...` in helmcharts/networkofnetwork. Full runbook in demo/README.md.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "variable": [ @@ -21,11 +21,323 @@ { "key": "maha_discovery", "value": "http://localhost:9401" + }, + { + "key": "bharat_front_door", + "value": "https://localhost:9306" + }, + { + "key": "maha_front_door", + "value": "https://localhost:9406" + }, + { + "key": "bharat_network_id", + "value": "bharatvistar.oan.local" + }, + { + "key": "maha_network_id", + "value": "mahavistara.oan.local" } ], "item": [ { - "name": "Bharat (network A \u2014 has mandi)", + "name": "Identity documents (issue #74)", + "description": "Each network's two published documents, fetched from its front door exactly where a peer reads them.\n\n**Before running:** Postman → Settings → turn OFF *SSL certificate verification* (the stack uses a local CA), and run `make host-descriptor NETWORK_ID=...` for each network with this branch's code.\n\nRun each Descriptor request before its JWKS request: the JWKS checks use the kids the Descriptor named.\n\nPostman cannot verify an Ed25519 signature, so these check shape, keys and the governance/traffic key split. For the signature and the tampering cases run `make e2e-descriptor NETWORK_ID=... PEER=...`.", + "item": [ + { + "name": "1. Bharat — Network Descriptor", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'bharat');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "pm.test('not cached stale (Cache-Control: no-cache)', () => {", + " pm.expect(pm.response.headers.get('Cache-Control') || '').to.include('no-cache');", + "});", + "const d = pm.response.json();", + "pm.test('every required field of the Descriptor contract', () => {", + " ['version','networkId','name','operator','revision','status','issuedAt',", + " 'operatingProfile','schemaPacks','services','operationalKeys','signature']", + " .forEach(f => pm.expect(d, f).to.have.property(f));", + " pm.expect(d.version).to.eql('1.0');", + " pm.expect(d.operator.name).to.be.a('string').and.not.empty;", + " pm.expect(d.revision).to.be.a('number').and.above(0);", + " pm.expect(['Active','Suspended','Retired']).to.include(d.status);", + " pm.expect(d.issuedAt).to.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$/);", + " pm.expect(d.operatingProfile).to.be.an('array').that.is.not.empty;", + "});", + "pm.test('networkId is the network this was fetched for', () => {", + " pm.expect(d.networkId).to.eql(pm.variables.get(prefix + '_network_id'));", + "});", + "pm.test('discovery URL and schema packs are https', () => {", + " pm.expect(d.services.discovery).to.match(/^https:\\/\\//);", + " d.schemaPacks.forEach(u => pm.expect(u).to.match(/^https:\\/\\//));", + "});", + "pm.test('signed with Ed25519 by a governance key that is NOT a traffic key', () => {", + " pm.expect(d.signature.algorithm).to.eql('Ed25519');", + " pm.expect(d.signature.value).to.match(/^[A-Za-z0-9_-]{86}$/);", + " pm.expect(d.operationalKeys).to.be.an('array').that.is.not.empty;", + " pm.expect(d.operationalKeys).to.not.include(d.signature.keyId);", + "});", + "pm.collectionVariables.set(prefix + '_governance_kid', d.signature.keyId);", + "pm.collectionVariables.set(prefix + '_operational_kids', JSON.stringify(d.operationalKeys));" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_front_door}}/.well-known/openagrinet", + "host": [ + "{{bharat_front_door}}" + ], + "path": [ + ".well-known", + "openagrinet" + ] + }, + "description": "The signed ID card: who the network is, its status, revision, versions, schema packs, discovery URL and which keys may sign traffic." + } + }, + { + "name": "2. Bharat — JWKS", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'bharat');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "const keys = pm.response.json().keys;", + "pm.test('every key is a 32-byte Ed25519 signing key with a unique kid', () => {", + " pm.expect(keys).to.be.an('array').that.is.not.empty;", + " const kids = keys.map(k => k.kid);", + " pm.expect(new Set(kids).size).to.eql(kids.length);", + " keys.forEach(k => {", + " pm.expect(k.kty).to.eql('OKP');", + " pm.expect(k.crv).to.eql('Ed25519');", + " pm.expect(k.alg).to.eql('Ed25519');", + " pm.expect(k.use).to.eql('sig');", + " pm.expect(k.x).to.match(/^[A-Za-z0-9_-]{43}$/);", + " });", + "});", + "pm.test('publishes the governance key and every delegated traffic key', () => {", + " const gov = pm.collectionVariables.get(prefix + '_governance_kid');", + " const ops = JSON.parse(pm.collectionVariables.get(prefix + '_operational_kids') || '[]');", + " pm.expect(gov, 'run the Descriptor request first').to.be.a('string');", + " const kids = keys.map(k => k.kid);", + " pm.expect(kids).to.include(gov);", + " ops.forEach(kid => pm.expect(kids).to.include(kid));", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_front_door}}/.well-known/jwks.json", + "host": [ + "{{bharat_front_door}}" + ], + "path": [ + ".well-known", + "jwks.json" + ] + }, + "description": "The public keys that verify the Descriptor (governance) and the delegated traffic keys (operational)." + } + }, + { + "name": "3. Maha — Network Descriptor", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'maha');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "pm.test('not cached stale (Cache-Control: no-cache)', () => {", + " pm.expect(pm.response.headers.get('Cache-Control') || '').to.include('no-cache');", + "});", + "const d = pm.response.json();", + "pm.test('every required field of the Descriptor contract', () => {", + " ['version','networkId','name','operator','revision','status','issuedAt',", + " 'operatingProfile','schemaPacks','services','operationalKeys','signature']", + " .forEach(f => pm.expect(d, f).to.have.property(f));", + " pm.expect(d.version).to.eql('1.0');", + " pm.expect(d.operator.name).to.be.a('string').and.not.empty;", + " pm.expect(d.revision).to.be.a('number').and.above(0);", + " pm.expect(['Active','Suspended','Retired']).to.include(d.status);", + " pm.expect(d.issuedAt).to.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$/);", + " pm.expect(d.operatingProfile).to.be.an('array').that.is.not.empty;", + "});", + "pm.test('networkId is the network this was fetched for', () => {", + " pm.expect(d.networkId).to.eql(pm.variables.get(prefix + '_network_id'));", + "});", + "pm.test('discovery URL and schema packs are https', () => {", + " pm.expect(d.services.discovery).to.match(/^https:\\/\\//);", + " d.schemaPacks.forEach(u => pm.expect(u).to.match(/^https:\\/\\//));", + "});", + "pm.test('signed with Ed25519 by a governance key that is NOT a traffic key', () => {", + " pm.expect(d.signature.algorithm).to.eql('Ed25519');", + " pm.expect(d.signature.value).to.match(/^[A-Za-z0-9_-]{86}$/);", + " pm.expect(d.operationalKeys).to.be.an('array').that.is.not.empty;", + " pm.expect(d.operationalKeys).to.not.include(d.signature.keyId);", + "});", + "pm.collectionVariables.set(prefix + '_governance_kid', d.signature.keyId);", + "pm.collectionVariables.set(prefix + '_operational_kids', JSON.stringify(d.operationalKeys));" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_front_door}}/.well-known/openagrinet", + "host": [ + "{{maha_front_door}}" + ], + "path": [ + ".well-known", + "openagrinet" + ] + }, + "description": "Same as Bharat, for Maha Vistara." + } + }, + { + "name": "4. Maha — JWKS", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'maha');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "const keys = pm.response.json().keys;", + "pm.test('every key is a 32-byte Ed25519 signing key with a unique kid', () => {", + " pm.expect(keys).to.be.an('array').that.is.not.empty;", + " const kids = keys.map(k => k.kid);", + " pm.expect(new Set(kids).size).to.eql(kids.length);", + " keys.forEach(k => {", + " pm.expect(k.kty).to.eql('OKP');", + " pm.expect(k.crv).to.eql('Ed25519');", + " pm.expect(k.alg).to.eql('Ed25519');", + " pm.expect(k.use).to.eql('sig');", + " pm.expect(k.x).to.match(/^[A-Za-z0-9_-]{43}$/);", + " });", + "});", + "pm.test('publishes the governance key and every delegated traffic key', () => {", + " const gov = pm.collectionVariables.get(prefix + '_governance_kid');", + " const ops = JSON.parse(pm.collectionVariables.get(prefix + '_operational_kids') || '[]');", + " pm.expect(gov, 'run the Descriptor request first').to.be.a('string');", + " const kids = keys.map(k => k.kid);", + " pm.expect(kids).to.include(gov);", + " ops.forEach(kid => pm.expect(kids).to.include(kid));", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_front_door}}/.well-known/jwks.json", + "host": [ + "{{maha_front_door}}" + ], + "path": [ + ".well-known", + "jwks.json" + ] + }, + "description": "Same as Bharat, for Maha Vistara." + } + } + ] + }, + { + "name": "Bharat (network A — has mandi)", "description": "Network A. Publishes its OWN mandi catalogue, and crawls B every 30s so B's weather also answers here.\n\nPorts: adapter 9300, discovery 9301.", "item": [ { @@ -162,7 +474,7 @@ ] }, { - "name": "Maha (network B \u2014 has weather)", + "name": "Maha (network B — has weather)", "description": "Network B. Publishes its OWN weather catalogue and shares it with A.\n\nB does NOT crawl: FEDERATION_CRAWL_PEERS=off. Otherwise each network mirrors the other and B ends up re-importing its own catalogue through A's copy of it.\n\nPorts: adapter 9400, discovery 9401.", "item": [ { From 3c4e73d6324a355c78b81ea1a21bfc51ae55ad0d Mon Sep 17 00:00:00 2001 From: Prajwal TD <157502711+tdprajwal@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:31:24 +0530 Subject: [PATCH 5/5] fix: say why host-descriptor fails when the network is not set up here network.sh runs under set -e, so reading a missing .env exited make with no message. Check for the network's state first and name the fix. --- networkofnetwork/bin/network.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/networkofnetwork/bin/network.sh b/networkofnetwork/bin/network.sh index 249ee966..cb49cad1 100755 --- a/networkofnetwork/bin/network.sh +++ b/networkofnetwork/bin/network.sh @@ -100,6 +100,10 @@ cmd_host_descriptor() { local dir; dir="$(network_dir "$network")" local descriptor="$dir/.well-known/openagrinet" + # The state lives with the checkout that brought the network up. Without + # this, the .env read below fails under set -e and make exits with no word. + [ -f "$dir/.env" ] || die "$network is not set up in $STATE -- run 'make network NETWORK_ID=$network' from this checkout first" + # A revision must only ever INCREASE -- another network refuses a descriptor # offering one at or below what it already holds, which is what stops a # withdrawn declaration being replayed as current. So it is derived from the