diff --git a/networkofnetwork/.env.example b/networkofnetwork/.env.example new file mode 100644 index 00000000..d5b1fb93 --- /dev/null +++ b/networkofnetwork/.env.example @@ -0,0 +1,229 @@ +# Copy to .env and fill in every value. bin/setup.py refuses +# to touch the registry while any of them is still a placeholder -- the registry +# is append-only, so a row seeded with a placeholder burns that id for good. +# +# One line per variable, and one meaning per name. Where a name carries a +# prefix, the prefix says WHOSE the value is: MAUSAMGRAM_/AGMARKNET_/VISTAAR_/ +# POCRA_ are the four upstream entities, CONSUMER_/NETWORK_/PROVIDER_ are the +# three adapters, REGISTRY_/KEYCLOAK_/DISCOVERY_ are the services. +# +# Every credential below is a shipped default, which means it is public. Change +# all of them before the stack is reachable by anyone but you. The adapter +# keypairs are the exception: setup.py generates those into keys/keys.json. + +# ---- images ---------------------------------------------------------------- +# Pinned, not latest, so what is deployed is in the diff. The adapter image and +# the adapter configs in this repo move together: a config names plugins by id +# and an id is a .so inside the image. +ADAPTER_IMAGE=ghcr.io/openagrinet/network-adapter:0.0.1 +DISCOVERY_IMAGE=ghcr.io/openagrinet/discovery-service:0.0.1 +# Source dir for `docker compose build discovery`; nothing else reads it. +DISCOVERY_CONTEXT=../../discovery-service +# That build's base images. Compose already defaults them to public ones because +# the Dockerfile's own dhi.io/* defaults 401 without a Hardened Images +# subscription. Uncomment only if you have one. +#DISCOVERY_BUILD_IMAGE=dhi.io/golang:1.27-alpine-dev +#DISCOVERY_RUNTIME_IMAGE=dhi.io/static:20260611-musl-alpine +# The two mock upstreams. No capability uses one; `make up` does not start them. +MOCKIMD_IMAGE=ghcr.io/nisargabd/oan-mockimd:latest +MOCKAGMARKNET_IMAGE=ghcr.io/nisargabd/oan-mockagmarknet:latest + +# ---- ports ----------------------------------------------------------------- +# All published on 127.0.0.1 except the reverse proxy's 80/443, and the loopback +# binding is written literally in docker-compose.yml rather than taken from a +# variable. Reach the rest over an SSH tunnel. +REGISTRY_PORT=8081 +KEYCLOAK_PORT=8080 +KEYCLOAK_ADMIN_PORT=9990 +DISCOVERY_PORT=8090 +PROVIDER_ADAPTER_PORT=9200 +NETWORK_ADAPTER_PORT=9201 +CONSUMER_ADAPTER_PORT=9202 +# Only for looking at a mock directly; the adapter reaches it by service name. +MOCKIMD_PORT=9100 +MOCKAGMARKNET_PORT=9101 +# The reverse-proxy profile (nginx-proxy-manager). 80/443 are public, admin is not. +REVERSE_PROXY_HTTP_PORT=80 +REVERSE_PROXY_HTTPS_PORT=443 +NPM_ADMIN_PORT=81 +# The observability profile (hyperdx + clickhouse). +HYPERDX_UI_PORT=8085 +HYPERDX_OTLP_GRPC_PORT=4317 +HYPERDX_OTLP_HTTP_PORT=4318 +CLICKHOUSE_HTTP_PORT=8123 + +# ---- registry tier --------------------------------------------------------- +# CHANGE every credential in this block before the stack is exposed. +REGISTRY_IMAGE_TAG=v2.0.0 +# registry-db. Named REGISTRY_DB_ because discovery-db is a separate database. +REGISTRY_DB_NAME=registry +REGISTRY_DB_USER=postgres +REGISTRY_DB_PASSWORD=postgres +# The Keycloak login setup.py exchanges for the registry write token. +REGISTRY_ADMIN_USER=no-user +REGISTRY_ADMIN_PASSWORD=no-user-password +KEYCLOAK_REALM=sunbird-rc +KEYCLOAK_ADMIN_USER=admin +KEYCLOAK_ADMIN_PASSWORD=admin +# Two Keycloak clients: admin-api mints tokens, registry-frontend logs users in. +KEYCLOAK_ADMIN_CLIENT_ID=admin-api +KEYCLOAK_ADMIN_CLIENT_SECRET=1b46c0f2-6a1c-4a2f-8b0f-2b6b30bcd2b2 +KEYCLOAK_CLIENT_ID=registry-frontend +# Password Keycloak assigns to registry users it provisions. +KEYCLOAK_DEFAULT_USER_PASSWORD=abcd@123 + +# ---- telemetry ------------------------------------------------------------- +# Both components ship to this collector; a bare host:port, no scheme. +OTLP_ENDPOINT=hyperdx:4317 +# The deployment name on every stream, from both components. +OTEL_ENVIRONMENT=dev +# ADAPTERS ONLY, all three signals. false builds no exporter and never dials -- +# set it for `make up-core`, which does not start hyperdx. +ADAPTER_OTEL_ENABLED=true +# DISCOVERY ONLY, and a separate switch: `none` for `make up-core`. +DISCOVERY_OTEL_EXPORTER=otlp + +# ---- discovery ------------------------------------------------------------- +# The network this deployment belongs to; pairs with the spec it validates against. +BECKN_NETWORK_ID=oan-dev +BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml +# Required when DISCOVERY_OTEL_EXPORTER is otlp: it attributes discovery's spans. +# Needs no keypair and no registry row -- discovery does not sign. +DISCOVERY_SUBSCRIBER_ID=discovery.oan.dev +# The sector. Checked against the build's sector list; every OAN component emits it. +DISCOVERY_DOMAIN=Agriculture +# GeoJSON shapes ONE catalog may add to the spatial index; past it, publish is +# PARTIAL and the excess is unsearchable. Count shapes, not resources. +# +# 50000 against the image's default of 256, because that number counts SHAPES +# while the cost it guards is incurred per H3 CELL -- and the two differ by up +# to 8192 apiece. A Point is one cell; a state-sized polygon is the ceiling. So +# a points-only catalog can raise this a long way and a polygon-heavy one +# cannot. The mandi catalog is 272 markets carrying a Point under both +# coverageAreas and market.location: 544 shapes, 544 cells, against the ~2.1M +# cells the default stands in for. Measured 2026-09-12 -- 50000 Points is +# roughly 3 s and 18 MB of index, so this is sized for where the catalog is +# going rather than where it is. +# +# RAISE IT BEFORE A BULK PUBLISH, not after. While a catalog is over the +# ceiling, republishing a resource that sits past the cut-off clears its +# geometry rows and writes none back -- it drops off the map silently. +# +# Interim: discovery-service#32 moves the budget onto cells, where it stops +# depending on what you publish and this line goes away. +DISCOVERY_MAX_GEOMETRIES_PER_CATALOG=50000 +# Requests per second one client may make, and the bucket it may burst to. +# Effectively off: the image defaults to 20/40, which is under any scripted +# sweep, and a throttled request NACKs with AUT_RATE_LIMITED and no `catalogs` +# -- so it reads as an empty result, not as backpressure. Cannot be 0: the +# boot requires burst >= rps > 0. +DISCOVERY_RATE_LIMIT_RPS=100000 +DISCOVERY_RATE_LIMIT_BURST=100000 + +# ---- capability schemas ---------------------------------------------------- +# The ref in OpenAgriNet/network-specs that setup.py builds the four +# SchemaRegistry rows from. Those rows say what a capability MEANS -- the pack +# defining its attributes -- as against the *_MAPPING_URL values below, which +# say how one upstream serves it. A branch, tag or commit; the path is derived +# from the capability code, so schema///attributes.yaml. +# +# GET THIS RIGHT BEFORE THE FIRST RUN. The registry checks the URL's shape and +# never fetches it, so a ref that does not serve the pack stores happily and +# fails at whoever resolves it. It also cannot be corrected: SchemaRegistry +# rejects updates and its delete is soft, keeping the unique index. Today main +# does NOT serve these packs -- they are on schema-packs-v0.1 until that merges. +SCHEMA_PACK_REF=main + +# ---- the three adapter identities ------------------------------------------ +# setup.py registers these three and generates a keypair for each. A participant +# id IS the network identity a signature is verified against, so it must be +# hostname-shaped. Pick them once: the registry cannot update or truly delete a +# row, so an id is never reusable. +CONSUMER_SUBSCRIBER_ID=consumer.oan.dev +NETWORK_SUBSCRIBER_ID=network.oan.dev +PROVIDER_SUBSCRIBER_ID=provider.oan.dev +# The address PEERS are told to POST to, published in each registry row. Nothing +# inside this stack resolves it -- routing here is by compose service name -- so +# a wrong value costs nothing locally and everything on a shared network. +CONSUMER_BASE_URL=https://consumer.oan.dev +NETWORK_BASE_URL=https://network.oan.dev +PROVIDER_BASE_URL=https://provider.oan.dev + +# ---- the four upstream providers ------------------------------------------- +# setup.py creates all four plus a capability binding each -- seven participants +# and four bindings in all. The registry is unreachable from outside the stack, +# so there is no second way to create them. +# +# THE BINDING KEY IS |. The provider adapter builds +# it from the incoming payload and matches it against its own config; setup.py +# renders that config AND seeds the registry from these same values, which is +# what keeps the two from disagreeing. Change one, re-run setup.py. +# A mismatch is a 404, either NET_ENTITY_NOT_FOUND or "no active record". +# +# All four are REAL external APIs -- the stack needs egress to each. + +# WeatherObservation, from IMD Mausamgram. One ready-made header value, "Basic " +# prefix included, sent verbatim (authScheme header) -- IMD issues a single +# opaque token, not a user/password pair. +MAUSAMGRAM_PARTICIPANT_ID=mausamgram +MAUSAMGRAM_CAPABILITY=openagrinet:WeatherObservation +MAUSAMGRAM_BASE_URL=https://mausamgram.imd.gov.in +MAUSAMGRAM_PATH=/nwpapi/get-daily +MAUSAMGRAM_AUTH= +MAUSAMGRAM_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/mausamgram/weather-observation.select.yaml + +# MandiPrice, from Agmarknet. authScheme tokenQuery: the adapter POSTs the +# access name and password to the token URL and sends the token it gets back as +# a query parameter. Those two reach the adapter as environment variables only. +# TOKEN_URL IS ABSOLUTE, not a path: it is rendered into the config verbatim and +# POSTed as given, so a bare path fails with unsupported protocol scheme "". +# KNOWN AND UNFIXED: "no records" comes back as HTTP 400, which the adapter +# treats as permanent -- so an empty market reads as ERROR, not "no prices". +AGMARKNET_PARTICIPANT_ID=agmarknet +AGMARKNET_CAPABILITY=openagrinet:MandiPrice +AGMARKNET_BASE_URL= +AGMARKNET_PATH=/v1/fetch-agmarknet-vistaar +AGMARKNET_TOKEN_URL= +AGMARKNET_ACCESS_NAME= +AGMARKNET_PASSWORD= +AGMARKNET_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/agmarknet/mandi-price.select.yaml + +# KnowledgeAdvisory, from Bharat Vistaar -- the entity that runs the corpus, not +# the docs-pipeline that filled it. authScheme oauth2 client credentials, and a +# POST. The two secrets reach the adapter as environment variables only. +VISTAAR_PARTICIPANT_ID=bharat-vistaar +VISTAAR_CAPABILITY=openagrinet:KnowledgeAdvisory +VISTAAR_BASE_URL= +VISTAAR_PATH=/docs-pipeline-api/search +VISTAAR_TOKEN_URL= +VISTAAR_CLIENT_ID= +VISTAAR_CLIENT_SECRET= +VISTAAR_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/knowledge/knowledge-advisory.select.yaml + +# AgricultureFacility, from POCRA. A POST, and the only capability needing no +# credential -- its registry row publishes authScheme none. +# TWO MEASURED BEHAVIOURS: it takes exactly ONE facility type per call and +# caches on message_id for ten minutes while IGNORING location; and after ~20 +# calls it answers 200 with an empty catalogue, indistinguishable from "nothing +# nearby". That is why searchConcurrency is pinned to 1. Do not raise it. +POCRA_PARTICIPANT_ID=pocra +POCRA_CAPABILITY=openagrinet:AgricultureFacility +POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in +POCRA_PATH=/search +POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/pocra/agriculture-facility.select.yaml +FEDERATION_INTENT={"filters": {"type": "jsonpath", "expression": "$.catalogs[*] ? (@.isActive == true)"}} + +# on|off. off means this network crawls nobody -- it still ANSWERS another +# network's crawl, which is the network module's job and unaffected. +FEDERATION_CRAWL_PEERS=on + +# How long a peer may serve a copy of our catalogues, in seconds. It is also +# what paces their re-crawl: the crawler refreshes at projectionTtl/3, floored +# at 30s, so 30 here means a peer re-crawls us every 30 seconds. Short on +# purpose -- it is what makes a catalog visibly cross during a demo. +FEDERATION_PROJECTION_TTL=30 + +# Upper bound on the re-crawl interval. Recomputed from the peers found on +# each pass, so this is also how long a deployment with no peers at startup +# waits before looking again. +FEDERATION_REFRESH_SECONDS=30 diff --git a/networkofnetwork/.gitignore b/networkofnetwork/.gitignore new file mode 100644 index 00000000..8d7fbe83 --- /dev/null +++ b/networkofnetwork/.gitignore @@ -0,0 +1,8 @@ +# One directory per network, and all of it generated: .env, the keypair, TLS +# material, the rendered adapter config and the signed documents. +# +# Ignored as a WHOLE rather than by pattern -- a rule naming .env and keys/ +# would not catch networks//tls/server.key, and the cost of +# missing one is a private key in a public repository. +networks/ +__pycache__/ diff --git a/networkofnetwork/Makefile b/networkofnetwork/Makefile new file mode 100644 index 00000000..6aeabfa4 --- /dev/null +++ b/networkofnetwork/Makefile @@ -0,0 +1,76 @@ +# Network of networks, locally. +# +# Runs a whole OAN network by name, as many times as you like on one machine -- +# each with its own registry, its own discovery, its own adapter and a front +# door the others reach it at. Two of them is federation with nothing stubbed. +# +# make network NETWORK_ID=bharatvistar.oan.local +# make network NETWORK_ID=mahavistara.oan.local +# make ls +# +# Self-contained: its own .env.example, registry schemas, setup and compose +# file. Nothing here reads or writes anything outside this directory. +# +# Anchored to this file's own directory rather than $(PWD), so +# `make -C networkofnetwork network ...` works from the repo root. + +NET := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/network.sh + +.PHONY: help network setup host-descriptor e2e-descriptor admit down destroy teardown ps logs ls + +# Default target: running a bare `make` in a directory that can delete a +# Postgres volume should print the menu, not pick something from it. +help: + @$(NET) + +# ------------------------------------------------------------- one network + +# Everything: ports, certificates, .env, containers, registry seed, front door. +# Safe to re-run -- it is also how you apply a config change. +network: ; @$(NET) up $(NETWORK_ID) + +# Re-seed the registry and re-render the adapter config only. +setup: ; @$(NET) setup $(NETWORK_ID) + +# Generate and sign the two documents this network HOSTS at its own domain -- +# the descriptor and its jwks -- from its registry entry. Run it after +# `network`, and again whenever the entry changes. +# +# Named for the documents, not for the verb: "publish" on its own reads as +# publishing a CATALOG, which is a different thing entirely and goes to the +# adapter's /publish/ mount. +host-descriptor: ; @$(NET) host-descriptor $(NETWORK_ID) + +# End-to-end check of the hosted descriptor and jwks: publishes a new revision, +# fetches it over TLS as PEER would, tampers with it, and restores what was +# served. Writes nothing to either registry. +# +# make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +e2e-descriptor: ; @python3 $(dir $(NET))e2e_descriptor.py $(NETWORK_ID) $(PEER) + +# Record our decision to deal with another network: fetch its descriptor over +# TLS, verify the signature against the key it publishes, and store it. +# +# Two networks, so two variables. NETWORK_ID is US -- whose registry is written +# -- and PEER is them. It is one direction only; run it both ways for a pair. +# +# make admit NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +admit: ; @$(NET) admit $(NETWORK_ID) $(PEER) + +down: ; @$(NET) down $(NETWORK_ID) + +# Deletes the data AND releases the ports, so the block can be reused. +destroy: ; @$(NET) destroy $(NETWORK_ID) + +# Destroy EVERY deployed network. Asks first; TEARDOWN_YES=1 skips the prompt. +# +# make teardown +teardown: ; @$(NET) teardown + +ps: ; @$(NET) ps $(NETWORK_ID) +logs: ; @$(NET) logs $(NETWORK_ID) $(SVC) + +# ----------------------------------------------------------- every network + +# What is deployed on this machine, and which ports each one holds. +ls: ; @$(NET) ls diff --git a/networkofnetwork/bin/e2e_descriptor.py b/networkofnetwork/bin/e2e_descriptor.py new file mode 100644 index 00000000..1f5f6d29 --- /dev/null +++ b/networkofnetwork/bin/e2e_descriptor.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""End-to-end check of a network's hosted Network Descriptor and JWKS. + + make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local + +Needs both networks up (`make network`). It publishes a new revision with the +current oanfed.py, fetches it over TLS through the network's front door exactly +as a peer would, then tampers with the served files and checks each forgery is +refused. + +It leaves the network as it found it: the served documents and the governance +key are restored on exit, pass or fail. Nothing is written to either registry -- +admission is computed, not recorded. +""" + +import base64, copy, json, os, pathlib, subprocess, sys, tempfile, urllib.request + +HERE = pathlib.Path(__file__).resolve().parent +STATE = HERE.parent / "networks" +sys.path.insert(0, str(HERE)) +import oanfed as o +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +results = [] + + +def check(name, ok, detail=""): + results.append((ok, name)) + print(("PASS " if ok else "FAIL ") + name + (f" -- {detail}" if detail else "")) + + +def ports(network): + out = subprocess.run([sys.executable, str(HERE / "network-ports.py"), "ports", network], + capture_output=True, text=True, check=True).stdout + return dict(line.split("=", 1) for line in out.split()) + + +def run(*args): + p = subprocess.run([sys.executable, str(HERE / "oanfed.py"), *args], + capture_output=True, text=True) + return p.returncode, (p.stdout + p.stderr).strip() + + +def search(registry_port, network): + request = urllib.request.Request( + f"http://127.0.0.1:{registry_port}/api/v1/Participant/search", + data=json.dumps({"filters": {"participantId": {"eq": network}}}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=10) as response: + found = json.load(response) + rows = found if isinstance(found, list) else found.get("data", []) + return rows[0] if rows else None + + +def main(network, peer): + net_dir = STATE / network + well_known, gov = net_dir / ".well-known", net_dir / "keys" / "governance.key" + ca = str(STATE / "ca" / "ca.crt") + ours, theirs = ports(network), ports(peer) + door, registry = ours["FRONT_DOOR_PORT"], ours["REGISTRY_PORT"] + + files = ("openagrinet", "jwks.json") + original = {f: (well_known / f).read_bytes() if (well_known / f).exists() else None + for f in files} + gov_original = gov.read_bytes() if gov.exists() else None + + def publish(revision, *extra, key=gov, network_id=network, registry_url=None, out=net_dir): + return run("publish", "--network-id", network_id, + "--registry-url", registry_url or f"http://127.0.0.1:{registry}", + "--governance-key", str(key), "--revision", str(revision), + "--out", str(out), *extra) + + def verify(target=network, *extra): + return run("verify", "--network-id", target, "--resolve", f"127.0.0.1:{door}", *extra) + + def verify_trusted(target=network): + return verify(target, "--ca-file", ca) + + def curl(path): + p = subprocess.run(["curl", "-s", "-D", "-", "-o", "/dev/null", "--cacert", ca, + "--resolve", f"{network}:{door}:127.0.0.1", + f"https://{network}:{door}{path}"], capture_output=True, text=True) + return p.stdout.lower() + + served = lambda f: json.loads((well_known / f).read_text()) + + try: + # ---- happy path ---------------------------------------------------- + held = served("openagrinet")["revision"] if original["openagrinet"] else 0 + code, out = publish(held + 1) + check("publish the next revision from the registry", code == 0, out.splitlines()[0]) + if code != 0: + return + check("governance key is mode 600", (gov.stat().st_mode & 0o777) == 0o600) + new = {f: (well_known / f).read_bytes() for f in files} + + def restore_new(): + for f, b in new.items(): + (well_known / f).write_bytes(b) + + d, k = served("openagrinet"), served("jwks.json") + + for path in ("/.well-known/openagrinet", "/.well-known/jwks.json"): + h = curl(path) + first = h.splitlines()[0] if h else "no response" + check(f"GET {path} over TLS: 200, application/json, no-cache", + " 200" in first and "content-type: application/json" in h and "no-cache" in h, + first) + h = curl("/.well-known/nothing") + check("an unknown well-known path is 404", " 404" in (h.splitlines() or [""])[0]) + + code, out = verify_trusted() + check("a peer verifies it over TLS", code == 0 and f"revision {held + 1}" in out, + out.splitlines()[0]) + + entry = search(registry, network) + traffic = base64.b64decode(entry["keys"][0]["key"].removeprefix("base64:")) + identity = json.loads((net_dir / "keys/keys.json").read_text())["network"] + check("the delegated key is the registry key and the adapter's signing key", + d["operationalKeys"] == [o.jwk(traffic)["kid"]] + and base64.b64decode(identity["signingPublic"]) == traffic) + gov_kid = d["signature"]["keyId"] + check("the governance key is published, not delegated, not in the registry", + gov_kid in {x["kid"] for x in k["keys"]} and gov_kid not in d["operationalKeys"] + and o.public_of(o.load_or_create_key(gov)) != traffic) + check("Ed25519, version 1.0, every required field", + d["signature"]["algorithm"] == "Ed25519" and d["version"] == "1.0" + and all(f in d for f in ("networkId", "name", "operator", "revision", "status", + "issuedAt", "operatingProfile", "schemaPacks", "services"))) + + current = search(theirs["REGISTRY_PORT"], network) + fetched_d, fetched_k = o.Fetcher(ca_file=ca, resolve=f"127.0.0.1:{door}").fetch(network) + record = o.admit_record(current, fetched_d, fetched_k) + check(f"{peer} would store only the traffic key at the new revision (not written)", + [x["key"] for x in record["keys"]] == [base64.b64encode(traffic).decode()] + and record["revision"] == held + 1, + f"{peer} holds revision {current.get('revision') if current else None}") + try: + o.admit_record({**(current or {"participantId": network}), "revision": held + 1}, + fetched_d, fetched_k) + check("re-reading the same revision changes nothing", False) + except o.StaleRevision: + check("re-reading the same revision changes nothing", True) + + # ---- publish refuses ----------------------------------------------- + unchanged = lambda: (well_known / "openagrinet").read_bytes() == new["openagrinet"] + for revision in (held + 1, held, 0): + code, out = publish(revision) + check(f"publish refuses revision {revision} (not increasing)", + code != 0 and unchanged(), out) + code, out = publish(held + 2, "--status", "Paused") + check("publish refuses an unknown status", code != 0, out.splitlines()[-1]) + code, out = publish(held + 2, "--projection-ttl", "-5") + check("publish refuses a negative projectionTtl, writes nothing", + code != 0 and unchanged(), out) + os.chmod(gov, 0o644) + code, out = publish(held + 2) + os.chmod(gov, 0o600) + check("publish refuses a governance key others can read", code != 0, out) + traffic_key = pathlib.Path(tempfile.mkdtemp()) / "traffic.key" + traffic_key.write_text(o.b64url(base64.b64decode(identity["signingPrivate"])) + "\n") + traffic_key.chmod(0o600) + code, out = publish(held + 2, key=traffic_key) + check("publish refuses the traffic key as the governance key", code != 0, out) + code, out = publish(held + 2, network_id="nobody.example.org", + out=pathlib.Path(tempfile.mkdtemp())) + check("publish refuses a network not in the registry", code != 0, out) + code, out = publish(held + 2, "--registry-timeout", "2", + registry_url="http://127.0.0.1:1") + check("publish fails cleanly when the registry is down", + code != 0 and "Traceback" not in out, out) + + # ---- a peer refuses tampered documents ----------------------------- + def tampered(name, expect, mutate_d=None, mutate_k=None, raw=None): + dd, kk = copy.deepcopy(d), copy.deepcopy(k) + if mutate_d: + mutate_d(dd) + if mutate_k: + mutate_k(kk) + (well_known / "openagrinet").write_text(raw if raw is not None else json.dumps(dd)) + (well_known / "jwks.json").write_text(json.dumps(kk)) + code, out = verify_trusted() + check(f"peer refuses: {name}", code != 0 and "Traceback" not in out and expect in out, out) + restore_new() + + bad = "signature does not verify" + tampered("status changed after signing", bad, lambda x: x.update(status="Retired")) + tampered("discovery URL redirected", bad, + lambda x: x["services"].update(discovery="https://evil.example/d")) + tampered("revision bumped without re-signing", bad, + lambda x: x.update(revision=x["revision"] + 100)) + tampered("governance key also delegated for traffic", "also delegated", + lambda x: x["operationalKeys"].append(gov_kid)) + tampered("signature removed", "no signature", lambda x: x.pop("signature")) + tampered("algorithm set to none", "unsupported", + lambda x: x["signature"].update(algorithm="none")) + tampered("governance key dropped from the JWKS", "no published", + mutate_k=lambda x: x.update(keys=[y for y in x["keys"] if y["kid"] != gov_kid])) + tampered("delegated key dropped from the JWKS", "not in the JWKS", + mutate_k=lambda x: x.update(keys=[y for y in x["keys"] if y["kid"] == gov_kid])) + tampered("duplicate kid in the JWKS", "two keys", + mutate_k=lambda x: x["keys"].append(dict(x["keys"][0]))) + tampered("governance key swapped for an attacker's", bad, + mutate_k=lambda x: x["keys"].__setitem__(0, { + **o.jwk(o.public_of(Ed25519PrivateKey.generate())), "kid": gov_kid})) + tampered("descriptor names another network", "publishes a descriptor for", + lambda x: x.update(networkId=peer)) + tampered("unknown version", "version", lambda x: x.update(version="9.9")) + tampered("not JSON", "parsing", raw="oops") + tampered("larger than 1 MiB", "parsing", raw='{"pad":"' + "x" * (1 << 20) + '"}') + (well_known / "openagrinet").unlink() + code, out = verify_trusted() + check("peer refuses: descriptor missing (404)", code != 0 and "404" in out, out) + restore_new() + code, out = verify_trusted(peer) + check(f"peer refuses: {peer}'s name at {network}'s door (TLS name mismatch)", + code != 0 and "Traceback" not in out, out[:160]) + code, out = verify() + check("peer refuses: certificate from an untrusted CA", code != 0, out[:160]) + + finally: + for f, b in original.items(): + if b is None: + (well_known / f).unlink(missing_ok=True) + else: + (well_known / f).write_bytes(b) + if gov_original is None: + gov.unlink(missing_ok=True) + else: + gov.write_bytes(gov_original) + gov.chmod(0o600) + + restored = all(((well_known / f).read_bytes() if (well_known / f).exists() else None) == b + for f, b in original.items()) + check(f"{network} restored to what it served before", restored) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit("usage: e2e_descriptor.py NETWORK_ID PEER") + main(sys.argv[1], sys.argv[2]) + failed = [name for ok, name in results if not ok] + print(f"\n{len(results) - len(failed)}/{len(results)} passed") + sys.exit(1 if failed else 0) diff --git a/networkofnetwork/bin/network-certs.sh b/networkofnetwork/bin/network-certs.sh new file mode 100755 index 00000000..b45d2725 --- /dev/null +++ b/networkofnetwork/bin/network-certs.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Give each network a TLS certificate for its own name. +# +# The contract says a network is reached over https, and the registry's +# Participant schema pins baseUrl to ^https://. Serving plain HTTP locally would +# mean either weakening that schema or storing a URL the stack cannot use -- both +# of which hide the thing a POC is supposed to exercise. +# +# So: one local CA, and a certificate per network signed by it. Peers verify +# against the CA, which is real certificate verification. What a laptop still +# cannot do is prove it OWNS the domain -- that is the one property this setup +# does not and cannot demonstrate, and it is why the CA is trusted explicitly by +# path rather than installed into any trust store. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Everything written here is private key material, so it all lands under +# networks/, which is gitignored as a whole. +NETWORKS="$(dirname "$HERE")/networks" +CA_DIR="$NETWORKS/ca" +DAYS=825 + +mkdir -p "$CA_DIR" + +if [ ! -f "$CA_DIR/ca.crt" ]; then + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$CA_DIR/ca.key" -out "$CA_DIR/ca.crt" -days "$DAYS" \ + -subj "/CN=OAN federation local CA" 2>/dev/null + chmod 600 "$CA_DIR/ca.key" + echo "created the local CA at $CA_DIR/ca.crt" +fi + +for network in "$@"; do + dir="$NETWORKS/$network/tls" + mkdir -p "$dir" + + # The name must be in a subjectAltName: common name alone has not been + # accepted by anything for years, and a cert that fails only at verification + # time is worse than one that fails here. + openssl req -newkey rsa:2048 -nodes \ + -keyout "$dir/server.key" -out "$dir/server.csr" \ + -subj "/CN=$network" 2>/dev/null + + # The upstream names as well as the network's own. This network's mock + # providers are reached at https://., because the registry + # pins a provider's baseUrl to ^https:// just as it does a network's -- so the + # front door has to answer for them, and that means holding their names. + san="DNS:$network" + for upstream in mausamgram agmarknet vistaar pocra; do + san="$san,DNS:$upstream.$network" + done + + openssl x509 -req -in "$dir/server.csr" \ + -CA "$CA_DIR/ca.crt" -CAkey "$CA_DIR/ca.key" -CAcreateserial \ + -out "$dir/server.crt" -days "$DAYS" \ + -extfile <(printf 'subjectAltName=%s\nbasicConstraints=CA:FALSE\n' "$san") 2>/dev/null + + rm -f "$dir/server.csr" + chmod 600 "$dir/server.key" + + # A copy of the CA beside this network's own material, so the compose file + # mounts one directory rather than reaching across to a shared one. The + # adapter trusts it by path, which is what lets an outbound call verify + # another network's certificate with no code that knows about local CAs. + cp "$CA_DIR/ca.crt" "$dir/ca.crt" + echo "issued a certificate for $network" +done + +echo +echo "Other networks verify against $CA_DIR/ca.crt. Pass it with --ca-file." diff --git a/networkofnetwork/bin/network-env.py b/networkofnetwork/bin/network-env.py new file mode 100755 index 00000000..e3f7642e --- /dev/null +++ b/networkofnetwork/bin/network-env.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Write one network's .env, from quick-start/.env.example. + +Every network runs the SAME docker-compose.network.yml. What differs between +them is this file: the network's identity, the host ports it publishes, and +where its own state lives. Everything else -- images, the spec URL, the mapping +URLs -- is read from the example rather than restated here, so this does not +drift when the example gains a variable. + +Ports come from bin/network-ports.py, which allocates a block per network and +remembers it. Nothing here picks a port. + +Upstream capabilities point at this network's own mock servers. The federation +demo never calls them; they exist because bin/setup.py refuses to seed the +registry while any required value is still a placeholder. No real credential is +used, and none belongs in a file like this. + +Run: network-env.py +""" +import pathlib +import socket +import subprocess +import sys + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent +EXAMPLE = ROOT / ".env.example" +# Generated per-network state, in one place and gitignored as a whole -- .env, +# keys, TLS material and the rendered adapter config all carry secrets. +STATE = ROOT / "networks" + +# Every value whose placeholder would stop setup.py's preflight. None is secret. +# +# They must be https: the registry's Participant schema pins baseUrl to +# ^https://..., so an http:// value is refused at seeding time with a validation +# error naming the pattern. That is why these are hostnames under the network +# rather than the "http://mock-imd:9100" the containers actually answer on -- +# the registry record is metadata, and this demo never invokes these upstreams. +MOCK_CREDENTIALS = { + "MAUSAMGRAM_BASE_URL": "https://mausamgram.{network}", + # The path the MOCK serves, not IMD's real /nwpapi/get-daily. The front door + # answers this hostname and proxies to mock-imd, so the path has to be the + # mock's or the call 404s after everything else worked. + "MAUSAMGRAM_PATH": "/get-daily", + "MAUSAMGRAM_AUTH": "Basic bW9jazptb2Nr", + "AGMARKNET_BASE_URL": "https://agmarknet.{network}", + "AGMARKNET_TOKEN_URL": "https://agmarknet.{network}/token", + "AGMARKNET_ACCESS_NAME": "mock", + "AGMARKNET_PASSWORD": "mock", + "VISTAAR_BASE_URL": "https://vistaar.{network}", + "VISTAAR_TOKEN_URL": "https://vistaar.{network}/token", + "VISTAAR_CLIENT_ID": "mock", + "VISTAAR_CLIENT_SECRET": "mock", + "POCRA_BASE_URL": "https://pocra.{network}", +} + +# Upstream participant ids are per network: two networks seeding the same +# registry ids would be two different participants with one name the moment +# anything looked across them. +UPSTREAM_IDS = ("MAUSAMGRAM_PARTICIPANT_ID", "AGMARKNET_PARTICIPANT_ID", + "VISTAAR_PARTICIPANT_ID", "POCRA_PARTICIPANT_ID") + + +def read_example(): + """The example's keys in order, and their values.""" + if not EXAMPLE.exists(): + sys.exit(f"network-env: {EXAMPLE} is missing") + order, values = [], {} + for line in EXAMPLE.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, _, value = stripped.partition("=") + key = key.strip() + order.append(key) + values[key] = value.strip() + return order, values + + +def ports_for(network): + """This network's allocated block, as the variables it fills.""" + out = subprocess.run([sys.executable, str(HERE / "network-ports.py"), "ports", network], + capture_output=True, text=True, check=True).stdout + return dict(line.split("=", 1) for line in out.splitlines() if line) + + +def resolve(values, network): + """One network's values: the example, with what must differ overridden.""" + out = dict(values) + out.update({k: v.format(network=network) for k, v in MOCK_CREDENTIALS.items()}) + out.update(ports_for(network)) + + # The identity. ONE participant: the network IS the party, however many + # layers it serves, so there are no per-role subscriber ids. + out["BECKN_NETWORK_ID"] = network + out["DISCOVERY_SUBSCRIBER_ID"] = f"discovery.{network}" + + # The address another network reaches this one at, and what the descriptor + # publishes as services.discovery. + # + # It ends in /discover, and that is not decoration: the adapter takes the + # ACTION FROM THE URL -- the router strips the module's mount path and + # routes on what is left. A base url ending at /network arrives as an empty + # action and is refused with "endpoint '' is not supported". + # + # https and a hostname because the Participant schema refuses anything else. + # Inside the stack the modules reach each other by localhost, which their + # own config carries. + out["NETWORK_BASE_URL"] = f"https://{network}/network/discover" + + # Where this network's own state lives. Read by docker-compose.network.yml + # for its bind mounts, and by setup.py for keys and the rendered config. + out["NETWORK_DIR"] = str(STATE / network) + + # The crawl's own settings. Defaults rather than questions: a demo that + # asked for a page size before it would start would be asking about the one + # thing it is least likely to need changed. + # The UPPER bound on how often peers are re-crawled. A peer's own + # projectionTtl can shorten it, but never lengthen it -- and the interval is + # recomputed from the peers found on each pass, so a deployment that had NO + # peers when it started waits this long before looking again. 900 meant a + # network admitted after startup was not crawled for fifteen minutes. + out["FEDERATION_REFRESH_SECONDS"] = "30" + out["FEDERATION_PAGE_SIZE"] = "0" + out["FEDERATION_UPDATE_MODE"] = "MERGE" + # How long a peer may serve a copy of our catalogues. The crawler refreshes + # at projectionTtl/3 floored at 30s, so 30 means a peer re-crawls us every + # 30 seconds -- short enough to watch a catalog cross during a demo. + out["FEDERATION_PROJECTION_TTL"] = "30" + # What breadth of a peer's catalogue to mirror: all of it. + # + # The star does the work. Discovery will not take a bare selector -- it + # wants a test, not a projection -- and it refuses a test that narrows + # nothing, so exists(), like_regex, "starts with" and != are all out. But a + # star with ONE equality is fine, and isActive is the one field every + # catalog has. + # + # Deliberately not a list of the capabilities we speak: that would be a + # fifth place the capability set is written, and adding a sixth capability + # would silently stop it being mirrored. + # + # Broad here does not mean mirroring a peer's whole index: context.networkId + # and schemaContext still narrow what comes back. + out["FEDERATION_INTENT"] = '{"filters": {"type": "jsonpath", "expression": "$.catalogs[*] ? (@.isActive == true)"}}' + out["FEDERATION_PUBLISH_RETRY_SECONDS"] = "30" + + # No collector runs here, and discovery REFUSES to start with an exporter + # configured and nowhere to export to. The example turns telemetry on for a + # stack that has hyperdx; this one does not. + out["DISCOVERY_OTEL_EXPORTER"] = "none" + out["ADAPTER_OTEL_ENABLED"] = "false" + + # Built locally, because federation runs ahead of any published image. + out["ADAPTER_IMAGE"] = "oan/network-adapter:local" + + # One label, used where a value has to be short and unique per network. + short = network.split(".")[0] + for key in UPSTREAM_IDS: + if key in out: + out[key] = f"{out[key]}.{short}" + + return out + + +def warn_if_bound(network, resolved): + """Say which of this network's ports something else already holds. + + A warning, not an error: a port can be released between writing this file + and starting the stack, so refusing here would be refusing on stale + information. But finding out now beats finding out from a compose failure + that names the port and not the service. + """ + busy = [] + for name, port in sorted(ports_for(network).items()): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", int(port))) + except OSError: + busy.append(f" {port} {name}") + if busy: + print(f"network-env: these ports are already in use on this machine:") + print("\n".join(busy)) + print("network-env: free them, or `network destroy` whatever holds them.") + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: network-env.py ") + network = argv[1].strip() + if not network: + sys.exit("network-env: which network?") + + order, values = read_example() + resolved = resolve(values, network) + warn_if_bound(network, resolved) + + directory = STATE / network + directory.mkdir(parents=True, exist_ok=True) + path = directory / ".env" + # Keys the example does not have go last, in a stable order, so a + # regenerated file diffs cleanly against the one it replaces. + extra = [k for k in sorted(resolved) if k not in order] + path.write_text("\n".join(f"{k}={resolved[k]}" for k in list(order) + extra if k in resolved) + "\n") + path.chmod(0o600) + print(f"network-env: wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/network-ports.py b/networkofnetwork/bin/network-ports.py new file mode 100755 index 00000000..abb61ea5 --- /dev/null +++ b/networkofnetwork/bin/network-ports.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Give each network its own block of host ports, once, and remember it. + +Two networks on one machine cannot both publish 9200. Each gets a contiguous +block, and the block it got is RECORDED -- so bringing a network back up lands +on the ports it had, and `make network-ls` can say what is where. + +Recorded rather than derived from the name. Hashing a network id into a block +needs no file, but two names can hash to the same block and the only symptom is +a port already in use at start-up, on whichever network happened to come second. +A file costs one more thing to keep, and cannot collide. + +The file is the allocation, not a cache: deleting it and bringing a network back +up moves it to a different block, which is why `network destroy` clears its own +entry rather than the whole file. +""" +import json +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +STATE = HERE.parent / "networks" +ALLOCATIONS = STATE / "ports.json" + +# Where the blocks start, and how wide one is. 100 apart so a block's ports stay +# readable -- 9300, 9301, 9302 belongs to one network at a glance, which matters +# when two stacks are running and a log line names a port. +BASE = 9300 +WIDTH = 100 + +# What a block holds, in order. The names are the .env variables they fill. +PORTS = [ + "ADAPTER_PORT", + "DISCOVERY_PORT", + "REGISTRY_PORT", + "KEYCLOAK_PORT", + "MOCKIMD_PORT", + "MOCKAGMARKNET_PORT", + "FRONT_DOOR_PORT", +] + + +def load(): + if not ALLOCATIONS.exists(): + return {} + return json.loads(ALLOCATIONS.read_text()) + + +def save(allocations): + ALLOCATIONS.parent.mkdir(parents=True, exist_ok=True) + ALLOCATIONS.write_text(json.dumps(allocations, indent=2, sort_keys=True) + "\n") + + +def block_for(network): + """This network's block index, allocating the lowest free one if it has none.""" + allocations = load() + if network in allocations: + return allocations[network] + + taken = set(allocations.values()) + index = 0 + while index in taken: + index += 1 + allocations[network] = index + save(allocations) + return index + + +def ports_for(network): + """The network's ports, as the .env variables they fill.""" + start = BASE + block_for(network) * WIDTH + return {name: start + offset for offset, name in enumerate(PORTS)} + + +def release(network): + """Forget a network's block, so the next one may take it.""" + allocations = load() + if allocations.pop(network, None) is not None: + save(allocations) + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: network-ports.py [network]") + + command = argv[1] + if command == "list": + for network, index in sorted(load().items(), key=lambda kv: kv[1]): + start = BASE + index * WIDTH + print(f"{network:40} {start}-{start + len(PORTS) - 1}") + return 0 + + if len(argv) < 3: + sys.exit(f"usage: network-ports.py {command} ") + network = argv[2] + + if command == "ports": + for name, port in ports_for(network).items(): + print(f"{name}={port}") + return 0 + if command == "release": + release(network) + return 0 + sys.exit(f"unknown command: {command}") + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/network.sh b/networkofnetwork/bin/network.sh new file mode 100755 index 00000000..cb49cad1 --- /dev/null +++ b/networkofnetwork/bin/network.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# Stand up one whole OAN network, locally, by name. +# +# Run it again with a different name and you have two, side by side, able to +# reach each other -- which is the whole point. Everything that differs between +# them comes from that network's own .env, generated here, so this script is the +# one place that knows how to point the shared compose file at one of them: +# +# COMPOSE_PROJECT_NAME which project the containers belong to +# COMPOSE_ENV_FILES which .env compose itself reads +# QUICKSTART_ROOT where setup.py reads .env and writes keys and configs +# +# Usage: network.sh +# +# up certs, env, config, containers, registry seed, front door +# setup re-seed the registry and re-render configs only +# host-descriptor sign the two documents this network hosts at its own domain +# admit agree to deal with another network: admit +# down stop it, keep the data +# destroy stop it and delete the data, and release its ports +# teardown destroy EVERY deployed network (asks first) +# ps|logs what it is doing +# ls every network on this machine and the ports it holds +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(dirname "$HERE")" +STATE="$ROOT/networks" +COMPOSE="$ROOT/docker-compose.yml" + +# The bridge every network shares. Created once, outside any project, because a +# project-owned network would vanish with whichever project was torn down first. +FEDERATION_BRIDGE=oan-federation + +die() { echo "network: $*" >&2; exit 1; } + +need_network() { + [ -n "${1:-}" ] || die "which network? e.g. make network NETWORK_ID=bharatvistar.oan.local" +} + +# project turns a network id into a compose project name. Docker allows a narrow +# set of characters, and a network id is a domain -- so the dots go. +project() { echo "oan-${1//./-}"; } + +network_dir() { echo "$STATE/$1"; } + +compose() { + local network="$1"; shift + COMPOSE_PROJECT_NAME="$(project "$network")" \ + COMPOSE_ENV_FILES="$(network_dir "$network")/.env" \ + docker compose -f "$COMPOSE" "$@" +} + +ensure_bridge() { + docker network inspect "$FEDERATION_BRIDGE" >/dev/null 2>&1 && return + echo "network: creating the shared bridge $FEDERATION_BRIDGE" + docker network create "$FEDERATION_BRIDGE" >/dev/null +} + +cmd_up() { + local network="$1" + ensure_bridge + # Every directory the compose file bind-mounts, created BEFORE anything + # starts. Docker creates a missing bind-mount source as a ROOT-OWNED + # directory, and then nothing in this script can write the file that was + # supposed to be there -- including on every later run. + mkdir -p "$(network_dir "$network")"/{.well-known,config,tls,keys} + "$HERE/network-certs.sh" "$network" + python3 "$HERE/network-env.py" "$network" + # The core first, and the adapter deliberately NOT yet: its config does not + # exist until setup renders it, and docker creates a DIRECTORY at a bind-mount + # source that is missing. The adapter would then fail with "adapter.yaml is a + # directory" on every restart, for ever, with nothing saying why. + echo "network: starting $network" + compose "$network" up -d registry-postgres registry-keycloak registry-service \ + discovery-postgres discovery-service mock-imd mock-agmarknet + cmd_setup "$network" + echo + echo "network: $network is up" + python3 "$HERE/network-ports.py" ports "$network" | sed 's/^/ /' +} + +cmd_setup() { + local network="$1" + echo "network: seeding the registry for $network" + # Keys, the registry seed and the schema registry. It writes into the + # network's own directory, which is what QUICKSTART_ROOT selects. + QUICKSTART_ROOT="$(network_dir "$network")" python3 "$HERE/setup.py" + # The adapter config, which setup.py deliberately does not render: this + # deployment runs ONE adapter and the three-role templates do not exist here. + python3 "$HERE/render-adapter.py" "$network" + # Now the config exists, so the adapter can start. Recreated rather than + # started, because a container reads its config once and this is also how a + # config change is applied. + compose "$network" up -d --force-recreate adapter wellknown +} + +cmd_host_descriptor() { + local network="$1" + local dir; dir="$(network_dir "$network")" + local descriptor="$dir/.well-known/openagrinet" + + # The state lives with the checkout that brought the network up. Without + # this, the .env read below fails under set -e and make exits with no word. + [ -f "$dir/.env" ] || die "$network is not set up in $STATE -- run 'make network NETWORK_ID=$network' from this checkout first" + + # A revision must only ever INCREASE -- another network refuses a descriptor + # offering one at or below what it already holds, which is what stops a + # withdrawn declaration being replayed as current. So it is derived from the + # last one published rather than asked for: publishing twice without changing + # anything would otherwise be refused by everyone who already has it. + local revision=1 + if [ -f "$descriptor" ]; then + revision=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['revision'] + 1)" "$descriptor") + fi + + # How long another network may serve a copy of our catalogues, and -- the + # half that shows in a demo -- how often it must re-crawl to keep them. The + # crawler refreshes at projectionTtl/3, floored at 30s, so 30 here means a + # peer re-crawls us every 30 seconds. + # + # From the env so a demo can shorten it without editing anything. Absent + # leaves oanfed's own default of an hour. + local ttl_arg=() + local ttl; ttl="$(sed -n 's/^FEDERATION_PROJECTION_TTL=//p' "$dir/.env" 2>/dev/null)" + [ -n "$ttl" ] && ttl_arg=(--projection-ttl "$ttl") + + # The BASE registry url: oanfed appends /api/v1 to it itself. + # + # The governance key signs the descriptor and nothing else, so it is its own + # file -- generated on first publish -- and never the adapters' traffic key, + # which oanfed reads from the registry and delegates in the descriptor. + python3 "$HERE/oanfed.py" publish \ + --network-id "$network" \ + --registry-url "http://127.0.0.1:$(port_of "$network" REGISTRY_PORT)" \ + --governance-key "$dir/keys/governance.key" \ + --revision "$revision" \ + "${ttl_arg[@]}" \ + --out "$dir" +} + +port_of() { + python3 "$HERE/network-ports.py" ports "$1" | sed -n "s/^$2=//p" +} + +# admit records OUR decision to deal with another network, in OUR registry. +# +# network.sh admit +# +# --resolve is what makes a network id reachable from this machine: the id is a +# domain that only resolves on the shared bridge, so the request carries it as +# its Host and the connection goes to that network's published front-door port. +# The certificate is still checked against the id, so nothing is relaxed -- only +# where the connection goes. +cmd_admit() { + local us="$1" them="$2" + need_network "${them:-}" + local env_file; env_file="$(network_dir "$us")/.env" + local value; value() { sed -n "s/^$1=//p" "$env_file"; } + + python3 "$HERE/oanfed.py" admit \ + --network-id "$them" \ + --ca-file "$STATE/ca/ca.crt" \ + --resolve "127.0.0.1:$(port_of "$them" FRONT_DOOR_PORT)" \ + --registry-url "http://127.0.0.1:$(port_of "$us" REGISTRY_PORT)" \ + --keycloak-url "http://127.0.0.1:$(port_of "$us" KEYCLOAK_PORT)" \ + --keycloak-issuer "registry-keycloak:8080" \ + --realm "$(value KEYCLOAK_REALM)" \ + --client-id "$(value KEYCLOAK_CLIENT_ID)" \ + --username "$(value REGISTRY_ADMIN_USER)" \ + --password "$(value REGISTRY_ADMIN_PASSWORD)" +} + +cmd_down() { compose "$1" down; } +cmd_ps() { compose "$1" ps; } +cmd_logs() { shift; compose "$@" logs -f --tail=100; } + +# cmd_teardown destroys EVERY deployed network. +# +# A separate command rather than `destroy` with no argument: destroy already +# refuses an empty network id, and making that spelling mean "all of them" +# would turn the commonest typo into the most destructive outcome. +# +# It names what it is about to delete and waits, because the data it removes -- +# every registry, every published catalog, both crawler databases -- cannot be +# rebuilt from this directory. Set TEARDOWN_YES=1 to skip the prompt in a +# script. +cmd_teardown() { + local networks + networks="$(python3 "$HERE/network-ports.py" list | awk '{print $1}')" + + if [ -z "$networks" ]; then + echo "nothing deployed" + return 0 + fi + + echo "this will DESTROY, with their data and ports:" + echo "$networks" | sed 's/^/ /' + + if [ "${TEARDOWN_YES:-}" != "1" ]; then + printf "type yes to continue: " + local answer; read -r answer + [ "$answer" = "yes" ] || { echo "nothing done"; return 1; } + fi + + local network + for network in $networks; do + cmd_destroy "$network" + done +} + +cmd_destroy() { + local network="$1" + compose "$network" down -v + python3 "$HERE/network-ports.py" release "$network" + rm -rf "$(network_dir "$network")" + echo "network: $network destroyed, and its ports released" +} + +cmd_ls() { python3 "$HERE/network-ports.py" list; } + +main() { + local command="${1:-}" + case "$command" in + ls) cmd_ls ;; + up|setup|down|destroy|ps) + need_network "${2:-}"; "cmd_$command" "$2" ;; + # Its own line rather than joining the list above: that branch dispatches + # to "cmd_$command", and a hyphen cannot appear in a shell function name. + host-descriptor) + need_network "${2:-}"; cmd_host_descriptor "$2" ;; + teardown) cmd_teardown ;; + admit) need_network "${2:-}"; cmd_admit "$2" "${3:-}" ;; + logs) need_network "${2:-}"; compose "$2" logs -f --tail=100 "${3:-}" ;; + *) sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//' ;; + esac +} + +main "$@" diff --git a/networkofnetwork/bin/oanfed.py b/networkofnetwork/bin/oanfed.py new file mode 100644 index 00000000..53f88354 --- /dev/null +++ b/networkofnetwork/bin/oanfed.py @@ -0,0 +1,1087 @@ +#!/usr/bin/env python3 +"""oanfed -- the operator's tool for federation. + +A network publishes exactly two files, at locations derived from its own domain: + + https://{networkId}/.well-known/openagrinet the signed Network Descriptor + https://{networkId}/.well-known/jwks.json the keys that verify it + +These are publication points, not APIs. Serving a static file and generating the +same JSON are equally valid, and nothing here assumes which. + +Admitting a peer is a DECISION, not a request, so it is a command an operator +runs and not an endpoint the adapter exposes. An adapter endpoint would mean +anything able to reach the adapter could widen who this network deals with, +which is exactly the thing admission exists to control. + +It lives beside setup.py because it does the same job: both mint a Keycloak +token and write Participant records into this network's registry. setup.py seeds +the network's own participants; this admits another network as a peer. + + oanfed.py publish generate and sign this network's two published documents + oanfed.py verify fetch another network's documents and check them + oanfed.py admit record the decision to deal with a peer; also the refresh + oanfed.py suspend stop dealing with a peer + oanfed.py reinstate undo a suspension; a peer cannot do this for itself + +Run a command with -h for its flags. +""" + +import argparse, base64, hashlib, http.client, json, os, pathlib, re, ssl, sys, urllib.error, urllib.parse, urllib.request +from datetime import datetime, timezone + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives import serialization as ser + +# The two fixed paths. They are NOT configurable: a peer derives them from the +# networkId alone, which is the whole reason a network can be found knowing +# nothing but its domain. +WELL_KNOWN_DESCRIPTOR = "/.well-known/openagrinet" +WELL_KNOWN_JWKS = "/.well-known/jwks.json" + +# Caps what will be read from a peer. These documents are small and a peer is +# not yet trusted -- it has not been admitted at the point they are fetched. +# Without a cap, the first thing an unknown host can do to us is stream until we +# run out of memory. +MAX_DOCUMENT_BYTES = 1 << 20 + +# The only signature algorithm this implementation signs or verifies with, by +# its fully-specified JOSE name (RFC 9864, which deprecates the polymorphic +# "EdDSA"). The contract leaves the choice open; narrowing it here means an +# unexpected value is refused rather than quietly treated as this one. +ALGORITHM = "Ed25519" + +# The Network Descriptor schema version this implementation writes and reads. A +# version it does not know is refused rather than half-understood. +DESCRIPTOR_VERSION = "1.0" +DESCRIPTOR_STATUSES = ("Active", "Suspended", "Retired") + +# How far ahead of our clock a peer's issuedAt may be before it is refused. A +# document from the future is a clock fault or a pre-signed replay waiting to +# happen; a few minutes absorbs ordinary skew. +MAX_CLOCK_SKEW_SECONDS = 300 + +ROLE_NETWORK = "network" + + +def as_role_list(role): + """A participant's role, however it was written. + + A list is what the schema now wants; a plain string is what older records + carry, and nothing rewrites a record just to change its shape. + """ + if role is None: + return [] + return list(role) if isinstance(role, list) else [role] + + +def has_role(record, role): + return any(str(r).lower() == role.lower() for r in as_role_list(record.get("role"))) +STATUS_ACTIVE, STATUS_INACTIVE = "active", "inactive" + + +class FederationError(Exception): + """Anything a caller is expected to act on rather than crash over.""" + + +# --- canonical form ------------------------------------------------------- + +# JCS serialises numbers the way ECMAScript does, which is exact for integers +# only up to 2**53. Beyond it two implementations can disagree on the digits. +MAX_SAFE_INTEGER = 2 ** 53 - 1 + + +def canonical(descriptor): + """The exact bytes that get signed: RFC 8785 (JCS). + + The signature is excluded, because it cannot cover itself. Everything else + is included: narrowing the covered set would let an unsigned field be + changed in flight, and the fields most worth tampering with -- status, + revision, the discovery URL, which keys may sign traffic -- are exactly the + ones a smaller set would be tempted to omit. + + JCS, and not "whatever this language's JSON encoder happens to emit", + because the point of a canonical form is that a DIFFERENT implementation + rebuilds the same bytes. + """ + without_signature = {k: v for k, v in descriptor.items() if k != "signature"} + return _jcs(without_signature).encode("utf-8") + + +def _jcs(value): + """One JCS value. Floats are refused rather than formatted. + + A descriptor has no use for a fraction, and number formatting is the one + part of JCS where implementations genuinely diverge -- so the safe rule is + that a non-integer never reaches a signature. + """ + if value is None or isinstance(value, bool): + return json.dumps(value) + if isinstance(value, int): + if abs(value) > MAX_SAFE_INTEGER: + raise FederationError(f"integer {value} is outside the range JCS signs exactly") + return str(value) + if isinstance(value, float): + raise FederationError(f"non-integer number {value!r} cannot be signed; use an integer") + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError as err: + raise FederationError("string is not valid Unicode (lone surrogate)") from err + # Python's escaping is JCS's: \b \f \n \r \t \" \\ as short forms, other + # controls as lowercase \u00xx, everything else (including '/') raw. + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list): + return "[" + ",".join(_jcs(v) for v in value) + "]" + if isinstance(value, dict): + # JCS sorts by UTF-16 code units, not code points. The two differ only + # above U+FFFF, which is exactly where a naive sort_keys goes wrong. + keys = sorted(value, key=lambda k: k.encode("utf-16-be")) + return "{" + ",".join(f"{_jcs(k)}:{_jcs(value[k])}" for k in keys) + "}" + raise FederationError(f"{type(value).__name__} cannot be signed") + + +# --- keys ----------------------------------------------------------------- + +def b64url(raw): + """base64url WITHOUT padding, which is what RFC 7517 says a JWKS carries.""" + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def unb64url(encoded): + if not isinstance(encoded, str) or not re.fullmatch(r"[A-Za-z0-9_-]*", encoded): + raise FederationError("value is not base64url") + return base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + + +def registry_key(jwk_x): + """A JWKS key in the form the registry stores. + + A JWKS carries base64url without padding. The registry's PublicKey schema + pins ^[A-Za-z0-9+/]{43}=$, which is standard base64 WITH padding. Both are + correct for their own format, so something has to convert -- and it is one + function rather than an inline re-encode at each call site, because getting + it wrong produces a key that is silently the wrong bytes rather than an + error. + """ + return base64.b64encode(unb64url(jwk_x)).decode() + + +def jwk(public): + """The JWK for a raw Ed25519 public key. + + The kid is the key's RFC 7638 thumbprint: derived from the key itself, so it + never needs configuring, can never point at the wrong key, and changes on + its own when a key is rotated -- which is what lets old and new overlap in + one JWKS without colliding. + """ + x = b64url(public) + members = {"crv": "Ed25519", "kty": "OKP", "x": x} + thumbprint = hashlib.sha256(_jcs(members).encode()).digest() + return {"kty": "OKP", "crv": "Ed25519", "alg": ALGORITHM, "use": "sig", + "kid": b64url(thumbprint), "x": x} + + +def load_or_create_key(path): + """Read the governance private key, generating one if the file is absent. + + Stored as the raw 32-byte seed, base64url-unpadded -- the same encoding the + public half uses in the JWKS, so an operator reading either file sees one + format rather than two. + + A key file anyone but its owner can read is refused, not used. This key + decides what the network says about itself, including which keys may sign + as it; a copy of it is the network. + """ + path = pathlib.Path(path) + if path.exists(): + if path.stat().st_mode & 0o077: + raise FederationError( + f"{path} is readable by group or others; chmod 600 it before signing with it") + raw = unb64url(path.read_text().strip()) + # 64 is Go's ed25519.PrivateKey -- the seed followed by the public half. + if len(raw) == 64: + raw = raw[:32] + if len(raw) != 32: + raise FederationError( + f"{path} holds {len(raw)} bytes; an Ed25519 private key is 32 (a seed) " + f"or 64 (a seed followed by its public half)") + return Ed25519PrivateKey.from_private_bytes(raw) + + private = Ed25519PrivateKey.generate() + raw = private.private_bytes( + encoding=ser.Encoding.Raw, + format=ser.PrivateFormat.Raw, + encryption_algorithm=ser.NoEncryption(), + ) + path.parent.mkdir(parents=True, exist_ok=True) + # Created 0600 rather than chmod-ed after: no window where it is readable. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as f: + f.write(b64url(raw) + "\n") + return private + + +def public_of(private): + return private.public_key().public_bytes( + encoding=ser.Encoding.Raw, format=ser.PublicFormat.Raw + ) + + +# --- the published documents: shape --------------------------------------- + +def _is_https_url(value): + if not isinstance(value, str): + return False + parsed = urllib.parse.urlsplit(value) + return parsed.scheme == "https" and bool(parsed.hostname) + + +def _parse_time(value): + if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", value): + raise FederationError(f"issuedAt {value!r} is not a UTC timestamp like 2026-09-17T10:00:00Z") + try: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + except ValueError as err: + raise FederationError(f"issuedAt {value!r} is not a real date") from err + + +def _nonempty_str(value): + return isinstance(value, str) and value.strip() != "" + + +def validate_keyset(keyset): + """A JWKS, checked; returns {kid: raw public key} for its Ed25519 keys. + + Keys of other types are allowed and skipped -- a network may publish more + than this implementation verifies with. A duplicate kid is refused outright: + with two keys under one name, "the key the signature names" means nothing. + """ + if not isinstance(keyset, dict) or not isinstance(keyset.get("keys"), list): + raise FederationError("JWKS is not an object with a keys array") + found, seen = {}, set() + for entry in keyset["keys"]: + if not isinstance(entry, dict) or not _nonempty_str(entry.get("kid")): + raise FederationError("JWKS holds a key without a kid") + kid = entry["kid"] + if kid in seen: + raise FederationError(f"JWKS holds two keys with kid {kid!r}") + seen.add(kid) + if entry.get("kty") != "OKP" or entry.get("crv") != "Ed25519": + continue + if entry.get("alg", ALGORITHM) != ALGORITHM: + raise FederationError(f"key {kid!r} declares alg {entry.get('alg')!r}") + if entry.get("use", "sig") != "sig": + raise FederationError(f"key {kid!r} is not a signing key") + raw = unb64url(entry.get("x", "")) + if len(raw) != 32: + raise FederationError(f"key {kid!r} is {len(raw)} bytes, want 32") + found[kid] = raw + return found + + +def validate_descriptor(descriptor, now=None): + """The Network Descriptor contract, field by field. + + Run before signing, so this network never publishes something a peer must + refuse, and after fetching, so a peer's document is checked against the + same rules rather than a looser reading of them. + """ + if not isinstance(descriptor, dict): + raise FederationError("descriptor is not a JSON object") + + def need(ok, message): + if not ok: + raise FederationError(message) + + need(descriptor.get("version") == DESCRIPTOR_VERSION, + f"descriptor version {descriptor.get('version')!r} is not {DESCRIPTOR_VERSION!r}") + network_id = descriptor.get("networkId") + need(isinstance(network_id, str) + and re.fullmatch(r"(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}", + network_id) is not None, + f"networkId {network_id!r} is not a lowercase DNS domain") + need(_nonempty_str(descriptor.get("name")), "name is missing") + operator = descriptor.get("operator") + need(isinstance(operator, dict) and _nonempty_str(operator.get("name")), + "operator.name is missing") + need("operatorId" not in operator or _nonempty_str(operator["operatorId"]), + "operator.operatorId is empty") + revision = descriptor.get("revision") + need(isinstance(revision, int) and not isinstance(revision, bool) + and 1 <= revision <= MAX_SAFE_INTEGER, + f"revision {revision!r} is not a positive integer") + need(descriptor.get("status") in DESCRIPTOR_STATUSES, + f"status {descriptor.get('status')!r} is not one of {', '.join(DESCRIPTOR_STATUSES)}") + issued = _parse_time(descriptor.get("issuedAt")) + now = now or datetime.now(timezone.utc) + need((issued - now).total_seconds() <= MAX_CLOCK_SKEW_SECONDS, + f"issuedAt {descriptor['issuedAt']} is in the future") + profile = descriptor.get("operatingProfile") + need(isinstance(profile, list) and profile and all(_nonempty_str(p) for p in profile), + "operatingProfile is not a non-empty list of versions") + packs = descriptor.get("schemaPacks") + need(isinstance(packs, list) and all(_is_https_url(p) for p in packs) + and len(set(packs)) == len(packs), + "schemaPacks is not a list of distinct https URLs") + services = descriptor.get("services") + need(isinstance(services, dict) and _is_https_url(services.get("discovery")), + "services.discovery is not an https URL") + contacts = descriptor.get("contacts") + need(contacts is None or (isinstance(contacts, dict) + and all(_nonempty_str(v) for v in contacts.values())), + "contacts is not an object of strings") + ttl = descriptor.get("projectionTtl") + need(ttl is None or (isinstance(ttl, int) and not isinstance(ttl, bool) and ttl >= 0), + f"projectionTtl {ttl!r} is not a non-negative integer") + delegated = descriptor.get("operationalKeys") + need(isinstance(delegated, list) and delegated + and all(_nonempty_str(k) for k in delegated) and len(set(delegated)) == len(delegated), + "operationalKeys is not a non-empty list of distinct kids") + + +# --- sign and verify ------------------------------------------------------ + +class StaleRevision(FederationError): + """The descriptor does not advance the revision. + + Its own type because admit DOUBLES AS THE REFRESH JOB: when it runs on a + schedule, finding nothing new is the normal outcome, not a failure. Treating + it as an error would mean a cron that alerts on every quiet run -- and an + operator who stops reading the alerts. + + A LOWER revision is a replay and a real refusal; the same revision is just + nothing to do. Both arrive here, and the caller tells them apart by + comparing the numbers it already holds. + """ + + +def sign(descriptor, private): + """Return a copy of the descriptor carrying the governance signature. + + The governance key signs this and nothing else. It is named by its own + thumbprint, so the signature cannot point at a different key than the one + that made it. + """ + signed = {k: v for k, v in descriptor.items() if k != "signature"} + signed["signature"] = { + "keyId": jwk(public_of(private))["kid"], + "algorithm": ALGORITHM, + "value": b64url(private.sign(canonical(signed))), + } + return signed + + +def verify(descriptor, keyset, now=None): + """Check a descriptor, its shape, and its signature against the JWKS. + + It answers ONE question -- did the holder of a published governance key + write this document -- and deliberately not whether the network should be + trusted. The keys came from the same domain as the document, so a network + can always verify its own lies. Admission is what decides trust. + + The key that signed the descriptor must not also be one it delegates for + traffic: the contract says the governance key never signs normal traffic, + and this is the one place that can be checked. + """ + validate_descriptor(descriptor, now) + keys = validate_keyset(keyset) + + signature = descriptor.get("signature") + if not isinstance(signature, dict): + raise FederationError("descriptor carries no signature") + if signature.get("algorithm") != ALGORITHM: + raise FederationError(f"unsupported signature algorithm: {signature.get('algorithm')!r}") + key_id = signature.get("keyId") + if key_id not in keys: + raise FederationError(f"no published Ed25519 key matches the signature's keyId {key_id!r}") + delegated = descriptor["operationalKeys"] + if key_id in delegated: + raise FederationError( + f"governance key {key_id!r} is also delegated for traffic; the two must differ") + missing = [k for k in delegated if k not in keys] + if missing: + raise FederationError(f"operationalKeys not in the JWKS: {', '.join(missing)}") + + try: + Ed25519PublicKey.from_public_bytes(keys[key_id]).verify( + unb64url(signature.get("value", "")), canonical(descriptor) + ) + except Exception as err: + raise FederationError("signature does not verify") from err + +# --- fetching a peer's documents ------------------------------------------ + +class ResolvingHTTPSConnection(http.client.HTTPSConnection): + """Connects to a given address while still verifying the peer's own name. + + The way `curl --resolve` works. The request still carries the networkId as + its Host and the certificate is still checked against it, so nothing about + verification is relaxed -- only where the connection goes. It exists because + a network id is a DNS domain, and an operator's own machine usually cannot + resolve one belonging to a stack running in containers. + """ + + def __init__(self, server_name, address, **kwargs): + host, _, port = address.partition(":") + super().__init__(host, int(port or 443), **kwargs) + self._server_name = server_name + + def connect(self): + super(http.client.HTTPSConnection, self).connect() + self.sock = self._context.wrap_socket(self.sock, server_hostname=self._server_name) + + +class Fetcher: + """Retrieves and verifies another network's published documents. + + scheme is "https" in any real deployment. It is settable because a local POC + has no certificate authority and cannot prove domain ownership -- which is + precisely the property a real deployment relies on, so this is a knob that + must never be turned in one. + """ + + def __init__(self, scheme="https", ca_file=None, resolve=None, timeout=10): + self.scheme, self.resolve, self.timeout = scheme or "https", resolve, timeout + # Real certificate verification against a DIFFERENT root, not skipping + # it: a peer presenting a certificate this CA did not sign, or one whose + # name does not match, is still refused. A local stack has no publicly + # trusted certificate, and the alternative -- turning verification off -- + # would quietly remove the check in the one place a POC is meant to + # prove it. + self.context = ssl.create_default_context(cafile=ca_file) if ca_file else None + + def fetch(self, network_id): + """Retrieve a network's descriptor and keys, and verify one against the other. + + A verified descriptor is a CANDIDATE, not a trusted peer. Both documents + came from the same host, so this proves self-consistency and nothing + more -- any network can sign its own claims. Admission is the separate, + local decision that turns a candidate into a peer. + """ + if not network_id: + raise FederationError("no networkId given") + + descriptor = self._get(network_id, WELL_KNOWN_DESCRIPTOR) + keyset = self._get(network_id, WELL_KNOWN_JWKS) + + # The descriptor says who it is for. A document served at one domain + # that names another is either a misconfiguration or an attempt to be + # admitted under someone else's name, and neither should get further. + if descriptor.get("networkId") != network_id: + raise FederationError( + f"{network_id} publishes a descriptor for {descriptor.get('networkId')!r}") + verify(descriptor, keyset) + return descriptor, keyset + + def _get(self, network_id, path): + where = f"{self.scheme}://{network_id}{path}" + try: + if self.scheme == "https": + connection = ( + ResolvingHTTPSConnection(network_id, self.resolve, + context=self.context, timeout=self.timeout) + if self.resolve else + http.client.HTTPSConnection(network_id, context=self.context, + timeout=self.timeout) + ) + else: + host = self.resolve or network_id + connection = http.client.HTTPConnection(host, timeout=self.timeout) + + connection.request("GET", path, headers={"Accept": "application/json", + "Host": network_id}) + response = connection.getresponse() + if response.status != 200: + raise FederationError(f"{where} returned {response.status} {response.reason}") + body = response.read(MAX_DOCUMENT_BYTES) + except FederationError: + raise + except Exception as err: + raise FederationError(f"fetching {where}: {err}") from err + finally: + try: + connection.close() + except Exception: + pass + + try: + return json.loads(body) + except json.JSONDecodeError as err: + raise FederationError(f"parsing {where}: {err}") from err + + +# --- admission ------------------------------------------------------------ + +DEFAULT_PROJECTION_TTL = 3600 # seconds; finite, and deliberately short + + +def projection_ttl(descriptor): + """What a descriptor permits, in seconds. + + The three cases are genuinely different and none may be folded into + another: absent is consent to decide for ourselves, zero is a refusal to be + cached, and a positive value is a bounded licence. + """ + declared = descriptor.get("projectionTtl") + if declared is None: + return DEFAULT_PROJECTION_TTL + if declared < 0: + raise FederationError(f"projectionTtl is negative: {declared}") + if declared == 0: + raise FederationError( + "peer declares projectionTtl 0 (live query only), which is not implemented") + return declared + + +def operational_keys(descriptor, keyset): + """The keys a peer delegated for traffic, and only those. + + Its governance key is published in the same JWKS but deliberately left out: + whatever is recorded here is what the peer's requests are verified against, + and the governance key must never pass that check. Which keys are delegated + comes from the SIGNED descriptor, so a JWKS edit alone cannot add one. + """ + delegated = set(descriptor.get("operationalKeys") or []) + return [k for k in keyset.get("keys", []) if k.get("kid") in delegated] + + +# What an entry is in relation to us. +# +# self this deployment's own entry +# source a peer we consume from -- our crawler asks it for its catalogs +# requester a peer that consumes from us -- it calls us and we verify its +# signatures, but we do not crawl it +# both a peer in both directions, which is what mutual admission produces +FEDERATION_ROLES = ("self", "source", "requester", "both") +DEFAULT_FEDERATION_ROLE = "both" + + +def admit_record(current, fresh, keyset, federation_role=DEFAULT_FEDERATION_ROLE): + """Turn a verified descriptor into the registry record to write. + + current is None on first admission; on every later call it is what the + registry already holds, which is what makes the revision check possible. + + Two rules, and each exists because of a specific way a peer could otherwise + take something back or take something over: + + - The revision must increase, so a superseded descriptor cannot be + replayed. That, plus the documents being served from the network's own + domain, is what the contract rests identity on. + - Status is carried over and never read from the descriptor, so a peer an + operator suspended cannot reinstate itself by publishing a new file. + What the peer says about itself is a claim; what we recorded is a + decision. + """ + network_id = fresh.get("networkId") + if not network_id: + raise FederationError("descriptor carries no networkId") + + ttl = projection_ttl(fresh) + status = STATUS_ACTIVE + if current is not None: + if (current.get("participantId") or "").lower() != network_id.lower(): + raise FederationError( + f"descriptor is for {network_id!r}, stored peer is " + f"{current.get('participantId')!r}") + held = current.get("revision") or 0 + if fresh.get("revision", 0) <= held: + raise StaleRevision( + f"descriptor revision did not increase: hold {held}, " + f"offered {fresh.get('revision', 0)}") + status = current.get("status", STATUS_ACTIVE) + + admitted_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return { + "participantId": network_id, + "name": fresh.get("name", ""), + # A peer is a node, not an upstream: it signs, and its signatures are + # verified against the keys recorded here. + "type": "node", + "status": status, + "baseUrl": (fresh.get("services") or {}).get("discovery", ""), + # A LIST, because role says what a participant serves and a network can + # serve several layers. We only know this peer as a network -- what it + # runs behind that is its own business and not in its descriptor. + "role": [ROLE_NETWORK], + "keys": [ + { + "alg": "ed25519", + "key": registry_key(k["x"]), + # validFrom is required, and is recorded as the moment we + # admitted the peer -- the only moment we can actually attest + # to. The peer's published documents carry no validity window, + # and inventing one would be recording a claim nobody made. + "validFrom": admitted_at, + "status": "active", + } + for k in operational_keys(fresh, keyset) + ], + "revision": fresh.get("revision", 0), + "projectionTtl": ttl, + # Stored VERBATIM, not interpreted. What a peer can speak is its + # statement; rewriting it here would make our copy disagree with the + # document we read it from. The crawler derives the context URL it + # filters on from these at crawl time. + "schemaPacks": list(fresh.get("schemaPacks") or []), + # Carried over on a refresh. A later descriptor says nothing about the + # relationship -- that is OUR decision about a peer, not a claim it + # makes about itself -- so refreshing must not quietly widen it. + "federationRole": (current or {}).get("federationRole") or federation_role, + } + + +# --- this network's own registry ------------------------------------------ + +class ReadOnlyRegistry: + """What publish needs: searches, and no credentials. + + Separate from Registry because the privilege is different. Writing a + participant is an operator's decision and needs a Keycloak token; reading + one back to describe ourselves needs nothing, because the search endpoint is + public. Sharing one class would mean publish asking for a password it has no + use for. + """ + + def __init__(self, url, timeout=30): + self.url, self.timeout = url.rstrip("/"), timeout + + def _search(self, entity, filters): + request = urllib.request.Request( + f"{self.url}/api/v1/{entity}/search", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + found = json.load(response) + except urllib.error.HTTPError as err: + raise FederationError( + f"registry search {entity} returned {err.code}: " + f"{err.read().decode()[:200]}") from err + except Exception as err: + raise FederationError(f"reaching the registry at {self.url}: {err}") from err + return found.get("data", found) if isinstance(found, dict) else found + + def participant(self, participant_id): + rows = self._search("Participant", {"participantId": {"eq": participant_id}}) + return rows[0] if rows else None + + def schema_packs(self): + """The schema pack URLs this deployment records itself as speaking. + + Active only: a retired capability should stop being advertised without + anyone having to remember to edit a descriptor. + """ + rows = self._search("SchemaRegistry", {"status": {"eq": "active"}}) + return sorted({r["schemaUrl"] for r in rows if r.get("schemaUrl")}) + + +class Registry: + """The SunbirdRC registry this network runs. + + The same API surface setup.py uses: a Keycloak token, then POST to create + and PUT to update. Kept as its own class only so the token is minted once + per command rather than per call. + """ + + def __init__(self, args): + self.url = args.registry_url.rstrip("/") + self.keycloak = args.keycloak_url.rstrip("/") + self.issuer, self.realm = args.keycloak_issuer, args.realm + self.client_id, self.username, self.password = args.client_id, args.username, args.password + self.timeout = args.registry_timeout + self._bearer = None + + def bearer(self): + if self._bearer: + return self._bearer + body = urllib.parse.urlencode({ + "grant_type": "password", "client_id": self.client_id, + "username": self.username, "password": self.password, + }).encode() + # Keycloak sits behind PROXY_ADDRESS_FORWARDING, so it builds the + # token's issuer from the Host header. The registry validates that + # issuer against its own configured address -- the CONTAINER-INTERNAL + # one -- so a token minted without this header is rejected with a 401 + # that says nothing useful. + endpoint = (f"{self.keycloak}/auth/realms/{self.realm}" + f"/protocol/openid-connect/token") + request = urllib.request.Request( + endpoint, data=body, + headers={"X-Forwarded-Host": self.issuer, "X-Forwarded-Proto": "http", + "Content-Type": "application/x-www-form-urlencoded"}) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = json.load(response) + except urllib.error.HTTPError as err: + raise FederationError( + f"keycloak {endpoint} returned {err.code}: {err.read().decode()[:200]}") from err + except Exception as err: + raise FederationError(f"reaching keycloak at {endpoint}: {err}") from err + if not payload.get("access_token"): + # Keycloak answers 200 with an empty body when the forwarded host + # does not match what the registry expects, so a missing token is + # its own case rather than an HTTP error. + raise FederationError( + "keycloak issued no token -- check the credentials and --keycloak-issuer") + self._bearer = payload["access_token"] + return self._bearer + + def _call(self, method, path, body=None, authenticated=True): + headers = {"Content-Type": "application/json"} + if authenticated: + headers["Authorization"] = f"Bearer {self.bearer()}" + request = urllib.request.Request( + f"{self.url}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers=headers) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + raw = response.read() + except urllib.error.HTTPError as err: + raise FederationError( + f"registry {method} {path} returned {err.code}: {err.read().decode()[:300]}" + ) from err + return json.loads(raw) if raw else None + + def find(self, participant_id): + """The stored record for a participant, or None. + + Only participantId is filtered on: it is the schema's uniqueIndexFields, + so the registry already guarantees at most one match. + """ + found = self._call("POST", "/api/v1/Participant/search", + {"filters": {"participantId": {"eq": participant_id}}}, + authenticated=False) + records = found.get("data", found) if isinstance(found, dict) else found + return records[0] if records else None + + def write(self, record, existing): + """Create a record, or update the one already there.""" + if existing and existing.get("osid"): + return self._call("PUT", f"/api/v1/Participant/{existing['osid']}", record) + return self._call("POST", "/api/v1/Participant", record) + + def set_status(self, participant_id, status): + """Change only the status of an existing record. + + Nothing is fetched from the peer. Suspension is a decision about a + network we have stopped trusting, and asking that network's own server + to take part in it would be absurd. + """ + existing = self.find(participant_id) + if not existing: + raise FederationError(f"{participant_id} is not in this registry") + if not has_role(existing, ROLE_NETWORK): + raise FederationError(f"{participant_id} is not a peer network") + updated = {k: v for k, v in existing.items() if not k.startswith("os")} + updated["status"] = status + # Normalised on the way out. A record written before role became a list + # carries a plain string, and sending it back unchanged is refused -- + # the schema now wants an array. Reading either shape and writing one is + # what lets both sit in the registry at once. + updated["role"] = as_role_list(existing.get("role")) + self.write(updated, existing) + return updated + + +# --- commands ------------------------------------------------------------- + +def registered_signing_keys(entry, now=None): + """The entry's Ed25519 keys that are usable now or about to be. + + These are the operational keys: the adapters sign traffic with them and the + registry already holds them, plural, for rotation overlap. A key not yet + valid IS included -- a peer has to know a key before it starts signing with + it. A revoked or expired one is not: publishing it would delegate a key this + network has already withdrawn. + """ + now = now or datetime.now(timezone.utc) + usable = [] + for k in entry.get("keys") or []: + if str(k.get("alg", "")).lower() != "ed25519" or k.get("status", "active") != "active": + continue + until = k.get("validUntil") + if until and datetime.fromisoformat(until.replace("Z", "+00:00")) <= now: + continue + raw = base64.b64decode(str(k.get("key", "")).removeprefix("base64:")) + if len(raw) != 32: + raise FederationError(f"registry key {k.get('osid', '?')} is not a 32-byte Ed25519 key") + usable.append(raw) + return usable + + +def last_published_revision(path): + """The revision already on disk, or 0. Unreadable is an error, not 0. + + Treating a corrupt file as "nothing published" would let a revision go + backwards, and every peer holding the higher one would refuse ours. + """ + if not path.exists(): + return 0 + try: + return int(json.loads(path.read_text())["revision"]) + except Exception as err: + raise FederationError(f"cannot read the revision in {path}: {err}") from err + + +def write_atomically(path, document): + """Replace a published file in one step, so a reader never sees half of it.""" + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n") + os.replace(tmp, path) + + +def cmd_publish(args): + """Generate and sign this network's two published documents FROM THE REGISTRY. + + The registry entry is the source of truth, not these flags. What a network + says about itself in public is then the same thing it holds about itself + internally, by construction -- rather than two descriptions of one network + that happen to agree on the day someone typed them. + + Two kinds of key, as the contract requires: + + governance signs the descriptor and nothing else. Its private half is + --governance-key; it is NOT in the registry, so no peer can + verify a request signed with it. + operational sign traffic. Read from the registry entry; the descriptor + delegates them by kid in operationalKeys, under the + governance signature. + + Reading the registry needs no credentials: its search endpoint is public. + """ + out = pathlib.Path(args.out) / ".well-known" + descriptor_path, jwks_path = out / "openagrinet", out / "jwks.json" + + held = last_published_revision(descriptor_path) + if args.revision <= held: + raise FederationError( + f"revision {args.revision} does not increase on the published {held}; " + f"every peer holding {held} would refuse it") + + registry = ReadOnlyRegistry(args.registry_url, args.registry_timeout) + entry = registry.participant(args.network_id) + if entry is None: + raise FederationError( + f"{args.network_id} is not in this registry -- seed it first, " + f"then publish from it") + + operational = registered_signing_keys(entry) + if not operational: + raise FederationError( + f"{args.network_id} has no active Ed25519 key in the registry to delegate") + + governance = load_or_create_key(args.governance_key) + if public_of(governance) in operational: + # The one configuration that silently breaks the contract: the key that + # vouches for the network would also be accepted on its traffic. + raise FederationError( + f"{args.governance_key} is also a registered traffic key; the governance " + f"key must be separate -- use a new file") + + governance_jwk = jwk(public_of(governance)) + operational_jwks = [jwk(raw) for raw in operational] + + descriptor = { + "version": DESCRIPTOR_VERSION, + "networkId": entry["participantId"], + "name": entry.get("name") or entry["participantId"], + "operator": {"name": args.operator or entry.get("name") or entry["participantId"]}, + "revision": args.revision, + "status": args.status, + "issuedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "operatingProfile": [p.strip() for p in args.operating_profile.split(",") if p.strip()], + # From the schema registry, which is where this deployment records what + # it can speak. Typing them again here would let the two disagree. + "schemaPacks": registry.schema_packs(), + # The entry's baseUrl IS the federated discovery endpoint -- the same + # field admission reads back out of a peer's descriptor, so what we + # publish and what a peer stores about us are one string. + "services": {"discovery": entry.get("baseUrl", "")}, + # The delegation: which JWKS keys may sign traffic as this network. + "operationalKeys": [k["kid"] for k in operational_jwks], + } + if args.projection_ttl is not None: + descriptor["projectionTtl"] = args.projection_ttl + + signed = sign(descriptor, governance) + keyset = {"keys": [governance_jwk, *operational_jwks]} + # Checked exactly as a peer will check it, before anything is written: a + # document this network would itself refuse never goes out. + verify(signed, keyset) + + # Keys first. A peer fetching between the two writes then finds the old + # descriptor with a JWKS that still verifies it -- never a descriptor naming + # a key that is not published yet. + out.mkdir(parents=True, exist_ok=True) + write_atomically(jwks_path, keyset) + write_atomically(descriptor_path, signed) + + print(f"{args.network_id} now hosts its descriptor at revision {args.revision}") + print(f" descriptor {descriptor_path}") + print(f" keys {jwks_path}") + print(f" governance {governance_jwk['kid']}") + print(f" operational {', '.join(descriptor['operationalKeys'])}") + return 0 + + +def cmd_verify(args): + """Fetch another network's documents and check them.""" + descriptor, keyset = fetcher_from(args).fetch(args.network_id) + print(f"verified {descriptor['networkId']} at revision {descriptor.get('revision')}, " + f"status {descriptor.get('status')}") + print(f" discovery {(descriptor.get('services') or {}).get('discovery')}") + print(f" keys {len(keyset.get('keys', []))} published") + print(" this proves the documents are self-consistent, NOT that the network " + "should be trusted -- admission is that decision") + return 0 + + +def cmd_admit(args): + """Record the decision to deal with a peer. Also the refresh job.""" + descriptor, keyset = fetcher_from(args).fetch(args.network_id) + + registry = Registry(args) + existing = registry.find(args.network_id) + try: + record = admit_record(existing, descriptor, keyset, args.federation_role) + except StaleRevision: + # Nothing written, and not a failure: this command is also the refresh + # job, and a refresh that finds nothing new is its normal outcome. + print(f"{args.network_id} is unchanged at revision " + f"{existing.get('revision')}; nothing written") + return 0 + registry.write(record, existing) + + verb = "refreshed" if existing else "admitted" + print(f"{verb} {record['participantId']} at revision {record['revision']}, " + f"status {record['status']}") + print(f" discovery {record['baseUrl']}") + print(f" projectionTtl {record['projectionTtl']}s") + print(f" keys {len(record['keys'])} published") + print(f" relationship {record['federationRole']}") + return 0 + + +def cmd_suspend(args): + """Stop dealing with a peer.""" + Registry(args).set_status(args.network_id, STATUS_INACTIVE) + print(f"suspended {args.network_id}; it is no longer crawled and no longer answered") + return 0 + + +def cmd_reinstate(args): + """Undo a suspension. A peer cannot do this for itself.""" + Registry(args).set_status(args.network_id, STATUS_ACTIVE) + print(f"reinstated {args.network_id}") + return 0 + + +def fetcher_from(args): + return Fetcher(scheme=args.scheme, ca_file=args.ca_file or None, + resolve=args.resolve or None, timeout=args.timeout) + + +# --- flags ---------------------------------------------------------------- + +def add_peer_access(parser): + """How an operator reaches a peer's published documents. + + Shared by verify and admit so the two cannot disagree about what + verification means -- the difference between them is what they do with the + result, not how they obtain it. + """ + parser.add_argument("--scheme", default="https", + help="http only for a stack with no certificates at all") + parser.add_argument("--ca-file", default="", + help="verify the peer's certificate against the CAs in this PEM file") + parser.add_argument("--resolve", default="", + help="reach the peer at this host:port instead of resolving its domain") + parser.add_argument("--timeout", type=float, default=10, help="per-request timeout") + + +def add_registry(parser): + parser.add_argument("--registry-url", required=True, + help="this network's own registry, e.g. http://localhost:10081") + parser.add_argument("--keycloak-url", required=True, + help="the registry's Keycloak, e.g. http://localhost:10080") + parser.add_argument("--keycloak-issuer", default="sunbird-registry-keycloak:8080", + help="the Host the registry expects the token's issuer to carry") + parser.add_argument("--realm", default="sunbird-rc", help="Keycloak realm") + parser.add_argument("--client-id", default="registry-frontend", help="Keycloak client id") + parser.add_argument("--username", required=True, help="registry admin user") + parser.add_argument("--password", required=True, help="registry admin password") + parser.add_argument("--registry-timeout", type=float, default=30, + help="per-request timeout for our own registry") + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="oanfed.py", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + publish = commands.add_parser("publish", help="generate and sign this network's documents") + publish.add_argument("--network-id", required=True, + help="this network's DNS domain, which is its identity") + publish.add_argument("--registry-url", required=True, + help="this network's own registry, which holds the entry " + "this descriptor is generated FROM") + publish.add_argument("--registry-timeout", type=float, default=30, + help="per-request timeout for the registry") + publish.add_argument("--operator", default="", + help="organisation answering for this network; " + "defaults to the registered name") + publish.add_argument("--revision", type=int, required=True, + help="descriptor revision; must increase on every republish") + publish.add_argument("--operating-profile", default="OAN/v0.1,Beckn/v2", + help="comma-separated version stack") + publish.add_argument("--status", default="Active", choices=DESCRIPTOR_STATUSES, + help="what this network declares itself to be") + publish.add_argument("--projection-ttl", type=int, default=None, + help="seconds a peer may keep what it crawls; omit to let peers decide") + publish.add_argument("--out", required=True, help="directory to write .well-known/ into") + publish.add_argument("--governance-key", required=True, + help="the governance private key file; it signs the descriptor and " + "NEVER traffic, so it must not be a registered key. " + "Generated (mode 600) if absent") + publish.set_defaults(run=cmd_publish) + + verify_cmd = commands.add_parser("verify", help="fetch and check another network's documents") + verify_cmd.add_argument("--network-id", required=True, help="the peer to check") + add_peer_access(verify_cmd) + verify_cmd.set_defaults(run=cmd_verify) + + admit = commands.add_parser("admit", help="record the decision to deal with a peer") + admit.add_argument("--network-id", required=True, + help="the peer to admit, which is its domain") + admit.add_argument("--federation-role", default=DEFAULT_FEDERATION_ROLE, + choices=[r for r in FEDERATION_ROLES if r != "self"], + help="what this peer is to us: source (we crawl it), " + "requester (it calls us), or both. Only set on first " + "admission -- a refresh keeps what was decided") + add_peer_access(admit) + add_registry(admit) + admit.set_defaults(run=cmd_admit) + + suspend = commands.add_parser("suspend", help="stop dealing with a peer") + suspend.add_argument("--network-id", required=True, help="the peer to suspend") + add_registry(suspend) + suspend.set_defaults(run=cmd_suspend) + + reinstate = commands.add_parser("reinstate", help="undo a suspension") + reinstate.add_argument("--network-id", required=True, help="the peer to reinstate") + add_registry(reinstate) + reinstate.set_defaults(run=cmd_reinstate) + + args = parser.parse_args(argv) + try: + return args.run(args) + except FederationError as err: + print(f"oanfed {args.command}: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/networkofnetwork/bin/render-adapter.py b/networkofnetwork/bin/render-adapter.py new file mode 100755 index 00000000..f8502968 --- /dev/null +++ b/networkofnetwork/bin/render-adapter.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Render this network's ONE adapter config. + +quick-start's setup.py does everything that is hard and shared -- generating the +keypair, seeding the registry, registering the schemas -- and is reused as-is. +What it cannot do is render a config it has never heard of, so this does that +one step and nothing else. + +It reads what setup.py already wrote: the network's keys.json and its .env. No +value is invented here. + +Run: render-adapter.py +""" +import json +import os +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parent +TEMPLATE = ROOT / "config" / "adapter.yaml.tmpl" +STATE = ROOT / "networks" + + +def read_env(path): + """The network's .env, as a dict. Same shape setup.py reads.""" + values = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip() + return values + + +def main(argv): + if len(argv) < 2: + sys.exit("usage: render-adapter.py ") + network = argv[1].strip() + directory = STATE / network + + keys_file = directory / "keys" / "keys.json" + env_file = directory / ".env" + for required in (keys_file, env_file): + if not required.exists(): + sys.exit(f"render-adapter: {required} is missing -- run setup first") + + env = read_env(env_file) + # One identity for the whole network, so any role's entry carries it. The + # file is still keyed by role because setup.py writes it that way. + identity = json.loads(keys_file.read_text())["network"] + + def value(name, default=None): + got = env.get(name, default) + if got is None: + sys.exit(f"render-adapter: {name} is not set in {env_file}") + return got + + def binding(prefix): + return f"{value(prefix + '_PARTICIPANT_ID')}|{value(prefix + '_CAPABILITY')}" + + substitutions = ( + ("__BECKN_NETWORK_ID__", identity["participantId"]), + ("__KEY_ID__", identity["keyOsid"]), + ("__SIGNING_PRIVATE__", identity["signingPrivate"]), + ("__SIGNING_PUBLIC__", identity["signingPublic"]), + ("__ENCR_PRIVATE__", identity["encrPrivate"]), + ("__ENCR_PUBLIC__", identity["encrPublic"]), + ("__NETWORK_BASE_URL__", value("NETWORK_BASE_URL")), + ("__MAUSAMGRAM_BINDING_KEY__", binding("MAUSAMGRAM")), + ("__AGMARKNET_BINDING_KEY__", binding("AGMARKNET")), + ("__VISTAAR_BINDING_KEY__", binding("VISTAAR")), + ("__POCRA_BINDING_KEY__", binding("POCRA")), + ("__MAUSAMGRAM_PARTICIPANT_ID__", value("MAUSAMGRAM_PARTICIPANT_ID")), + ("__AGMARKNET_PARTICIPANT_ID__", value("AGMARKNET_PARTICIPANT_ID")), + ("__VISTAAR_PARTICIPANT_ID__", value("VISTAAR_PARTICIPANT_ID")), + ("__POCRA_PARTICIPANT_ID__", value("POCRA_PARTICIPANT_ID")), + ("__AGMARKNET_TOKEN_URL__", value("AGMARKNET_TOKEN_URL")), + ("__VISTAAR_TOKEN_URL__", value("VISTAAR_TOKEN_URL")), + # Empty disables peer crawling: the crawler builds no peer half without + # a network id, so /crawl/peers refuses and the refresh loop never + # starts. Answering another network's crawl is unaffected. + ("__FEDERATION_NETWORK_ID__", + "" if value("FEDERATION_CRAWL_PEERS", "on").lower() == "off" + else value("BECKN_NETWORK_ID")), + ("__FEDERATION_REFRESH_SECONDS__", value("FEDERATION_REFRESH_SECONDS", "900")), + ("__FEDERATION_PAGE_SIZE__", value("FEDERATION_PAGE_SIZE", "0")), + ("__FEDERATION_UPDATE_MODE__", value("FEDERATION_UPDATE_MODE", "MERGE")), + ("__FEDERATION_INTENT__", value("FEDERATION_INTENT", "")), + ("__FEDERATION_PUBLISH_RETRY_SECONDS__", value("FEDERATION_PUBLISH_RETRY_SECONDS", "30")), + ("__OTEL_ENABLED__", value("ADAPTER_OTEL_ENABLED", "false")), + ("__OTLP_ENDPOINT__", value("OTLP_ENDPOINT", "hyperdx:4317")), + ("__OTEL_ENVIRONMENT__", value("OTEL_ENVIRONMENT", "dev")), + ) + + rendered = TEMPLATE.read_text() + for placeholder, replacement in substitutions: + rendered = rendered.replace(placeholder, replacement) + + # A placeholder left behind is a value nobody noticed was missing, and the + # adapter would start with "__SIGNING_PRIVATE__" as a key. + if "__" in rendered: + leftover = sorted({w for w in rendered.split() if w.startswith("__")}) + sys.exit(f"render-adapter: unrendered placeholders remain: {leftover}") + + out_dir = directory / "config" + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / "adapter.yaml" + if out.is_dir(): + sys.exit(f"render-adapter: {out} is a directory -- docker created it by " + f"starting the adapter before this ran. Remove it and retry.") + out.write_text(rendered) + out.chmod(0o600) # holds a private key + print(f"render-adapter: wrote {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/networkofnetwork/bin/setup.py b/networkofnetwork/bin/setup.py new file mode 100755 index 00000000..bf720b97 --- /dev/null +++ b/networkofnetwork/bin/setup.py @@ -0,0 +1,661 @@ +#!/usr/bin/env python3 +"""Prepare the stack: generate the adapter keypairs, seed the registry, render +the adapter configs. + + python3 bin/setup.py + +`make up` runs this as step 2, which is where it belongs -- the adapter configs +it renders are bind-mounted files, and an adapter started before they exist +leaves a directory in their place. + +WHAT IT WRITES. Seven participants, four capability bindings, four schemas: + + 3 x node one per adapter -- consumer, network, provider -- each with the + public halves of a keypair. The private halves stay in + keys/keys.json and never reach the registry. + 4 x upstream the APIs this deployment calls. ALL FOUR are real external + hosts now, whose base URLs come from .env -- no mock is in + use by any capability. An upstream signs nothing, so it needs + no role and no keys. + 4 x binding a ProviderSchema row per capability: which upstream answers + it, the method and path, timeouts, and the mapping URL. + 4 x schema a SchemaRegistry row per capability, saying what the + capability MEANS rather than who serves it: the pack in + network-specs that defines its attributes. One per + capability, so two providers of one capability share a row. + +This is all of it. Nothing has to be created by hand afterwards, and nothing +can be from outside the VM -- the registry has no route through the gateway and +publishes on loopback only, which is why seeding lives here rather than in a +Postman request. + +The binding keys are the load-bearing part. A provider step answers only when +the key it was configured with matches the one built from the incoming payload, +and both sides come from the same .env values in this one run -- which is what +keeps them from disagreeing. To point a capability somewhere else: edit .env, +re-run this, recreate the provider adapter. + +Safe to re-run. Keys are generated once and reused from keys/keys.json, so the +identities already in the registry stay valid; participants and bindings that +exist are left alone rather than recreated, because this registry's delete is +soft and holds the unique index -- a deleted participantId cannot be reused. + +Needs python3 and the cryptography package: + + pip install cryptography +""" +import base64, json, os, pathlib, sys, time, urllib.error, urllib.parse, urllib.request + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from cryptography.hazmat.primitives import serialization as ser + +# Where this run's .env, keys and rendered adapter configs live. +# +# Defaults to the quick-start directory itself, which is every existing use. +# QUICKSTART_ROOT overrides it so one checkout can set up more than one network: +# each network gets its own directory, because keys.json and the rendered +# adapter configs carry that network's identity and a second setup run in the +# same place would overwrite the first one's. +ROOT = pathlib.Path(os.environ.get("QUICKSTART_ROOT") or pathlib.Path(__file__).resolve().parent.parent) +KEYS = ROOT / "keys" / "keys.json" +# No ADAPTERS directory here: this deployment runs ONE adapter, and its config +# is rendered by bin/render-adapter.py from config/adapter.yaml.tmpl. This script +# does the part that is the same either way -- keys, the registry, the schemas. + +REPO = pathlib.Path(__file__).resolve().parent.parent + + +def load_dotenv(): + """Read .env into the environment. + + Nothing sources .env before this runs -- `make up` shells straight out to + python3 -- and a real environment variable wins so a one-off override still + works: + + REGISTRY_PORT=9081 python3 bin/setup.py + """ + path = ROOT / ".env" + if not path.exists(): + sys.exit("setup: no .env -- copy .env.example to .env first") + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, value = line.split("=", 1) + os.environ.setdefault(name.strip(), value.strip()) + + +def env(name, default=None): + v = os.environ.get(name, default) + if v is None: + sys.exit(f"setup: {name} is not set -- is it missing from .env?") + if is_placeholder(v): + sys.exit(f"setup: {name} is still the .env.example placeholder {v!r}.\n" + f"{PLACEHOLDER_WHY}") + return v + + +def is_placeholder(value): + """True for a value still in .env.example's `` form. + + Every placeholder in the template is angle-bracketed, and no real base URL, + path, credential or id can be -- so this is an exact test rather than a + guess.""" + v = value.strip() + return v.startswith("<") and v.endswith(">") + + +PLACEHOLDER_WHY = """\ + Fill it in in .env, then re-run. Nothing has been written yet. + + This is refused rather than passed through because the registry is + APPEND-ONLY: it cannot update a record, and its delete is soft and keeps the + unique index, so an id is never reusable. Seeding a participant with a + placeholder base URL would burn that id permanently -- the capability could + only be recovered under a NEW id, which is the one thing the settled naming + was meant to avoid.""" + + +# Everything that has to be a real value before the first registry write. +# +# Checked together, up front, for two reasons. seed() runs BEFORE render(), so a +# placeholder in a render-only key like VISTAAR_TOKEN_URL would otherwise be +# found only after seven rows already existed -- unfixable, per the note above. +# And one list beats being sent back to .env ten times in a row. +REQUIRED = [ + "BECKN_NETWORK_ID", + "CONSUMER_BASE_URL", "NETWORK_BASE_URL", "PROVIDER_BASE_URL", + "MAUSAMGRAM_PARTICIPANT_ID", "MAUSAMGRAM_CAPABILITY", "MAUSAMGRAM_BASE_URL", + "MAUSAMGRAM_AUTH", "MAUSAMGRAM_MAPPING_URL", + "AGMARKNET_PARTICIPANT_ID", "AGMARKNET_CAPABILITY", "AGMARKNET_BASE_URL", + "AGMARKNET_TOKEN_URL", "AGMARKNET_ACCESS_NAME", "AGMARKNET_PASSWORD", + "AGMARKNET_MAPPING_URL", + "VISTAAR_PARTICIPANT_ID", "VISTAAR_CAPABILITY", "VISTAAR_BASE_URL", + "VISTAAR_PATH", "VISTAAR_TOKEN_URL", "VISTAAR_CLIENT_ID", + "VISTAAR_CLIENT_SECRET", "VISTAAR_MAPPING_URL", + "POCRA_PARTICIPANT_ID", "POCRA_CAPABILITY", "POCRA_BASE_URL", + "POCRA_PATH", "POCRA_MAPPING_URL", +] + + +def preflight(): + """Refuse to touch the registry while any required value is unfilled.""" + missing = [k for k in REQUIRED if os.environ.get(k) is None] + unfilled = [k for k in REQUIRED + if os.environ.get(k) is not None + and is_placeholder(os.environ[k])] + if not missing and not unfilled: + return + lines = ["setup: .env is not ready, so nothing was written.\n"] + if unfilled: + lines.append(" Still the .env.example placeholder:") + lines += [f" {k}={os.environ[k]}" for k in unfilled] + if missing: + lines.append(" Absent from .env entirely:") + lines += [f" {k}" for k in missing] + lines.append(PLACEHOLDER_WHY) + sys.exit("\n".join(lines)) + + +# --------------------------------------------------------------------- keys + +def b64(raw): + return base64.b64encode(raw).decode() + + +def ed25519_pair(): + k = Ed25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def x25519_pair(): + k = X25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def load_or_generate_keys(): + """One identity for the whole network, shared by all three adapters. + + The network IS the participant: networkId and participantId are the same + string, and there is ONE registry entry and ONE keypair behind it. + + Two identities -- an adapter's and the network's -- reintroduce the question + of which key signs what, and they make the three-adapter and consolidated + deployments diverge: in one shape the consumer layer is a separate + registered party, in the other it is the same process. One identity means a + peer sees the same network either way, and the layer split stays a local + deployment detail. + + The dict is still keyed by role because the config templates are per-role + and each needs the same values substituted. All three entries carry the same + participantId and the same keypair. + """ + network = env("BECKN_NETWORK_ID") + if not network: + sys.exit("setup: BECKN_NETWORK_ID is required -- it is this network's identity") + + if KEYS.exists(): + print("keys: reusing keys/keys.json") + identities = json.loads(KEYS.read_text()) + # The keypair persists; the id is read from .env each run. This registry + # cannot update a record, so changing an id in .env seeds a NEW + # participant rather than editing one -- and the same keypair moving to + # the new id is what makes that a rename rather than a rekey. + for role in ROLES: + if identities[role]["participantId"] != network: + print(f" {role}: id is now {network}, keeping the keypair") + identities[role]["participantId"] = network + KEYS.write_text(json.dumps(identities, indent=2)) + return identities + + print(f"keys: generating one identity for {network}") + sign_private, sign_public = ed25519_pair() + encr_private, encr_public = x25519_pair() + identities = {} + for role in ROLES: + identities[role] = {"participantId": network, + "signingPrivate": sign_private, "signingPublic": sign_public, + "encrPrivate": encr_private, "encrPublic": encr_public} + # parents=True because under QUICKSTART_ROOT the network's directory itself + # may be new, not just the keys/ inside it. + KEYS.parent.mkdir(parents=True, exist_ok=True) + KEYS.write_text(json.dumps(identities, indent=2)) + KEYS.chmod(0o600) + return identities + + +# ----------------------------------------------------------------- registry + +def registry_url(): + return f"http://localhost:{env('REGISTRY_PORT', '8081')}" + + +def token(): + body = urllib.parse.urlencode({ + "client_id": env("KEYCLOAK_CLIENT_ID", "registry-frontend"), + "grant_type": "password", + "username": env("REGISTRY_ADMIN_USER", "no-user"), + "password": env("REGISTRY_ADMIN_PASSWORD", "no-user-password"), + }).encode() + # Keycloak sits behind PROXY_ADDRESS_FORWARDING, so it builds the token's + # issuer from these headers. Without them it answers with an empty body. + # + # registry-keycloak:8080 is the CONTAINER-INTERNAL address, and is + # deliberately not KEYCLOAK_PORT. The registry validates the issuer against + # OAUTH2_RESOURCES_0_URI, which names that internal address -- so a token + # minted with the host port in its issuer is rejected with a 401 and an + # empty body, however the port is published. + # + # It must match this compose file's keycloak SERVICE NAME. It does not match + # quick-start's, which is why this copy differs from that one by this line. + req = urllib.request.Request( + f"http://localhost:{env('KEYCLOAK_PORT', '8080')}/auth/realms/" + f"{env('KEYCLOAK_REALM', 'sunbird-rc')}/protocol/openid-connect/token", + data=body, headers={"X-Forwarded-Host": "registry-keycloak:8080", + "X-Forwarded-Proto": "http"}) + with urllib.request.urlopen(req, timeout=30) as r: + payload = json.load(r) + if not payload.get("access_token"): + sys.exit("setup: keycloak issued no token -- check the KEYCLOAK_* values in .env") + return payload["access_token"] + + +def post(entity, payload, bearer): + # No {"EntityName": {...}} wrapper: this registry takes the record itself, + # and a wrapper comes back as "extraneous key [...] is not permitted". + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}", method="POST", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {bearer}"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + except urllib.error.HTTPError as e: + # The registry answers a rejected write with a JSON envelope, but not + # always: a 401 comes back with an empty body. Decoding blindly turns + # that into a JSONDecodeError traceback that says nothing about what + # went wrong, so report the status instead. + raw = e.read() + try: + return json.loads(raw) + except ValueError: + detail = raw.decode(errors="replace").strip()[:200] or "(empty body)" + sys.exit(f"setup: the registry refused a write -- HTTP {e.code}: {detail}\n" + f" A 401 here usually means the token was minted for a different\n" + f" issuer than the registry validates against.") + + +def search(entity, filters): + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}/search", method="POST", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r).get("data", []) + + +def wait_for_registry(): + for _ in range(60): + try: + search("Participant", {}) + return + except Exception: + time.sleep(2) + sys.exit("setup: the registry did not come up -- check `docker compose ps`\n" + " and that REGISTRY_PORT in .env matches the published port") + + +def signing_key_block(public_key): + """The published half of the signing keypair. + + Bare base64, with no encoding label in front of it: what a verifier hands + to a base64 decoder is the value as published, and a label left on fails + every verification with a decode error pointing nowhere near the registry. + + No friendly key id, because nothing could look one up: the registry assigns + an osid on write and that is what a sender names in the Authorization + header. No use either -- alg carries the purpose, ed25519 signs.""" + return [{"alg": "ed25519", "key": public_key, "status": "active", + "validFrom": "2026-01-01T00:00:00Z", "validUntil": "2030-01-01T00:00:00Z"}] + + +def node(participant_id, name, role, base_url, public_key, federation_role=None): + """A participant that speaks Beckn. + + One level, no wrapper object: type decides which fields apply. baseUrl must + be https for a node, and the id must be hostname-shaped -- it is the + identity a signature is checked against. + + baseUrl IS PUBLISHED FOR PEERS, and is the one field here that a real + network reads rather than this stack. Nothing inside this stack resolves it + -- routing between the adapters is the router plugin's config, which uses + the compose service names -- so a wrong value costs nothing locally and + everything on a shared network, where it is the address a peer POSTs to. + That is why it comes from .env rather than being built from the id: this + registry cannot update a record, so a fabricated URL is permanent.""" + record = {"participantId": participant_id, "name": name, "type": "node", + "status": "active", "baseUrl": base_url, + "role": role, "keys": signing_key_block(public_key)} + if federation_role: + record["federationRole"] = federation_role + return record + + +def upstream(participant_id, name, base_url): + """An ordinary API the provider adapter calls. + + No role and no keys: it has never heard of Beckn, so it signs nothing and + nothing verifies it. Both are permitted by the schema but neither is read -- + a signature is checked against the node identity that signed it. + + No credential either. The adapter presents one from its own config, naming + the environment variable it comes from, so nothing secret is held here.""" + return {"participantId": participant_id, "name": name, "type": "upstream", + "status": "active", "baseUrl": base_url} + + +def ensure_binding(bearer, participant_id, capability, path, mapping_url, + method="GET"): + """Create a capability binding only when absent. + + actions is a list, not a map: the registry treats every nested object as an + entity and injects an osid into it, which a map cannot carry. It is also + what lets one action be retired without touching the others. + + mappings is one reference carrying both directions, because the response + mapping reads what the request mapping resolved. + + method is a parameter because it is the provider's contract, not a network + convention: Mausamgram and Agmarknet answer a GET whose query the mapping + builds, while the knowledge provider and POCRA take a POST with a JSON body. + The adapter reads it from this row, so nothing about it is compiled in.""" + binding = f"{participant_id}|{capability}" + if search("ProviderSchema", {"bindingKey": {"eq": binding}}): + print(f" {binding}: already present") + return + result = post("ProviderSchema", { + "bindingKey": binding, "participantId": participant_id, + "capabilityCode": capability, "status": "active", + "actions": [{"action": "select", "method": method, "path": path, + "mappings": mapping_url, + "timeoutMs": 15000, "retryMax": 2, + "status": "active"}]}, bearer) + print(f" {binding}: {result['params']['status']} " + f"{result['params'].get('errmsg', '')[:160]}") + + +# The four capability schemas, keyed by the .env variable each capability code +# comes from so the code itself is never written twice. +# +# The display name mirrors the pack's own info.title, minus its "OpenAgriNet -" +# and "Attributes" framing. The version is per capability rather than one +# setting for all four, because a capability can move to v0.2 while the others +# stay -- that is the point of versioning the packs separately. +CAPABILITY_SCHEMAS = ( + ("MAUSAMGRAM_CAPABILITY", "Weather Observation", "v0.1"), + ("AGMARKNET_CAPABILITY", "Mandi Price", "v0.1"), + ("VISTAAR_CAPABILITY", "Knowledge Advisory", "v0.1"), + ("POCRA_CAPABILITY", "Agriculture Facility", "v0.1"), +) + + +def schema_url(capability, version, ref): + """Build a pack URL from the capability code, which already contains the + pack directory's name: openagrinet:WeatherObservation is served out of + schema/WeatherObservation. Deriving it means the two cannot drift. + + The registry pattern-checks the result against exactly this shape, so a + capability whose pack is laid out differently is refused rather than + stored -- which is the behaviour to want, since all eight packs in + network-specs are //attributes.yaml today.""" + # openagrinet.github.io, NOT raw.githubusercontent.com: the latter 404s for + # every path under network-specs, so a crawl asking with schemaContext + # derived from these URLs matched nothing and mirrored zero catalogs. The + # published pages are what capability examples carry in their @context. + # + # ref is unused here -- the pages site serves one published version and has + # no per-ref path -- but is kept so the caller's SCHEMA_PACK_REF still reads + # as the thing that would select one if the layout gains it. + del ref + return ("https://openagrinet.github.io/network-specs/" + f"schema/{capability.partition(':')[2]}/{version}" + "/attributes.yaml") + + +def ensure_capability_schema(bearer, capability, name, version, ref): + """Create a capability's schema row only when absent. + + THE REGISTRY NEVER FETCHES schemaUrl -- it only checks its shape. So a row + pointing at a ref that does not serve the pack stores perfectly happily and + fails later, at whoever resolves it. Getting SCHEMA_PACK_REF right matters + more than the write succeeding. + + And it cannot be corrected afterwards: SchemaRegistry.json sets + additionalProperties false, so an update is rejected over the system fields + the registry itself merges in, while delete is soft and keeps the unique + index on capabilityCode. A wrong URL here is permanent for that code.""" + if search("SchemaRegistry", {"capabilityCode": {"eq": capability}}): + print(f" {capability}: already present") + return + result = post("SchemaRegistry", { + "capabilityCode": capability, "name": name, "version": version, + "schemaUrl": schema_url(capability, version, ref), + "status": "active"}, bearer) + print(f" {capability}: {result['params']['status']} " + f"{result['params'].get('errmsg', '')[:160]}") + + +def ensure_participant(bearer, participant_id, payload): + """Create only when absent. Delete here is soft and keeps the unique index, + so a recreate would fail on a duplicate key rather than replacing.""" + if search("Participant", {"participantId": {"eq": participant_id}}): + print(f" {participant_id}: already present") + return + result = post("Participant", payload, bearer) + status = result["params"]["status"] + print(f" {participant_id}: {status} {result['params'].get('errmsg', '')[:160]}") + + +def seed(identities): + wait_for_registry() + bearer = token() + + # Seven participants and four capability bindings, all of it from here. + # + # The registry is not reachable from outside this stack -- no published + # port beyond loopback and no proxy host in front of it -- so there is no + # second way to create these. Everything the network needs to answer a + # request has to exist by the time this returns. + # ONE entry. The network IS the participant, and `role` says what the + # NETWORK serves rather than which container exists -- a split + # three-adapter deployment and a consolidated one carry the same value. + # + # baseUrl is the network's own published address, which is what a peer + # reaches us at. Not any individual adapter's: routing between the layers is + # the routing config's job and never appears in the registry. + print("registry: one network identity") + identity = identities["network"] + ensure_participant(bearer, identity["participantId"], + node(identity["participantId"], + env("BECKN_NETWORK_NAME", identity["participantId"]), + list(ROLES), + env("NETWORK_BASE_URL"), + identity["signingPublic"], + # OURS. Stored rather than worked out: deciding + # whether an entry is this deployment's own by + # comparing ids against config has been got wrong + # twice, once badly enough that the crawler tried to + # crawl itself. + federation_role="self")) + + # All four upstreams are real external providers now -- no mock is in use + # by any capability, so every base URL comes from .env and the stack needs + # egress to all of them. + # + # No default base URL for weather any more: falling back to the mock's + # service name would seed a row that cannot serve the basic auth the + # capability is configured with, and the registry cannot update it after. + print("registry: four upstream providers") + weather = env("MAUSAMGRAM_PARTICIPANT_ID") + ensure_participant(bearer, weather, + upstream(weather, "IMD Mausamgram NWP", + env("MAUSAMGRAM_BASE_URL"))) + + # Mandi is the REAL Agmarknet now, because authScheme tokenQuery needs a + # token endpoint and mock-agmarknet has none. No default base URL for the + # same reason: falling back to the mock's service name would seed a row + # that cannot serve the configured scheme, and the registry cannot update + # it afterwards. + mandi = env("AGMARKNET_PARTICIPANT_ID") + ensure_participant(bearer, mandi, + upstream(mandi, "Agmarknet Vistaar", + env("AGMARKNET_BASE_URL"))) + + # The knowledge capability. Also a REAL external provider, so the stack + # needs egress to it and its base URL comes from .env rather than a service + # name. It is the only one whose action is a POST. + knowledge = env("VISTAAR_PARTICIPANT_ID") + ensure_participant(bearer, knowledge, + upstream(knowledge, "Bharat Vistaar knowledge retrieval", + env("VISTAAR_BASE_URL"))) + + # POCRA, serving AgricultureFacility. A real external provider too, and the + # only one needing no credential at all. + pocra = env("POCRA_PARTICIPANT_ID") + ensure_participant(bearer, pocra, + upstream(pocra, "POCRA agriculture facilities", + env("POCRA_BASE_URL"))) + + # And what each of them answers. The binding key is participantId piped to + # capabilityCode, and it has to match what the provider adapter was + # rendered with -- both come from the same .env, which is what keeps them + # from disagreeing. + print("registry: four capability bindings") + ensure_binding(bearer, weather, env("MAUSAMGRAM_CAPABILITY"), + env("MAUSAMGRAM_PATH", "/get-daily"), env("MAUSAMGRAM_MAPPING_URL")) + ensure_binding(bearer, mandi, env("AGMARKNET_CAPABILITY"), + env("AGMARKNET_PATH", "/v1/fetch-agmarknet-vistaar"), + env("AGMARKNET_MAPPING_URL")) + ensure_binding(bearer, knowledge, env("VISTAAR_CAPABILITY"), + env("VISTAAR_PATH"), env("VISTAAR_MAPPING_URL"), + method="POST") + ensure_binding(bearer, pocra, env("POCRA_CAPABILITY"), + env("POCRA_PATH"), env("POCRA_MAPPING_URL"), + method="POST") + + # And what each capability MEANS, which is a different question from who + # serves it. A binding above says mausamgram answers WeatherObservation + # over this path with this mapping; a row here says what a + # WeatherObservation is, by pointing at the schema pack that defines its + # attributes. One row per capability, not per binding -- a second provider + # for the same capability shares this row rather than adding one. + # + # SEPARATE FROM THE MAPPING URLS ABOVE, which live in helmcharts and + # describe one upstream's wire format. These are network-specs and describe + # the network's own vocabulary, so they move on their own ref. + print("registry: four capability schemas") + ref = env("SCHEMA_PACK_REF", "main") + for var, name, version in CAPABILITY_SCHEMAS: + ensure_capability_schema(bearer, env(var), name, version, ref) + + +def key_osids(identities): + """Read back each key's osid, and check the registry still holds the public + half we have the private half for. + + The Authorization header names a key by its osid rather than by the friendly + id, so the adapters have to be configured with the value the registry + assigned. + + The mismatch check matters because this registry cannot update a record and + its delete is soft: a participant seeded against an earlier keys.json keeps + that public key forever. Signing with a new private half would then produce + signatures nobody can verify -- and the failure would surface much later, as + an authentication error with no obvious cause.""" + # All three entries name the same participant, so this resolves one record + # three times. Left as a loop rather than special-cased: each role's config + # still needs the osid substituted, and one extra cached search is cheaper + # than a second code path. + for role, identity in identities.items(): + records = search("Participant", {"participantId": {"eq": identity["participantId"]}}) + keys = (records[0].get("keys") or []) if records else [] + if not keys: + sys.exit(f"setup: {identity['participantId']} has no published key") + + # Bare base64 now, but an older row may still carry the label, and a + # confusing mismatch error is worse than one tolerant line. + published = keys[0]["key"].removeprefix("base64:") + if published != identity["signingPublic"]: + sys.exit( + f"setup: {identity['participantId']} is registered with a different key.\n" + f" This registry cannot update a record, and its delete is soft and keeps\n" + f" the unique index, so the id cannot be reused. Either restore the\n" + f" matching keys/keys.json, or pick a new BECKN_NETWORK_ID\n" + f" in .env and re-run.") + identity["keyOsid"] = keys[0]["osid"] + return identities + + +# ------------------------------------------------------------------ configs + +# The role key is three things at once: the entry in keys/keys.json, the +# __ROLE_* placeholder prefix, and the config filename. They were spelled +# differently once -- role "exp" writing experience.yaml -- and a lookup table +# kept them apart. They are the same word now, so the table is gone. +# +# CHANGING A ROLE KEY IS NOT A RENAME. It makes this script generate a FRESH +# keypair for a participant the registry has already published a public key +# for, which it cannot update and whose delete is soft -- so the id cannot be +# reused either. The adapter then signs with a key nobody can verify, and it +# surfaces much later as an authentication error with no obvious cause. Renaming +# exp -> consumer therefore came with `make destroy` and a new keys.json. +ROLES = ("consumer", "network", "provider") + + +if __name__ == "__main__": + load_dotenv() + preflight() + identities = load_or_generate_keys() + seed(identities) + identities = key_osids(identities) + KEYS.write_text(json.dumps(identities, indent=2)) + print(f""" +ready. The registry holds seven participants and four capability bindings, and +the adapter configs are rendered, so nothing further has to be created by hand. + + {env('MAUSAMGRAM_PARTICIPANT_ID')}|{env('MAUSAMGRAM_CAPABILITY')} + {env('AGMARKNET_PARTICIPANT_ID')}|{env('AGMARKNET_CAPABILITY')} + {env('VISTAAR_PARTICIPANT_ID')}|{env('VISTAAR_CAPABILITY')} + {env('POCRA_PARTICIPANT_ID')}|{env('POCRA_CAPABILITY')} + +Those are the binding keys the provider adapter answers to. They were rendered +into its config from the same .env this seeded the registry from, which is what +keeps the two from disagreeing. A payload naming anything else is answered 404 +"this module serves no capability matching the request" -- explicit, but it +names the request rather than the mismatch, so compare it against these four. + +The left half of each key is the participant id, and it names the ENTITY that +operates the upstream -- not the capability, which is already the right half, +and not the deployment, which is only a base URL. It is also the string a +catalog publishes as offer.provider.id, so capability-examples/ can be run +against this stack unedited. + +All four upstreams are real external providers reached over the internet, so +this stack needs egress to each of them and no capability depends on a mock. +Repointing one is an .env edit and a re-run of this, but it takes a NEW +participant id: the registry cannot update a row and its delete is soft and +keeps the unique index, so an id seeded against one base URL keeps that base +URL for good. A re-run with the same id prints "already present" and changes +nothing. The registry is not reachable from outside this stack, so that write +happens from here. + +Next: `make up` continues to step 3 and starts the adapters. If you ran this +on its own, the adapters need recreating to pick up the rendered configs: + + docker compose up -d --force-recreate provider-adapter network-adapter consumer-adapter + +Then import ../api-collection/ -- both files, the collection and the +environment -- and run it. Eighteen requests across the four capabilities, +publish then discover then select in each, with nothing to fill in.""") diff --git a/networkofnetwork/bin/test_oanfed.py b/networkofnetwork/bin/test_oanfed.py new file mode 100644 index 00000000..4ebbcc61 --- /dev/null +++ b/networkofnetwork/bin/test_oanfed.py @@ -0,0 +1,234 @@ +"""Checks for publishing and verifying the Network Descriptor and JWKS. + + python3 -m unittest networkofnetwork/bin/test_oanfed.py + +No network, no registry: the registry is a stub returning one entry. +""" + +import base64, copy, json, os, pathlib, sys, tempfile, unittest +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +import oanfed as o +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +NETWORK = "network-a.example.org" + + +def raw_public(private): + return o.public_of(private) + + +class StubRegistry: + entry = None + + def __init__(self, *_): + pass + + def participant(self, participant_id): + return self.entry if participant_id == NETWORK else None + + def schema_packs(self): + return ["https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1"] + + +class Base(unittest.TestCase): + def setUp(self): + self.dir = pathlib.Path(tempfile.mkdtemp()) + self.traffic = Ed25519PrivateKey.generate() + StubRegistry.entry = { + "participantId": NETWORK, "name": "Network A", + "baseUrl": f"https://{NETWORK}/network/discover", + "keys": [self.registry_key(self.traffic)], + } + self._real = o.ReadOnlyRegistry + o.ReadOnlyRegistry = StubRegistry + + def tearDown(self): + o.ReadOnlyRegistry = self._real + + @staticmethod + def registry_key(private, **extra): + return {"alg": "ed25519", "status": "active", "validFrom": "2026-01-01T00:00:00Z", + "key": base64.b64encode(raw_public(private)).decode(), **extra} + + def publish(self, revision=1, **overrides): + args = SimpleNamespace( + network_id=NETWORK, registry_url="http://registry", registry_timeout=1, + operator="", revision=revision, operating_profile="OAN/v0.1,Beckn/v2", + status="Active", projection_ttl=None, out=str(self.dir), + governance_key=str(self.dir / "keys" / "governance.key")) + vars(args).update(overrides) + o.cmd_publish(args) + well_known = self.dir / ".well-known" + return (json.loads((well_known / "openagrinet").read_text()), + json.loads((well_known / "jwks.json").read_text())) + + +class Canonical(unittest.TestCase): + def test_keys_sort_by_utf16_code_units(self): + # RFC 8785 section 3.2.3: U+1F600 sorts BEFORE U+FB33 in UTF-16. + keys = ["€", "\r", "דּ", "1", "\U0001F600", "\u0080", "ö"] + out = o._jcs({k: 0 for k in keys}) + order = [json.loads("{" + p + "}").popitem()[0] for p in out[1:-1].split(",")] + self.assertEqual(order, ["\r", "1", "\u0080", "ö", "€", "\U0001F600", "דּ"]) + + def test_escaping_and_no_whitespace(self): + self.assertEqual(o._jcs({"b": "a/\u001f\né", "a": [1, True, None]}), + '{"a":[1,true,null],"b":"a/\\u001f\\né"}') + + def test_floats_huge_ints_and_lone_surrogates_are_refused(self): + for bad in (1.5, 2 ** 53, "\ud800"): + with self.assertRaises(o.FederationError): + o._jcs({"x": bad}) + + def test_signature_is_excluded(self): + d = {"a": 1} + self.assertEqual(o.canonical(d), o.canonical({**d, "signature": {"value": "x"}})) + + +class Publish(Base): + def test_documents_verify_and_split_the_keys(self): + descriptor, keyset = self.publish() + o.verify(descriptor, keyset) + governance = descriptor["signature"]["keyId"] + self.assertEqual(descriptor["operationalKeys"], [o.jwk(raw_public(self.traffic))["kid"]]) + self.assertNotIn(governance, descriptor["operationalKeys"]) + self.assertEqual({k["kid"] for k in keyset["keys"]}, + {governance, *descriptor["operationalKeys"]}) + for field in ("version", "networkId", "name", "operator", "revision", "status", + "issuedAt", "operatingProfile", "schemaPacks", "services"): + self.assertIn(field, descriptor) + self.assertEqual(descriptor["operatingProfile"], ["OAN/v0.1", "Beckn/v2"]) + + def test_governance_key_is_created_private_and_reused(self): + first, _ = self.publish(1) + key = self.dir / "keys" / "governance.key" + self.assertEqual(key.stat().st_mode & 0o777, 0o600) + second, _ = self.publish(2) + self.assertEqual(first["signature"]["keyId"], second["signature"]["keyId"]) + + def test_readable_governance_key_is_refused(self): + self.publish(1) + os.chmod(self.dir / "keys" / "governance.key", 0o644) + with self.assertRaisesRegex(o.FederationError, "chmod 600"): + self.publish(2) + + def test_revision_must_increase(self): + self.publish(3) + for revision in (3, 2): + with self.assertRaisesRegex(o.FederationError, "does not increase"): + self.publish(revision) + + def test_corrupt_published_descriptor_is_not_read_as_revision_zero(self): + self.publish(5) + (self.dir / ".well-known" / "openagrinet").write_text("{") + with self.assertRaises(o.FederationError): + self.publish(1) + + def test_governance_key_that_is_also_a_traffic_key_is_refused(self): + self.publish(1) + seed = o.unb64url((self.dir / "keys" / "governance.key").read_text().strip()) + governance = Ed25519PrivateKey.from_private_bytes(seed) + StubRegistry.entry["keys"].append(self.registry_key(governance)) + with self.assertRaisesRegex(o.FederationError, "governance key must be separate"): + self.publish(2) + + def test_only_usable_registry_keys_are_delegated(self): + revoked, expired, upcoming = (Ed25519PrivateKey.generate() for _ in range(3)) + future = (datetime.now(timezone.utc) + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + StubRegistry.entry["keys"] += [ + self.registry_key(revoked, status="revoked"), + self.registry_key(expired, validUntil="2020-01-01T00:00:00Z"), + self.registry_key(upcoming, validFrom=future), + {"alg": "x25519", "status": "active", "key": base64.b64encode(b"e" * 32).decode()}, + ] + descriptor, _ = self.publish() + self.assertEqual(descriptor["operationalKeys"], + [o.jwk(raw_public(k))["kid"] for k in (self.traffic, upcoming)]) + + def test_no_usable_traffic_key_is_refused(self): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, status="revoked")] + with self.assertRaisesRegex(o.FederationError, "no active Ed25519 key"): + self.publish() + + def test_invalid_descriptor_is_never_written(self): + StubRegistry.entry["baseUrl"] = "http://insecure/discover" + with self.assertRaisesRegex(o.FederationError, "services.discovery"): + self.publish() + self.assertFalse((self.dir / ".well-known" / "openagrinet").exists()) + + def test_unregistered_network_is_refused(self): + with self.assertRaisesRegex(o.FederationError, "not in this registry"): + self.publish(network_id="other.example.org") + + +class Verify(Base): + def setUp(self): + super().setUp() + self.descriptor, self.keyset = self.publish() + + def refuses(self, descriptor=None, keyset=None, match=""): + with self.assertRaisesRegex(o.FederationError, match): + o.verify(self.descriptor if descriptor is None else descriptor, + self.keyset if keyset is None else keyset) + + def test_any_change_to_a_signed_field_is_caught(self): + for field, value in (("status", "Suspended"), ("revision", 99), + ("services", {"discovery": "https://evil.example/d"}), + ("operationalKeys", [self.keyset["keys"][1]["kid"], "extra"])): + tampered = {**self.descriptor, field: value} + with self.subTest(field=field): + self.refuses(tampered) + + def test_signer_must_not_also_be_delegated_for_traffic(self): + governance = self.descriptor["signature"]["keyId"] + d = {**self.descriptor, "operationalKeys": [governance]} + self.refuses(d, match="also delegated for traffic") + + def test_delegated_key_missing_from_jwks(self): + keyset = {"keys": [k for k in self.keyset["keys"] + if k["kid"] == self.descriptor["signature"]["keyId"]]} + self.refuses(keyset=keyset, match="operationalKeys not in the JWKS") + + def test_malformed_signatures_fail_cleanly(self): + for signature in (None, {}, {"algorithm": "EdDSA"}, + {**self.descriptor["signature"], "keyId": "nope"}, + {**self.descriptor["signature"], "value": "!!not-base64!!"}, + {**self.descriptor["signature"], "value": ""}): + with self.subTest(signature=signature): + self.refuses({**self.descriptor, "signature": signature}) + + def test_malformed_jwks(self): + dup = copy.deepcopy(self.keyset) + dup["keys"].append(dict(dup["keys"][0])) + short = copy.deepcopy(self.keyset) + short["keys"][0]["x"] = o.b64url(b"short") + for keyset in ({}, {"keys": "x"}, {"keys": [{"kty": "OKP"}]}, dup, short): + with self.subTest(keyset=keyset): + self.refuses(keyset=keyset) + + def test_contract_fields(self): + future = (datetime.now(timezone.utc) + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + for field, value in (("version", "0.1"), ("networkId", "Not A Domain"), + ("name", ""), ("operator", {}), ("revision", 0), + ("revision", True), ("status", "active"), ("issuedAt", future), + ("issuedAt", "2026-02-30T00:00:00Z"), ("operatingProfile", []), + ("schemaPacks", ["http://x/y"]), ("projectionTtl", -1)): + with self.subTest(field=field, value=value): + with self.assertRaises(o.FederationError): + o.validate_descriptor({**self.descriptor, field: value}) + + def test_admit_stores_only_delegated_keys(self): + # A key added to the JWKS without the governance signature delegating it + # must not reach the peer record either. + injected = copy.deepcopy(self.keyset) + injected["keys"].append(o.jwk(raw_public(Ed25519PrivateKey.generate()))) + record = o.admit_record(None, self.descriptor, injected) + self.assertEqual([k["key"] for k in record["keys"]], + [base64.b64encode(raw_public(self.traffic)).decode()]) + + +if __name__ == "__main__": + unittest.main() diff --git a/networkofnetwork/config/adapter.yaml.tmpl b/networkofnetwork/config/adapter.yaml.tmpl new file mode 100644 index 00000000..4aa76a7d --- /dev/null +++ b/networkofnetwork/config/adapter.yaml.tmpl @@ -0,0 +1,505 @@ +# ONE OAN adapter, serving every layer this network has. +# +# The quick-start runs three adapters -- consumer, network, provider -- as three +# containers. That is a deployment choice, not what a network IS: a network is +# one participant with one identity, and the three layers are modules of it. +# This config is the consolidated shape, which is the one to reason about when +# there are two whole networks on the machine already. +# +# Each layer is a module on its own path of one port: +# +# /consumer/ what the experience layer posts to +# /network/ publish and discover, answered from this network's discovery +# /provider/ select, init, confirm, status -- answered by a capability, or +# routed to another network when the request is not ours +# /publish this network's catalogue system publishing inward +# /crawl/ operator endpoints: trigger a crawl, read its status +# +# Every module shares one identity, one keypair and one registry -- see the +# anchor below. Under one identity they are not three participants that happen +# to agree; they are one participant, and the config says so once. +appName: "__BECKN_NETWORK_ID__" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 9200 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +plugins: + # Ships metrics, traces and logs over OTLP/gRPC. All three flags false builds + # no exporter at all, which is the right setting for a stack with no collector. + otelsetup: + id: otelsetup + config: + serviceName: "__BECKN_NETWORK_ID__" + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + + # The crawler's OWN registry, separate from the modules' below. It is key + # distribution for verifying what the crawl fetches, not request handling, and + # the crawler refuses to start without it. + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + cacheTTL: 0 + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + maxResponseBytes: 1048576 + + # The crawler. One per deployment, app-level, because it is a background pass + # rather than something a request reaches. + crawler: + id: catalogcrawler + config: + dbDsn: "postgres://discovery:discovery@discovery-postgres:5432/catalogcrawler?sslmode=disable" + discoveryPushUrl: "http://discovery-service:8080/publish" + participantId: "__BECKN_NETWORK_ID__" + bppUri: "__NETWORK_BASE_URL__" + + # The federated half. Present means this network crawls what it has + # admitted; EMPTY means it crawls only its own providers -- it still + # ANSWERS another network's crawl, because that is the network module's + # job and nothing to do with this block. + # + # Rendered empty when FEDERATION_CRAWL_PEERS=off, which is how a + # deployment opts out of crawling without having to un-admit anyone. + federationNetworkId: "__FEDERATION_NETWORK_ID__" + federationSigningKey: "__SIGNING_PRIVATE__" + federationKeyId: "__KEY_ID__" + federationRefreshSeconds: "__FEDERATION_REFRESH_SECONDS__" + federationPageSize: "__FEDERATION_PAGE_SIZE__" + federationUpdateMode: "__FEDERATION_UPDATE_MODE__" + # What breadth of another network's catalogue to mirror. + # + # REQUIRED IN PRACTICE, though the code defaults it to an empty intent. + # Discovery refuses one that narrows nothing -- "an intent needs at least + # one of spatial or filters" for an empty one, and "this expression + # narrows nothing the index can serve" for a filter built to match all -- + # so a crawl with no intent configured silently mirrors zero catalogs. + # That is the whole of the long-standing "a published catalog never comes + # back from discover": the catalog was there, nothing ever asked for it. + # + # So a deployment must say what it wants. This asks for weather, which is + # what the mocks here publish; widen it per capability as more are + # published. + federationIntent: '__FEDERATION_INTENT__' + federationPublishRetrySeconds: "__FEDERATION_PUBLISH_RETRY_SECONDS__" + +modules: + + # ---------------------------------------------------------------- consumer + # + # What the experience layer posts to, and the only address it knows. It + # decides nothing: discovery goes to the network layer, everything + # transactional to the provider layer, and context.bppUri travels onward in + # the body untouched. + - name: consumer + path: /consumer/ + handler: + type: std + # bap because this module SENDS. role sets direction, not participant type. + role: bap + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + router: + id: router + config: + routingConfig: /app/config/routing-consumer.yaml + # No validateSign: the experience app is inside the trust boundary, so + # there is no network signature to check inbound. + steps: + - validateSchema + - addRoute + + # ----------------------------------------------------------------- network + # + # publish and discover. Routes both to this network's discovery service and + # enforces no payload shape of its own. + - name: network + path: /network/ + handler: + type: std + # bpp because this module RECEIVES. + role: bpp + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + router: + id: router + config: + routingConfig: /app/config/routing-network.yaml + steps: + - addRoute + + # ---------------------------------------------------------------- provider + # + # Where a select is answered -- or sent on, when it was addressed to another + # network. + - name: provider + path: /provider/ + handler: + type: std + role: bpp + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bpp + + # Only this module has one: a capability step turns an upstream's answer + # back into Beckn through a mapping file, and no other module calls an + # upstream. + mapper: + id: jsonmapper + config: + fetchTimeout: "10s" + cacheTTL: "1m" + negativeTTL: "1m" + maxMappingBytes: 262144 + maxCacheEntries: 200 + + providerSteps: + - id: WeatherObservation + config: + bindingKeys: "__MAUSAMGRAM_BINDING_KEY__" + # header, not basic: IMD issues one opaque token, and basic would + # build the header itself from a username and password pair. + __MAUSAMGRAM_PARTICIPANT_ID__: + authScheme: header + headerName: Authorization + headerValueEnv: MAUSAMGRAM_AUTH + + - id: MandiPrice + config: + bindingKeys: "__AGMARKNET_BINDING_KEY__" + # Agmarknet issues short-lived tokens. The field names are this + # provider's spelling, not a standard. tokenTtl is an estimate -- + # the response carries no expiry -- and a 401 drops the held token. + __AGMARKNET_PARTICIPANT_ID__: + authScheme: tokenQuery + tokenUrl: "__AGMARKNET_TOKEN_URL__" + tokenUserField: access_name + tokenUserEnv: AGMARKNET_ACCESS_NAME + tokenSecretField: password + tokenSecretEnv: AGMARKNET_PASSWORD + tokenResponseField: token + queryName: token + tokenTtl: 10m + + - id: KnowledgeAdvisory + config: + bindingKeys: "__VISTAAR_BINDING_KEY__" + __VISTAAR_PARTICIPANT_ID__: + authScheme: oauth2 + tokenUrl: "__VISTAAR_TOKEN_URL__" + clientIdEnv: VISTAAR_CLIENT_ID + clientSecretEnv: VISTAAR_CLIENT_SECRET + # A retrieval answer is larger than a price row: four passages + # measured 15.7 KB against the live corpus. + maxResponseBytes: 1048576 + + - id: AgricultureFacility + config: + bindingKeys: "__POCRA_BINDING_KEY__" + __POCRA_PARTICIPANT_ID__: + authScheme: none + # Must stay 1. POCRA answers 200 with an empty catalogue after + # roughly twenty calls in a few minutes. That is indistinguishable + # from a genuine "nothing nearby", so raising this risks telling a + # farmer there is no facility when there is. + searchConcurrency: 1 + + # Inbound signature validation is OFF: validateSign is absent, so this + # tier accepts unsigned traffic, and it answers unsigned because signAck + # is absent too. + # + # OUTBOUND signing is on: sign is listed, so a request forwarded to + # another network leaves with this network's signature. + steps: + - validateSchema + # There is no step here for "is this request ours to answer?". Each + # capability step asks it first, before its own binding check, so a + # deployment gets it by serving capabilities at all -- there is no list + # to leave it out of, and a fifth capability added below cannot miss it. + # + # Who we are comes from this module's subscriberId above; no step + # repeats it. A request for us is served exactly as before. One for + # anybody else is sent to its own context.bppUri, which the consumer + # copied from the catalog it chose. + # + - WeatherObservation + - MandiPrice + - KnowledgeAdvisory + - AgricultureFacility + # Signing is config, not code: a cross-network call leaves signed + # because the step is listed here. + # + # AFTER the capability steps, and that ordering matters. reqpreprocessor + # resolves this request's subscriber id FROM THE BODY, so on a request + # addressed to another network it resolves to THEM. The capability step + # above corrects it to us when it decides the request is not ours. + # Signing before that ran would look up the other network's keyset and + # fail with "failed to get signing key: keyset not found". + # + # It runs on every request, as every step does. On one that stays here + # that costs a signature nobody reads -- the header is set on the + # outbound request, and a request answered locally is never sent. + - sign + + # --------------------------------------------------------- provider publish + # + # This network's catalogue system publishing inward. A module of its own, not + # another action above: the routing step fails any action missing from its + # config, and listing select there would proxy it to the network layer instead + # of answering it locally. + - name: publish + # A TRAILING SLASH, so the action survives. Routing takes the action from + # what is left of the URL after the mount path is stripped, so an exact + # /publish mount leaves nothing and addRoute fails with "endpoint '' is not + # supported". Published to as POST /publish/publish. + path: /publish/ + handler: + type: std + # bap because this module SENDS. + role: bap + subscriberId: __BECKN_NETWORK_ID__ + plugins: + registry: + id: sunbirdRegistry + config: + url: http://registry-service:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + # 0 because no cache plugin is configured. A non-zero TTL here only warns + # at startup and caches nothing. + cacheTTL: 0 + # Plain seconds, per attempt. + timeout: 2 + retry_max: 1 + retry_wait_min: 100ms + retry_wait_max: 500ms + # A search response past this is refused, not truncated. + maxResponseBytes: 1048576 + + keyManager: + id: simplekeymanager + config: + subscriberId: __BECKN_NETWORK_ID__ + # The key's osid, not a friendly name: the registry indexes on it. + keyId: __KEY_ID__ + signingPrivateKey: "__SIGNING_PRIVATE__" + signingPublicKey: "__SIGNING_PUBLIC__" + encrPrivateKey: "__ENCR_PRIVATE__" + encrPublicKey: "__ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + # The extended layer is off: it fetches a resource's own @context per payload, + # and nothing here needs it. The settings below only apply if it is on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "raw.githubusercontent.com,schema.beckn.io,schema.nfh.global,openagrinet.github.io" + middleware: + - id: reqpreprocessor + config: + contextKeys: transaction_id,message_id + role: bap + router: + id: router + config: + routingConfig: /app/config/routing-publish.yaml + steps: + - validateSchema + - addRoute + + # ------------------------------------------------------------------- crawl + # + # Operator endpoints: /crawl/trigger, /crawl/peers, /crawl/status. + # + # Not published through the front door, which is what keeps them off the + # network -- the adapter has no signed-operator identity to check them against. + - name: crawl + path: /crawl/ + handler: + type: catalogCrawl + authDisabled: true + basePath: /crawl diff --git a/networkofnetwork/config/default.conf.template b/networkofnetwork/config/default.conf.template new file mode 100644 index 00000000..2108711b --- /dev/null +++ b/networkofnetwork/config/default.conf.template @@ -0,0 +1,152 @@ +# The network's front door. +# +# A network is known by its networkId, which is a domain. Everything another +# network needs therefore has to answer on that one name: the two published +# documents, and the Beckn endpoints. This server is what makes that true +# locally -- it is aliased to the networkId on the shared bridge, and it is the +# only container on it. +# +# It is also the whole of this network's public surface. The registry, the +# discovery service and the crawl endpoints are not reachable through it, which +# is what makes the boundary real rather than decorative. +# +# nginx:alpine runs envsubst over this file at start-up, so nothing below names +# a host or a port that is not supplied by the environment. + +server { + # TLS, because the contract says a network is reached over https and the + # registry's Participant schema pins baseUrl to ^https://. The certificate + # is signed by the local CA in networks/ca, and another network verifies + # against it. + # + # What this still cannot demonstrate is domain OWNERSHIP -- a laptop cannot + # -- which is why the CA is trusted by path rather than installed anywhere. + listen 443 ssl; + server_name ${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + # The two published documents, served as files. Static on purpose: the + # contract calls these publication points, not APIs, and a file is the + # smallest thing that satisfies it. + location /.well-known/ { + root /usr/share/nginx/html; + default_type application/json; + add_header Cache-Control "no-cache"; + } + + # Discovery: where another network's crawl sends its signed discover. + # + # This is the path the registry entry's baseUrl carries, so it is what a + # descriptor publishes as services.discovery and what admission records. + # + # A PROXY, not a second implementation. A signed discover from elsewhere + # reaches the same module a local caller reaches, or the two drift and only + # the federated path goes untested. + # + # Two locations because baseUrl cannot end in a slash -- the Participant + # schema's pattern refuses it -- so the exact form has to be matched as well + # as the subtree below it. + location = /network { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/network/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + location /network/ { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/network/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + + # The provider endpoints: where another network's select, init, confirm or + # status arrives, having been addressed here by a catalog's bppUri. + # + # The other direction of the same contract -- /network is how another + # network FINDS what we publish, this is how it then ACTS on it. + location = /provider { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/provider/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + location /provider/ { + proxy_pass http://${ADAPTER_HOST}:${ADAPTER_PORT}/provider/; + include /etc/nginx/conf.d/proxy-headers.inc; + } + + # Anything else is not part of the contract. 404 rather than a default page, + # so a misconfigured caller sees a refusal it can act on. + # + # /consumer, /publish and /crawl are deliberately among the "anything else": + # the first two are this deployment's own callers and the third is an + # operator endpoint, and none of them is another network's business. + location / { + return 404; + } +} + +# This network's mock upstream providers. +# +# SEPARATE server blocks, not more locations on the one above: these are +# different hosts, and the block above is this network's public surface -- +# another network's business. These are our own providers, reached only from +# inside. +# +# They are https on a hostname rather than http://mock-imd:9100 because the +# registry's Participant schema pins a provider's baseUrl to ^https:// exactly +# as it does a network's. The front door terminates TLS with the local CA the +# adapter already trusts, so the seeded URL is one the stack can actually call. +# +# One block each with a literal proxy_pass, rather than one block picking the +# upstream from $host: a proxy_pass carrying a variable is resolved per request +# and needs an explicit resolver, which is three more moving parts for two +# static destinations. +# +# Only the two providers that HAVE a mock. vistaar and pocra are in the +# certificate but have no container, so a call to them fails at the front door +# rather than at DNS. +server { + listen 443 ssl; + server_name mausamgram.${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + proxy_pass http://${MOCKIMD_HOST}:${MOCKIMD_INTERNAL_PORT}; + include /etc/nginx/conf.d/proxy-headers.inc; + } +} + +server { + listen 443 ssl; + server_name agmarknet.${NETWORK_ID}; + + ssl_certificate /etc/nginx/tls/server.crt; + ssl_certificate_key /etc/nginx/tls/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + + # The token endpoint the mock does not have. + # + # mockagmarknet checks for a FIXED token in the query string and issues + # none -- the real Agmarknet has a separate endpoint for that, and the mock + # reproduces only the checking half. So the adapter's tokenQuery scheme had + # a tokenUrl with nothing behind it and every mandi select failed at the + # exchange, before the provider was ever called. + # + # Answered here rather than by changing the mock: this is local scaffolding + # for a credential that is not real, and keeping it in the front door means + # the adapter still performs a genuine exchange over TLS rather than having + # the auth step stubbed out. + # + # local-mandi-token is mockagmarknet's own default, and the field name is + # what tokenResponseField in the adapter config asks for. + location = /token { + default_type application/json; + return 200 '{"token":"local-mandi-token","expires_in":600}'; + } + + location / { + proxy_pass http://${MOCKAGMARKNET_HOST}:${MOCKAGMARKNET_INTERNAL_PORT}; + include /etc/nginx/conf.d/proxy-headers.inc; + } +} diff --git a/networkofnetwork/config/init-crawler-db.sh b/networkofnetwork/config/init-crawler-db.sh new file mode 100755 index 00000000..f0427cd1 --- /dev/null +++ b/networkofnetwork/config/init-crawler-db.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# The crawler keeps its bookkeeping -- queues, cursors, staged catalogs -- in a +# database of its own on the discovery server. +# +# Created here, by postgres' own init hook, rather than by hand after the fact. +# Destroying a stack takes the volume with it, and without this the adapter +# restart-loops on "applying 0001_init.sql" against a database that is not there. +set -e +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE DATABASE catalogcrawler; +EOSQL diff --git a/networkofnetwork/config/proxy-headers.inc b/networkofnetwork/config/proxy-headers.inc new file mode 100644 index 00000000..4f354511 --- /dev/null +++ b/networkofnetwork/config/proxy-headers.inc @@ -0,0 +1,7 @@ +# Shared by every proxied location. +# +# The signature is over the body and the created/expires pair, so the body must +# arrive byte-identical. Buffering is fine; rewriting is not. +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_request_buffering on; diff --git a/networkofnetwork/config/registry/imports/realm-export.json b/networkofnetwork/config/registry/imports/realm-export.json new file mode 100644 index 00000000..0f5efa16 --- /dev/null +++ b/networkofnetwork/config/registry/imports/realm-export.json @@ -0,0 +1,2321 @@ +{ + "id": "sunbird-rc", + "realm": "sunbird-rc", + "displayName": "Sunbird Rc Core", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "a772a1cd-7904-4e5c-a864-5041fa69d491", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "42dba8cf-f483-4668-a087-cba46ed86ad2", + "name": "admin", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "5fa4077d-1686-4506-97a6-5bce1bce59dc", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "network_operator", + "description": "Network Operator - onboards and governs Providers", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "registryOperator", + "description": "Registry Operator - manages SchemaRegistry, Participant and ProviderSchema records", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "270ecc82-3249-475c-a851-d3ea162059b8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "0de39ec0-7602-4aa2-b54d-ab12e9bdb76f", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4259031b-736e-49eb-9e70-4a312a48e211", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9887e071-49b0-464b-b6fe-a1c585a709c7", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7d61f967-1dce-482f-96e5-9eff79eb4851", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-events", + "manage-identity-providers", + "manage-clients", + "view-identity-providers", + "manage-authorization", + "view-users", + "manage-users", + "manage-events", + "manage-realm", + "impersonation", + "view-authorization", + "query-clients", + "create-client", + "view-clients", + "query-users", + "query-realms", + "view-realm", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "e93b1761-fb32-46c5-bfa2-4b853c7b5573", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "bc32a137-07a1-40f0-b9fd-a6e64e27f99b", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4b5abd90-d6a2-4981-a50f-520292496f0b", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "99d2ed5f-00a9-44ed-8b9f-bdd7ba3facb8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "3fbd2cd5-0698-490e-a52f-ef528d001a62", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9b8b4f1c-5ed6-49ca-bec3-0a9a4867ad26", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7e6341ff-a1d8-4400-af94-3a007a06706a", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ad0c87da-9f34-4992-a83a-f6b924f1944d", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "badb0d4d-06da-45e8-a777-ef47f712d3ed", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "f8f48f0f-bd2a-4cb7-9b77-af69b9805c25", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ca0b1e94-6578-4295-abf4-ae99f7df7595", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ff7230eb-7dae-44a5-8f68-f68747f35589", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "890d054b-86f9-49f5-8dd9-14f62aa956de", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "627f3f4c-58e3-49f3-9989-a05d4d0a8752", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-api": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "2e9bfeff-129e-4072-9617-5847644aac24", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "attributes": {} + } + ], + "account": [ + { + "id": "5694c2d0-6d02-4182-bb09-78f4f5f1ec2d", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0612622f-dae7-48f8-8985-fe7e5ab8acc7", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "eeefbd57-94b8-4d7d-bf2f-075c39ccb746", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "9e9165b9-1170-47ab-802a-aecffefb3ab7", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "a8d0a100-e382-49ba-ac42-48dbf815a2de", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "08772792-146d-4676-ba2d-ce56b0104263", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0a2e7893-784e-47ef-ba35-4a26901350c0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + } + ], + "registry-frontend": [] + } + }, + "groups": [], + "defaultRole": { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "sunbird-rc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "3cc9ac60-b67d-4c57-8005-acd4d236b2dc", + "createdTimestamp": 1634296700339, + "username": "service-account-admin-api", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "admin-api", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-sunbird-rc", + "admin" + ], + "clientRoles": { + "realm-management": [ + "manage-users", + "manage-realm" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "username": "no-user", + "enabled": true, + "emailVerified": false, + "credentials": [ + { + "type": "password", + "value": "no-user-password", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-sunbird-rc", + "network_operator", + "registryOperator" + ] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "01326f76-7838-47fc-ae62-399a75c5ae38", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "f871d6fc-d997-4ac6-99fe-d797955bc9f0", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "8ab32c51-9aa0-4e28-80bf-0d6b53151354", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "25962708-6d45-47d9-8935-5db159234aac", + "clientId": "admin-api", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "*", + "http://localhost:4200/", + "http://localhost:4200/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "84ae9d6c-424f-47f0-9d4d-f2e98fed7339", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "98406938-b8db-4992-8519-917054f6ed0e", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "90d6b17a-5a06-4546-8091-960301f8147e", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b245b10b-606c-417c-bbc0-8f81a7a992a6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "961a8a92-1598-48ff-adee-1e5fee0df757", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "69c04ae8-6669-48e7-8234-08986a7f490d", + "clientId": "registry-frontend", + "name": "Registry Frontend", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "login_theme": "sunbird-rc", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b777b14f-b0e8-4da5-a802-092803319cbe", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/sunbird-rc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/sunbird-rc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "7160f35d-97d3-4730-9769-4b03b32e5191", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "b4695333-f842-4ef7-874e-99260e77b9cb", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "fc23d4b8-76c5-4e59-9305-10846b8bcefe", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "08f06ba5-3e60-4a0a-aaf9-f70bfc7ae99e", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e28227ee-cb54-4557-8908-01864f80055f", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "364a632f-b66a-4ca4-8bbe-ec2ce1af9df8", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "42cad815-4de0-4b67-abca-7f7aaf55e589", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "e3302def-d387-465c-a420-7ab01570e94a", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "63a3cb24-b124-428e-ac0f-253eb1fe485d", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2eb041ca-970a-45fd-a167-2a497579bc8c", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "2c02e9ce-7d86-4a5b-84b8-cf93114ddf26", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "2d39b55e-46c4-4dec-bd83-f081c708f544", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "e869fffd-f801-492d-a6c7-d6c6143817e5", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "daeb863b-4773-4668-98fb-403e93414eb2", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a98b9f93-ec39-4f3c-acb7-cd92161e3717", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c59a379a-3934-4e6f-be20-1803b0786d97", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0a299a91-277c-4f38-95e7-6c520f892b63", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "d51531b3-a8ea-44e2-a48f-69991f9166cc", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "b4b33a89-01db-468e-9a4e-c5ac58304fed", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "d1727e08-fb90-49ce-bb7e-d7a55a50ee64", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "564ae79d-e505-416c-b794-ddd3a3c21fde", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "986d2d9e-0d0d-4317-92b3-a7a8d9bec4de", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "e14ec2e9-0d24-4960-8779-00f769ccc01b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "8f044609-b615-4522-b9e7-8361cb08b0b3", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "341b838d-ba26-4280-b0af-3e5d3403c938", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "26a17e7e-1a6e-439f-a54a-05a63d1c91fb", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "2489b2c0-5b3a-4404-8428-be4ce653da72", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "8c7e1d96-bf79-42e6-9360-b5e7b8dddc8d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "01f959d8-123f-4263-ad6e-386e8b4d0e05", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "b47a30d6-3c49-4bbe-b15e-b0eb6cffc0f3", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "2cc8166d-6d77-4a85-9945-bc22b0f550e3", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "1d87a800-cc11-4d85-aa76-8a6d828e2269", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "42cc1538-f83a-4a94-b5a5-d16b80824a02", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "13ff6325-822e-4087-9e74-086de77fe89e", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e501527e-dec8-4fde-a539-8e77d86b5081", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "b4b519d8-070d-4dab-854e-d6e3b2b36205", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "ed42958b-6e78-42a9-9f40-2e40bd6c8dd0", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "saml-user-property-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "saml-role-list-mapper", + "oidc-full-name-mapper" + ] + } + }, + { + "id": "572219a7-3053-4940-87c5-ad94a6fb6dd3", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "8704a420-bf90-4e12-9e33-d21f39a2385b", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "8cf98455-916b-487a-8322-3f5d283400c2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "0b43488d-108b-41f5-ab6d-56a4ac8ff63c", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "6f0ebf9b-900a-4ca9-8fea-90719f218689", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b5a486b3-abf9-49a1-8dc6-dc5e20776681", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "be43420e-8ffc-4f53-b745-f2f0cd88f000", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f749bd77-72f2-4dc4-a65e-dd89b255f12f", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "a541cbb1-8a27-4061-a389-9f24ba1c2eb1", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "18504bf7-63f1-4848-b565-6348fa6b0048", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "497d8386-9a74-4b7b-a4e6-78bbbbb5d795", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "d964973c-2106-4db3-a814-f7a34ae7a1ce", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ff1b250-85b1-4709-8719-3eabcb34493f", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "43f683be-52e7-43cd-aa9e-6318b8079ad0", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "5195e46e-b2c3-49e3-8987-db8b19c45fc5", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "aca5480d-842c-4fa9-aff1-b8af8d51d82a", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "66a00ea9-7ec1-4450-905a-14b7f3f8e4bf", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "a24445a4-1988-4b5b-bde6-fa36dbd07e03", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "31c2bc3b-6eb1-4c4f-8464-3528f7445ef7", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4ccc8da9-0e1e-4f30-99c0-e2139f671a80", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "fa1183d1-af7a-40dd-ba85-d7d37867639c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "f5b5e49b-7cc9-4011-b9fa-60f0ef65e735", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5cf727d2-e25a-4c88-a55b-4eea9134adb1", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ec4f009-2f16-464b-8feb-a0bdc0dad195", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "a695a5e0-326f-4658-8518-a1769d97ad5f", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4611e0a9-a6a4-4e32-8500-e68877b464b1", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "55be727b-a17b-40c5-a5c3-c2d72c7f54cb", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "00ec3b72-3abb-4db3-ad2f-595bc2f7e086", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "82e4f3a3-744b-4d8a-8785-6eabaf9e05c9", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "0c0312b2-db7c-433c-ab15-20b18bfb5f4a", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ee0faa63-999c-42e6-8189-c22a5cc14dc5", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "506eed8f-88c9-4978-b13a-886f1efc45c0", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "14.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} diff --git a/networkofnetwork/config/registry/schemas/Participant.json b/networkofnetwork/config/registry/schemas/Participant.json new file mode 100644 index 00000000..fd8e5e93 --- /dev/null +++ b/networkofnetwork/config/registry/schemas/Participant.json @@ -0,0 +1,217 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Participant", + "type": "object", + "required": [ + "Participant" + ], + "properties": { + "Participant": { + "$ref": "#/definitions/Participant" + } + }, + "definitions": { + "Participant": { + "description": "Someone the network deals with. `type` says which kind, and decides which of the remaining fields apply: a node speaks Beckn and is addressed by its participantId; an upstream is an ordinary API our adapter calls. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": [ + "participantId", + "name", + "type", + "status", + "baseUrl" + ], + "properties": { + "participantId": { + "$ref": "#/definitions/ParticipantId" + }, + "name": { + "description": "Human label.", + "type": "string", + "minLength": 1, + "maxLength": 200, + "pattern": "\\S" + }, + "type": { + "$ref": "#/definitions/ParticipantType" + }, + "status": { + "$ref": "#/definitions/Status" + }, + "baseUrl": { + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https for a node, since that is its wire identity; an upstream may be plaintext, because nothing in the registry is sent to it.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + }, + "role": { + "description": "What this party does on the network, as a list. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs. A network is ONE participant however many adapters it runs, so this says what the network serves rather than which process exists -- a split three-adapter deployment and a consolidated one carry the same value. Required on a node. Permitted on an upstream, where nothing reads it. Search it with `contains`, not `eq`: an exact match against a list finds nothing, while `contains` matches both a list and an older single-string value.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "consumer", + "provider", + "network" + ] + } + }, + "keys": { + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header. Required on a node, since a node's signatures are verified against them. Permitted on an upstream, which does not sign anything this stack verifies.", + "type": "array", + "minItems": 1, + "maxItems": 8, + "uniqueItems": true, + "items": { + "$ref": "#/definitions/PublicKey" + } + }, + "revision": { + "description": "The revision of the peer's descriptor this record was built from. Only ever increases: a descriptor offering a revision at or below the stored one is a replay and is refused, which is what stops a withdrawn or superseded declaration being presented as current. Applies to role network.", + "type": "integer", + "minimum": 1 + }, + "federationRole": { + "description": "What this entry is, in relation to us. self is this deployment's own entry -- the network reading it. source is a peer we consume from: our crawler asks it for its catalogs. requester is a peer that consumes from us: it calls us and we verify its signatures, but we do not crawl it. both is a peer in both directions, which is what mutual admission produces. STORED rather than worked out: deciding whether an entry is our own by comparing its id against configuration has been got wrong twice, most memorably when a crawler tried to crawl itself. Absent on an entry written before this field existed, which is read as both. Applies to role network.", + "type": "string", + "enum": [ + "self", + "source", + "requester", + "both" + ] + }, + "schemaPacks": { + "description": "The schema packs a peer network declared in its descriptor, stored verbatim. Recorded rather than interpreted: what a peer can speak is its statement, and rewriting it here would make our copy disagree with the document it was read from. The crawler derives the JSON-LD context URL it filters on from these, because a pack URL names the pack's attributes.yaml while a message is interpreted against the matching context.jsonld. Applies to role network.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "projectionTtl": { + "description": "How long, in seconds, this peer permits us to keep what we crawled from it. Declared by the peer in its descriptor and recorded here verbatim: the network that owns the data decides how stale a copy of it may get, and a crawler that chose its own value would be deciding that for someone else. Always positive on a stored record -- a peer declaring 0 forbids caching entirely and is refused at admission rather than cached against its wishes. Applies to role network.", + "type": "integer", + "minimum": 1 + } + }, + "allOf": [ + { + "description": "A node speaks Beckn: it needs a role and keys, and its id is its wire identity so it must be a hostname.", + "if": { + "properties": { + "type": { + "const": "node" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "role", + "keys" + ], + "properties": { + "participantId": { + "maxLength": 253, + "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$" + }, + "baseUrl": { + "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + } + } + } + } + ] + }, + "PublicKey": { + "description": "A public key, held as material: it is public, so there is nothing to protect. Identified by the osid the registry assigns on write -- which is what a sender names in the Authorization header, and so the only id a verifier can look up. alg carries the purpose: ed25519 signs, x25519 encrypts.", + "type": "object", + "additionalProperties": true, + "required": [ + "alg", + "key", + "validFrom", + "status" + ], + "properties": { + "alg": { + "description": "The purpose, and the only thing that carries it now: ed25519 signs, x25519 encrypts. Both curves are 32-byte keys.", + "type": "string", + "enum": [ + "ed25519", + "x25519" + ] + }, + "key": { + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. Bare -- no encoding label in front of it. A truncated or wrong-curve key fails at write time.", + "type": "string", + "pattern": "^[A-Za-z0-9+/]{43}=$" + }, + "validFrom": { + "type": "string", + "format": "date-time" + }, + "validUntil": { + "description": "Absent means open-ended. Overlap with the next key's validFrom is the rotation window.", + "type": "string", + "format": "date-time" + }, + "status": { + "description": "revoked withdraws the key early. Effective only once the reader refreshes.", + "type": "string", + "enum": [ + "active", + "revoked" + ] + } + } + }, + "ParticipantId": { + "description": "Stable id, and the only id. For a node this is its network identity -- what field 1 of the Authorization header names, and what a signature is verified against; for an upstream it is the Beckn offer.provider.id.", + "type": "string", + "maxLength": 253, + "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" + }, + "Status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "ParticipantType": { + "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", + "type": "string", + "enum": [ + "node", + "upstream" + ] + } + }, + "_osConfig": { + "uniqueIndexFields": [ + "participantId" + ], + "indexFields": [ + "status", + "type", + "baseUrl" + ], + "privateFields": [], + "roles": [ + "registryOperator" + ], + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ] + } +} diff --git a/networkofnetwork/config/registry/schemas/ProviderSchema.json b/networkofnetwork/config/registry/schemas/ProviderSchema.json new file mode 100644 index 00000000..5744c682 --- /dev/null +++ b/networkofnetwork/config/registry/schemas/ProviderSchema.json @@ -0,0 +1,81 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ProviderSchema", + "type": "object", + "required": ["ProviderSchema"], + "properties": { "ProviderSchema": { "$ref": "#/definitions/ProviderSchema" } }, + + "definitions": { + + "ProviderSchema": { + "description": "One row is one provider and one capability. What varies per Beckn action — the URL, the method, the mapping, the timeout — varies inside actions[]. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": ["bindingKey", "participantId", "capabilityCode", "status", "actions"], + "properties": { + "bindingKey": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{2,63}\\|openagrinet:[A-Z][A-Za-z0-9]*$" + }, + "participantId": { "$ref": "#/definitions/ParticipantId" }, + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "status": { "$ref": "#/definitions/Status" }, + + "actions": { + "description": "At least one. uniqueItems compares whole objects, so it cannot pin one entry per action — verify/records.py does.", + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { "$ref": "#/definitions/ActionBinding" } + } + } + }, + + "ActionBinding": { + "description": "How to call this provider for one Beckn action: where, how, and with which mapping. Anything the upstream needs that the Beckn body cannot express is the adapter plugin's work, not a field here. status is per action, so one can be retired without touching the others. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", + "type": "object", + "additionalProperties": true, + "required": ["action", "method", "path", "mappings", "status"], + "properties": { + "action": { "$ref": "#/definitions/Action" }, + "method": { "type": "string", "enum": ["GET", "POST"] }, + "path": { "$ref": "#/definitions/Path" }, + "mappings": { "$ref": "#/definitions/MappingPath" }, + + "timeoutMs": { "type": "integer", "minimum": 1000, "maximum": 120000, "default": 15000 }, + "retryMax": { "type": "integer", "minimum": 0, "maximum": 5, "default": 0 }, + + "status": { "$ref": "#/definitions/Status" } + } + }, + + "Action": { + "description": "beckn v2.0.0 request actions. An on_* callback is not one: it is the response half of the action that made the call.", + "type": "string", + "enum": ["discover", "select", "init", "confirm", "status", + "track", "cancel", "update", "rate", "support"] + }, + + "MappingPath": { + "description": "Fully-qualified URL of one published mapping file, holding request: and response: as YAML block scalars. The action segment must equal the action it sits under — verify/records.py.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*/[A-Za-z0-9._~%-]*\\.(discover|select|init|confirm|status|track|cancel|update|rate|support)\\.ya?ml$" + }, + + "ParticipantId": { "type": "string", "maxLength": 253, "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" }, + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, + "Status": { "type": "string", "enum": ["active", "inactive"] }, + "Path": { "type": "string", "maxLength": 512, + "description": "Appended to that upstream's baseUrl. Single slashes: an empty segment is never deliberate, and many servers answer //a differently from /a. A trailing slash is allowed, because /api/ and /api are a distinction some APIs make.", + "pattern": "^(?!.*//)/[A-Za-z0-9._~%/-]*$" } + }, + + "_osConfig": { + "uniqueIndexFields": ["bindingKey"], + "indexFields": ["participantId", "capabilityCode", "status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/networkofnetwork/config/registry/schemas/SchemaRegistry.json b/networkofnetwork/config/registry/schemas/SchemaRegistry.json new file mode 100644 index 00000000..0efd0f1b --- /dev/null +++ b/networkofnetwork/config/registry/schemas/SchemaRegistry.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SchemaRegistry", + "type": "object", + "required": ["SchemaRegistry"], + "properties": { "SchemaRegistry": { "$ref": "#/definitions/SchemaRegistry" } }, + + "definitions": { + + "SchemaRegistry": { + "type": "object", + "additionalProperties": false, + "required": ["capabilityCode", "name", "version", "schemaUrl", "status"], + "properties": { + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "name": { "type": "string", "minLength": 1, "maxLength": 200, "pattern": "\\S" }, + "version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+$" }, + "schemaUrl": { "type": "string", "maxLength": 2000, + "pattern": "^https://openagrinet\\.github\\.io/network-specs/schema/[A-Za-z0-9]+/v[0-9]+\\.[0-9]+/[A-Za-z0-9_-]+\\.yaml$" }, + "status": { "$ref": "#/definitions/Status" } + } + }, + + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, + "Status": { "type": "string", "enum": ["active", "inactive"] } + }, + + "_osConfig": { + "uniqueIndexFields": ["capabilityCode"], + "indexFields": ["status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/networkofnetwork/config/routing-consumer.yaml b/networkofnetwork/config/routing-consumer.yaml new file mode 100644 index 00000000..64ebfb41 --- /dev/null +++ b/networkofnetwork/config/routing-consumer.yaml @@ -0,0 +1,37 @@ +# What the consumer layer does with each action. +# +# Both targets are THIS SAME adapter on another path. In the three-adapter +# quick-start they were two other containers; consolidated, the hop is a loopback +# call from one module to another. It stays a real HTTP hop rather than an +# in-process call because that is what makes the two shapes the same config: +# split these across containers and only the host in these URLs changes. +# +# targetType "url" appends the action to the path, so the base here is the module +# mount point and /discover or /select is added to it. +routingRules: + # Discovery is answered from this network's own discovery service, which the + # network module fronts. That includes catalogs crawled from other networks -- + # they were mirrored in, so a local query returns both. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network" + endpoints: + - discover + + # Everything transactional goes to the provider layer, and this layer decides + # nothing about where it ends up. Whether the request is ours to answer or + # belongs to another network is decided in ONE place -- routeByReceiver on the + # provider module, which is also where the capability steps that would + # otherwise wrongly answer it live. + # + # context.bppUri travels onward in the body untouched. + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/provider" + endpoints: + - select + - init + - confirm + - status diff --git a/networkofnetwork/config/routing-network.yaml b/networkofnetwork/config/routing-network.yaml new file mode 100644 index 00000000..4b8f5be3 --- /dev/null +++ b/networkofnetwork/config/routing-network.yaml @@ -0,0 +1,10 @@ +# The network layer: publish and discover both go to this network's discovery +# service, which is the only thing behind this module. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://discovery-service:8080" + endpoints: + - discover + - publish diff --git a/networkofnetwork/config/routing-publish.yaml b/networkofnetwork/config/routing-publish.yaml new file mode 100644 index 00000000..0472bf44 --- /dev/null +++ b/networkofnetwork/config/routing-publish.yaml @@ -0,0 +1,12 @@ +# This network's catalogue system publishing inward. +# +# It goes to the NETWORK module rather than straight to discovery, so a publish +# takes the same path a publish from anywhere else takes -- one place that +# decides what reaches discovery, rather than two. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://localhost:9200/network" + endpoints: + - publish diff --git a/networkofnetwork/demo/README.md b/networkofnetwork/demo/README.md new file mode 100644 index 00000000..6c6e6391 --- /dev/null +++ b/networkofnetwork/demo/README.md @@ -0,0 +1,296 @@ +# Network of networks — demo runbook + +Two whole OAN networks on one laptop, able to find and call each other. + +Every command and every response below was run end to end on 2026-10-07, from +an empty machine. Timings are what it actually took. + +Ports: `bharatvistar` gets 9300-9306, `mahavistara` 9400-9406. + +**Postman:** import `demo/oan-network-of-networks.postman_collection.json`. Two +folders, Bharat and Maha, with every request below already filled in. + +**B does not crawl.** `FEDERATION_CRAWL_PEERS=off` in its `.env`. With both +networks crawling, each mirrors the other, and B ends up re-importing its own +catalogue through A's copy of it -- and could overwrite its own with a stale +one. B still ANSWERS A's crawl; that is the network module, nothing to do with +this switch. And A's mandi is published `visibleTo` A alone, so there is +nothing of A's for B to take even if it did crawl. The script +allocates the next free block per network, so a third network just works. + +--- + +## Before you start + +Docker, Python 3, `openssl`, `curl`. The adapter image must exist locally: + +```bash +cd ~/Workspace/OAN/network-adapter +docker build --network=host -f Dockerfile.adapter-with-plugins -t oan/network-adapter:local . +``` + +`--network=host` is not optional on this machine — IPv6 is broken here and +`go mod download` hangs without it. Takes about 4 minutes cold. + +Everything else runs from `helmcharts/networkofnetwork`. + +--- + +## 1. Stand up two networks (~75s each) + +```bash +cd ~/Workspace/OAN/helmcharts/networkofnetwork + +make network NETWORK_ID=bharatvistar.oan.local +make network NETWORK_ID=mahavistara.oan.local +``` + +Each one brings up its own registry, Keycloak, discovery service, Postgres, two +mock providers, an adapter and a front door — then seeds the registry and +renders the adapter config. Re-running it is also how you apply a config +change. + +Check what is deployed: + +```bash +make ls +``` + +``` +bharatvistar.oan.local 9300-9306 +mahavistara.oan.local 9400-9406 +``` + +--- + +## 2. Each network hosts its own identity (instant) + +```bash +make host-descriptor NETWORK_ID=bharatvistar.oan.local +make host-descriptor NETWORK_ID=mahavistara.oan.local +``` + +``` +bharatvistar.oan.local now hosts its descriptor at revision 1 + descriptor networks/bharatvistar.oan.local/.well-known/openagrinet + keys networks/bharatvistar.oan.local/.well-known/jwks.json + governance + operational +``` + +Two signed documents per network, served over TLS at the network's own domain: +`/.well-known/openagrinet` (the Network Descriptor) and `/.well-known/jwks.json`. + +- **Signature:** Ed25519 (`"algorithm": "Ed25519"`, RFC 9864 name) over the + RFC 8785 (JCS) form of the descriptor minus its `signature`. Floats are + refused, so every implementation rebuilds the same bytes. +- **Two kinds of key**, as the contract requires. The **governance** key + (`keys/governance.key`, generated on first run, mode 600) signs the descriptor + and never traffic. The **operational** keys are the registry entry's active + keys -- the ones the adapters sign traffic with. The descriptor delegates them + by kid in `operationalKeys`, under the governance signature. +- **kid** is each key's RFC 7638 thumbprint, so a rotated key gets a new kid on + its own. To rotate an operational key, add it to the registry entry, re-run + `host-descriptor`, and revoke the old one once peers have refreshed. +- **revision** must increase: re-running bumps it, and `oanfed publish` refuses + one at or below what is already published. The JWKS is written before the + descriptor, each in one atomic step. + +Peers store only the delegated operational keys on `admit`, so the governance +key can never pass their traffic check. + +To check it end to end -- publish, fetch over TLS as the peer would, then +tamper with the served files and confirm each forgery is refused: + +```bash +python3 -m unittest bin/test_oanfed.py +make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +``` + +The second restores whatever the network served before and writes nothing to +either registry. + +--- + +## 3. Each network admits the other (instant) + +```bash +make admit NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local +make admit NETWORK_ID=mahavistara.oan.local PEER=bharatvistar.oan.local +``` + +``` +admitted mahavistara.oan.local at revision 1, status active + discovery https://mahavistara.oan.local/network/discover + projectionTtl 3600s + keys 1 published + relationship both +``` + +This fetches the other network's descriptor over TLS, verifies its signature +against the key it publishes, and records the decision to deal with it. Nothing +is trusted because it is on the same machine. + +--- + +## 4. Give each network different data + +This is the story: **A has only mandi prices, B has only weather.** Neither is +missing a capability — both adapters serve all four — the difference is what +each one has PUBLISHED. + +```bash +# A publishes mandi +curl -s -X POST localhost:9300/publish/publish \ + -H 'Content-Type: application/json' -d @demo/publish-catalog-mandi.json | jq . + +# B publishes weather +curl -s -X POST localhost:9400/publish/publish \ + -H 'Content-Type: application/json' -d @demo/publish-catalog.json | jq . +``` + +Both answer `ACCEPTED`. Each catalog is published `visibleTo` both networks, and +carries its own `bppId`/`bppUri` — catalog fields, which is how a consumer later knows where to +send a select. + +--- + +## 5. A crawls B + +```bash +curl -s -X POST localhost:9300/crawl/peers +``` + +No arguments. The crawler reads the registry: it lists the ACTIVE participants +with role `network`, drops its own entry, and takes each peer's discovery URL, +`projectionTtl` and `schemaPacks` from the record `make admit` wrote. So "who do +I crawl" is the set of networks you have admitted, and nothing else. + +It also re-crawls on its own every `federationRefreshSeconds` (900s here), so in +a long demo this happens again without being asked. + +``` +catalogcrawler: crawled a network networkId=mahavistara.oan.local catalogs=1 +``` + +--- + +## 6. The farmer asks ONE network and gets both + +```bash +curl -s -X POST localhost:9301/discover \ + -H 'Content-Type: application/json' -d @demo/discover-farmer.json | jq . +``` + +``` +catalog:mandi-price:agmarknet.bharatvistar -> select at bharatvistar.oan.local +catalog:weather-observation:mausamgram.mahavistara -> select at mahavistara.oan.local +``` + +One query, against network A only. A's own mandi catalogue, and B's weather +catalogue which A crawled. The farmer never named a network and never knew there +were two. + +The payload carries no `context.networkId`. An omitted audience means no +audience filter at all, so the farmer sees everything A holds — its own and its +mirrored copies alike, without having to know that either exists. + +**Use the no-networkId form for the demo.** Naming an audience filters to +catalogs published `visibleTo` that audience, and a crawled copy is stamped +with the audience the crawl asked with — which is not reliably ours: + +``` +ask A with no networkId -> mandi-price, weather-observation +ask A as mahavistara.oan.local -> mandi-price, weather-observation +ask A as bharatvistar.oan.local -> mandi-price (the mirror is missing) +``` + +The crawl asks the peer twice — once as the peer, once as us — and ours goes +last so our copy should win. Both calls are made (the peer's nginx log shows +two POSTs), but the second comes back empty while the same request sent by hand +returns the catalog. Unresolved; see section 3 of the todo. It does not affect +the farmer path, which names no audience. + +--- + +## 7. Select each one — one local, one across the network + +The consumer picks a catalog and copies its `bppId` into the select's +`context.receiverId`, and its `bppUri` into `context.bppUri`. Nothing else +decides where the request goes. + +(The context names the two ends by DIRECTION — `senderId` and `receiverId` — +not by Beckn role. `bapId`/`bppId` are the older spelling and appear in no +payload here. `bppUri` stays, because it addresses the PROVIDER a catalog is +served from rather than saying who sent the envelope.) + +**Mandi — `receiverId` is A, so A answers it itself:** + +```bash +curl -s -X POST localhost:9300/provider/select \ + -H 'Content-Type: application/json' -d @demo/select-mandi-local.json | jq . +``` + +Answers `on_select` from `agmarknet.bharatvistar`, with prices from A's own mock. + +**Weather — `receiverId` is B, so A sends it to B:** + +```bash +curl -s -X POST localhost:9300/provider/select \ + -H 'Content-Type: application/json' -d @demo/select-cross-network.json | jq . +``` + +Answers `on_select` from `mausamgram.mahavistara` — the OTHER network's provider, +with three days of forecast. + +Both requests arrive at the SAME endpoint on the SAME network. The only +difference is the `receiverId` the farmer's consumer copied out of the catalog. +In +A's log: + +``` +mahavistara.oan.local is not bharatvistar.oan.local; sending it to http://... +Signature generated: ... +Routing to ctx.Route to &model.Route{TargetType:"url", ...} +``` + +A recognised the request was not its own to answer, signed it as **itself**, and +sent it to the address the catalog carried. Nothing in A is configured with who +its peers are. + +--- + +## Tear down + +```bash +make down NETWORK_ID=bharatvistar.oan.local # stop, keep the data +make destroy NETWORK_ID=bharatvistar.oan.local # delete the data, free the ports + +make teardown # destroy EVERY network, asks first +``` + +--- + +## Known rough edges — say these before someone finds them + +| | | +|---|---| +| `federationIntent` decides what breadth of a peer's catalogue is mirrored, and is set to weather here. Discovery refuses an intent that narrows nothing, so there is no "mirror everything". | | +| Only `mausamgram` and `agmarknet` have mock containers. `vistaar` and `pocra` are in the certificate but have nothing behind them. | | +| `validateSign` is in no step chain, so a cross-network select is answered whether or not it is signed. | | +| Nothing re-reads a peer's descriptor on a schedule — `admit` is also the refresh, and a person has to run it. | | +| Nothing removes a mirrored catalog. Refresh happens; withdrawal does not, because discovery has no delete endpoint. | | +| A select routes on `receiverId` alone. Any id a caller invents is enough, and the address comes from the same request. | | + +--- + +## If something breaks + +| symptom | cause | +|---|---| +| adapter restart-loops on `applying 0001_init.sql` | a `destroy` took the crawler database with it. `CREATE DATABASE catalogcrawler` by hand, or destroy and bring it back up. | +| a rebuilt image seems to have no effect | `docker restart` reuses the container's existing layer. Re-run `make network NETWORK_ID=...`, which recreates. | +| `failed to get signing key: keyset not found` on a cross-network select | `sign` is running before the capability steps. It must come after them. | +| `endpoint '' is not supported` | a module mounted at an exact path with no trailing slash — routing reads the action from what is left of the URL. | +| a crawl returns `catalogs=0` | the intent, or the schema packs. Both are silent: discovery answers 200 with nothing. | +| a peer's front door answers as yourself | two stacks on one bridge share a DNS alias. Address containers by their project-prefixed names. | diff --git a/networkofnetwork/demo/discover-farmer.json b/networkofnetwork/demo/discover-farmer.json new file mode 100644 index 00000000..d2644706 --- /dev/null +++ b/networkofnetwork/demo/discover-farmer.json @@ -0,0 +1,19 @@ +{ + "context": { + "action": "discover", + "domain": "agriculture", + "messageId": "5e511275-d9ed-442a-964c-b4f4cc574c1b", + "timestamp": "2026-10-07T03:23:01Z", + "transactionId": "df14c457-d091-48ad-8bbc-6214001f0578", + "version": "2.0.0", + "senderId": "bharatvistar.oan.local" + }, + "message": { + "intent": { + "filters": { + "type": "jsonpath", + "expression": "$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \"Market\" || @.resourceAttributes.subjectCategories[*] == \"Weather\")" + } + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/discover.json b/networkofnetwork/demo/discover.json new file mode 100644 index 00000000..a7a1468a --- /dev/null +++ b/networkofnetwork/demo/discover.json @@ -0,0 +1,20 @@ +{ + "context": { + "action": "discover", + "domain": "agriculture", + "messageId": "5e511275-d9ed-442a-964c-b4f4cc574c1b", + "networkId": "mahavistara.oan.local", + "timestamp": "2026-10-07T03:23:01Z", + "transactionId": "df14c457-d091-48ad-8bbc-6214001f0578", + "version": "2.0.0", + "senderId": "bharatvistar.oan.local" + }, + "message": { + "intent": { + "filters": { + "expression": "$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \"Weather\")", + "type": "jsonpath" + } + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/oan-network-of-networks.postman_collection.json b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json new file mode 100644 index 00000000..3d8fed2f --- /dev/null +++ b/networkofnetwork/demo/oan-network-of-networks.postman_collection.json @@ -0,0 +1,570 @@ +{ + "info": { + "_postman_id": "d90ecfa3-df76-4844-a51c-32d73de9a2f9", + "name": "OAN — Network of Networks", + "description": "Two OAN networks side by side, each with its own registry, discovery and adapter.\n\n**Identity documents (#74):** run the *Identity documents* folder first -- each network's signed Descriptor and JWKS.\n\n**The demo, in order:**\n1. Bharat 1 — publish mandi into A\n2. Bharat 2 — search A, returns mandi only\n3. Maha 1 — publish weather into B\n4. wait ~30s, no manual crawl\n5. Bharat 2 again — now returns mandi AND weather\n6. Bharat 3 — select mandi, answered locally\n7. Bharat 4 — select weather, crosses to B\n\nBring the stack up with `make network NETWORK_ID=...` in helmcharts/networkofnetwork. Full runbook in demo/README.md.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "bharat_adapter", + "value": "http://localhost:9300" + }, + { + "key": "bharat_discovery", + "value": "http://localhost:9301" + }, + { + "key": "maha_adapter", + "value": "http://localhost:9400" + }, + { + "key": "maha_discovery", + "value": "http://localhost:9401" + }, + { + "key": "bharat_front_door", + "value": "https://localhost:9306" + }, + { + "key": "maha_front_door", + "value": "https://localhost:9406" + }, + { + "key": "bharat_network_id", + "value": "bharatvistar.oan.local" + }, + { + "key": "maha_network_id", + "value": "mahavistara.oan.local" + } + ], + "item": [ + { + "name": "Identity documents (issue #74)", + "description": "Each network's two published documents, fetched from its front door exactly where a peer reads them.\n\n**Before running:** Postman → Settings → turn OFF *SSL certificate verification* (the stack uses a local CA), and run `make host-descriptor NETWORK_ID=...` for each network with this branch's code.\n\nRun each Descriptor request before its JWKS request: the JWKS checks use the kids the Descriptor named.\n\nPostman cannot verify an Ed25519 signature, so these check shape, keys and the governance/traffic key split. For the signature and the tampering cases run `make e2e-descriptor NETWORK_ID=... PEER=...`.", + "item": [ + { + "name": "1. Bharat — Network Descriptor", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'bharat');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "pm.test('not cached stale (Cache-Control: no-cache)', () => {", + " pm.expect(pm.response.headers.get('Cache-Control') || '').to.include('no-cache');", + "});", + "const d = pm.response.json();", + "pm.test('every required field of the Descriptor contract', () => {", + " ['version','networkId','name','operator','revision','status','issuedAt',", + " 'operatingProfile','schemaPacks','services','operationalKeys','signature']", + " .forEach(f => pm.expect(d, f).to.have.property(f));", + " pm.expect(d.version).to.eql('1.0');", + " pm.expect(d.operator.name).to.be.a('string').and.not.empty;", + " pm.expect(d.revision).to.be.a('number').and.above(0);", + " pm.expect(['Active','Suspended','Retired']).to.include(d.status);", + " pm.expect(d.issuedAt).to.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$/);", + " pm.expect(d.operatingProfile).to.be.an('array').that.is.not.empty;", + "});", + "pm.test('networkId is the network this was fetched for', () => {", + " pm.expect(d.networkId).to.eql(pm.variables.get(prefix + '_network_id'));", + "});", + "pm.test('discovery URL and schema packs are https', () => {", + " pm.expect(d.services.discovery).to.match(/^https:\\/\\//);", + " d.schemaPacks.forEach(u => pm.expect(u).to.match(/^https:\\/\\//));", + "});", + "pm.test('signed with Ed25519 by a governance key that is NOT a traffic key', () => {", + " pm.expect(d.signature.algorithm).to.eql('Ed25519');", + " pm.expect(d.signature.value).to.match(/^[A-Za-z0-9_-]{86}$/);", + " pm.expect(d.operationalKeys).to.be.an('array').that.is.not.empty;", + " pm.expect(d.operationalKeys).to.not.include(d.signature.keyId);", + "});", + "pm.collectionVariables.set(prefix + '_governance_kid', d.signature.keyId);", + "pm.collectionVariables.set(prefix + '_operational_kids', JSON.stringify(d.operationalKeys));" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_front_door}}/.well-known/openagrinet", + "host": [ + "{{bharat_front_door}}" + ], + "path": [ + ".well-known", + "openagrinet" + ] + }, + "description": "The signed ID card: who the network is, its status, revision, versions, schema packs, discovery URL and which keys may sign traffic." + } + }, + { + "name": "2. Bharat — JWKS", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'bharat');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "const keys = pm.response.json().keys;", + "pm.test('every key is a 32-byte Ed25519 signing key with a unique kid', () => {", + " pm.expect(keys).to.be.an('array').that.is.not.empty;", + " const kids = keys.map(k => k.kid);", + " pm.expect(new Set(kids).size).to.eql(kids.length);", + " keys.forEach(k => {", + " pm.expect(k.kty).to.eql('OKP');", + " pm.expect(k.crv).to.eql('Ed25519');", + " pm.expect(k.alg).to.eql('Ed25519');", + " pm.expect(k.use).to.eql('sig');", + " pm.expect(k.x).to.match(/^[A-Za-z0-9_-]{43}$/);", + " });", + "});", + "pm.test('publishes the governance key and every delegated traffic key', () => {", + " const gov = pm.collectionVariables.get(prefix + '_governance_kid');", + " const ops = JSON.parse(pm.collectionVariables.get(prefix + '_operational_kids') || '[]');", + " pm.expect(gov, 'run the Descriptor request first').to.be.a('string');", + " const kids = keys.map(k => k.kid);", + " pm.expect(kids).to.include(gov);", + " ops.forEach(kid => pm.expect(kids).to.include(kid));", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_front_door}}/.well-known/jwks.json", + "host": [ + "{{bharat_front_door}}" + ], + "path": [ + ".well-known", + "jwks.json" + ] + }, + "description": "The public keys that verify the Descriptor (governance) and the delegated traffic keys (operational)." + } + }, + { + "name": "3. Maha — Network Descriptor", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'maha');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "pm.test('not cached stale (Cache-Control: no-cache)', () => {", + " pm.expect(pm.response.headers.get('Cache-Control') || '').to.include('no-cache');", + "});", + "const d = pm.response.json();", + "pm.test('every required field of the Descriptor contract', () => {", + " ['version','networkId','name','operator','revision','status','issuedAt',", + " 'operatingProfile','schemaPacks','services','operationalKeys','signature']", + " .forEach(f => pm.expect(d, f).to.have.property(f));", + " pm.expect(d.version).to.eql('1.0');", + " pm.expect(d.operator.name).to.be.a('string').and.not.empty;", + " pm.expect(d.revision).to.be.a('number').and.above(0);", + " pm.expect(['Active','Suspended','Retired']).to.include(d.status);", + " pm.expect(d.issuedAt).to.match(/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$/);", + " pm.expect(d.operatingProfile).to.be.an('array').that.is.not.empty;", + "});", + "pm.test('networkId is the network this was fetched for', () => {", + " pm.expect(d.networkId).to.eql(pm.variables.get(prefix + '_network_id'));", + "});", + "pm.test('discovery URL and schema packs are https', () => {", + " pm.expect(d.services.discovery).to.match(/^https:\\/\\//);", + " d.schemaPacks.forEach(u => pm.expect(u).to.match(/^https:\\/\\//));", + "});", + "pm.test('signed with Ed25519 by a governance key that is NOT a traffic key', () => {", + " pm.expect(d.signature.algorithm).to.eql('Ed25519');", + " pm.expect(d.signature.value).to.match(/^[A-Za-z0-9_-]{86}$/);", + " pm.expect(d.operationalKeys).to.be.an('array').that.is.not.empty;", + " pm.expect(d.operationalKeys).to.not.include(d.signature.keyId);", + "});", + "pm.collectionVariables.set(prefix + '_governance_kid', d.signature.keyId);", + "pm.collectionVariables.set(prefix + '_operational_kids', JSON.stringify(d.operationalKeys));" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_front_door}}/.well-known/openagrinet", + "host": [ + "{{maha_front_door}}" + ], + "path": [ + ".well-known", + "openagrinet" + ] + }, + "description": "Same as Bharat, for Maha Vistara." + } + }, + { + "name": "4. Maha — JWKS", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.variables.set('net', 'maha');" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const prefix = pm.variables.get('net');", + "pm.test('200 and JSON', () => {", + " pm.response.to.have.status(200);", + " pm.expect(pm.response.headers.get('Content-Type')).to.include('application/json');", + "});", + "const keys = pm.response.json().keys;", + "pm.test('every key is a 32-byte Ed25519 signing key with a unique kid', () => {", + " pm.expect(keys).to.be.an('array').that.is.not.empty;", + " const kids = keys.map(k => k.kid);", + " pm.expect(new Set(kids).size).to.eql(kids.length);", + " keys.forEach(k => {", + " pm.expect(k.kty).to.eql('OKP');", + " pm.expect(k.crv).to.eql('Ed25519');", + " pm.expect(k.alg).to.eql('Ed25519');", + " pm.expect(k.use).to.eql('sig');", + " pm.expect(k.x).to.match(/^[A-Za-z0-9_-]{43}$/);", + " });", + "});", + "pm.test('publishes the governance key and every delegated traffic key', () => {", + " const gov = pm.collectionVariables.get(prefix + '_governance_kid');", + " const ops = JSON.parse(pm.collectionVariables.get(prefix + '_operational_kids') || '[]');", + " pm.expect(gov, 'run the Descriptor request first').to.be.a('string');", + " const kids = keys.map(k => k.kid);", + " pm.expect(kids).to.include(gov);", + " ops.forEach(kid => pm.expect(kids).to.include(kid));", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_front_door}}/.well-known/jwks.json", + "host": [ + "{{maha_front_door}}" + ], + "path": [ + ".well-known", + "jwks.json" + ] + }, + "description": "Same as Bharat, for Maha Vistara." + } + } + ] + }, + { + "name": "Bharat (network A — has mandi)", + "description": "Network A. Publishes its OWN mandi catalogue, and crawls B every 30s so B's weather also answers here.\n\nPorts: adapter 9300, discovery 9301.", + "item": [ + { + "name": "1. Publish mandi catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/publish/publish", + "host": [ + "{{bharat_adapter}}/publish/publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"messageId\": \"bd83dc21-38f3-48fe-9be5-b82d4a213666\",\n \"transactionId\": \"794abc60-4d9b-463f-ba1d-926724ade7f8\",\n \"timestamp\": \"2026-10-07T03:50:03Z\",\n \"version\": \"2.0.0\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\"\n ],\n \"senderId\": \"bharatvistar.oan.local\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:mandi-price:agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_MH\",\n \"name\": \"Bharatvistar mandi prices\",\n \"shortDesc\": \"On-demand commodity prices for agricultural markets.\",\n \"longDesc\": \"Each Resource is one commodity at one market. Consumers discover by location or commodity and invoke the Provider for current prices.\"\n },\n \"isActive\": true,\n \"bppId\": \"bharatvistar.oan.local\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"provider\": {\n \"id\": \"agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\",\n \"shortDesc\": \"Live agricultural market price information.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:nashik:onion\",\n \"descriptor\": {\n \"code\": \"MANDI_PRICE_NASHIK_ONION\",\n \"name\": \"Onion prices at Nashik\",\n \"shortDesc\": \"Current onion prices for Nashik, Maharashtra.\"\n },\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"informationMode\": \"OnDemand\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n }\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:mandi-price:agmarknet.bharatvistar\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"FULL\",\n \"visibleTo\": [\n \"bharatvistar.oan.local\"\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A's own catalogue. visibleTo names ONLY A: there is no reason for another network to mirror it, and leaving B out is what stops B crawling it back." + }, + "response": [] + }, + { + "name": "2. Discover (the farmer's search)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_discovery}}/discover", + "host": [ + "{{bharat_discovery}}/discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"discover\",\n \"domain\": \"agriculture\",\n \"messageId\": \"5e511275-d9ed-442a-964c-b4f4cc574c1b\",\n \"timestamp\": \"2026-10-07T03:23:01Z\",\n \"transactionId\": \"df14c457-d091-48ad-8bbc-6214001f0578\",\n \"version\": \"2.0.0\",\n \"senderId\": \"bharatvistar.oan.local\"\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" || @.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "No context.networkId. An omitted audience means no audience filter at all, so this returns everything A holds -- its own mandi AND the weather it crawled from B. The farmer never names a network." + }, + "response": [] + }, + { + "name": "3. Select mandi (answered locally)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/provider/select", + "host": [ + "{{bharat_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"bppUri\": \"https://bharatvistar.oan.local/provider\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d23\",\n \"timestamp\": \"2026-10-07T06:12:01.330Z\",\n \"senderId\": \"seeker.bharatvistar.oan.local\",\n \"receiverId\": \"bharatvistar.oan.local\",\n \"senderUri\": \"https://seeker.bharatvistar.oan.local/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"provider\": {\n \"id\": \"agmarknet.bharatvistar\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"resourceIds\": [\n \"res:agmarknet:nashik:onion\"\n ]\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:nashik:onion\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"informationMode\": \"OnDemand\",\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n },\n \"supportedCommodities\": [\n {\n \"code\": \"149\",\n \"name\": \"Onion\"\n }\n ],\n \"market\": {\n \"district\": \"96\",\n \"state\": \"MH\",\n \"marketCode\": \"2056\",\n \"name\": \"Nashik\"\n }\n }\n }\n ]\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "context.receiverId is A, so A answers it from its own provider. Returns on_select with real prices from the mock." + }, + "response": [] + }, + { + "name": "4. Select weather (crosses to B)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{bharat_adapter}}/provider/select", + "host": [ + "{{bharat_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-26T06:12:01.330Z\",\n \"bppUri\": \"http://oan-mahavistara-oan-local-adapter:9200/provider\",\n \"senderId\": \"seeker-network-vistaar.da.gov.in\",\n \"receiverId\": \"mahavistara.oan.local\",\n \"senderUri\": \"https://seeker-network-vistaar.da.gov.in/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-08-26T00:00:00+05:30\",\n \"endsAt\": \"2026-08-30T23:59:59+05:30\"\n }\n },\n \"quantity\": {\n \"selected\": {\n \"measure\": {\n \"value\": \"1\",\n \"unit\": \"unit\"\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "SAME endpoint as 3. The only difference is context.receiverId, which the consumer copied out of the catalog: it names B, so A signs as itself and sends it to context.bppUri. The answer comes from B's provider." + }, + "response": [] + }, + { + "name": "5. Crawl B now (optional)", + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{bharat_adapter}}/crawl/peers", + "host": [ + "{{bharat_adapter}}/crawl/peers" + ] + }, + "description": "Not needed for the demo -- A crawls every 30s on its own. Use it only to skip the wait. Takes no body: the crawler reads who to crawl from its registry." + }, + "response": [] + } + ] + }, + { + "name": "Maha (network B — has weather)", + "description": "Network B. Publishes its OWN weather catalogue and shares it with A.\n\nB does NOT crawl: FEDERATION_CRAWL_PEERS=off. Otherwise each network mirrors the other and B ends up re-importing its own catalogue through A's copy of it.\n\nPorts: adapter 9400, discovery 9401.", + "item": [ + { + "name": "1. Publish weather catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_adapter}}/publish/publish", + "host": [ + "{{maha_adapter}}/publish/publish" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"bppUri\": \"https://mahavistara.oan.local/provider\",\n \"messageId\": \"faa5ae21-67ac-45c7-a2f7-92bea0a2587e\",\n \"transactionId\": \"0292de98-363b-4b6b-b61a-98ed5d7a4af5\",\n \"timestamp\": \"2026-10-07T03:19:08Z\",\n \"version\": \"2.0.0\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\"\n ],\n \"senderId\": \"mahavistara.oan.local\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"catalog:weather-observation:mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM_WX\",\n \"name\": \"Mahavistara weather forecasts\",\n \"shortDesc\": \"Daily weather forecasts for locations in Mahavistara.\",\n \"longDesc\": \"Each Resource is a point forecast. Consumers discover by location and invoke the Provider for a three day forecast.\"\n },\n \"isActive\": true,\n \"bppId\": \"mahavistara.oan.local\",\n \"bppUri\": \"https://mahavistara.oan.local/provider\",\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\",\n \"shortDesc\": \"Numerical weather prediction from IMD.\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX_POINT_FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Three day forecast for a point location.\"\n },\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-10-01T00:00:00+05:30\",\n \"endsAt\": \"2026-12-31T23:59:59+05:30\"\n }\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"catalog:weather-observation:mausamgram.mahavistara\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"FULL\",\n \"visibleTo\": [\n \"mahavistara.oan.local\",\n \"bharatvistar.oan.local\"\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "THE IMPORTANT FIELD IS visibleTo. It names A as well as B, which is the whole permission for A to mirror this. Drop A from it and A's crawl returns nothing -- no error, just nothing." + }, + "response": [] + }, + { + "name": "2. Discover (what B holds)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_discovery}}/discover", + "host": [ + "{{maha_discovery}}/discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"action\": \"discover\",\n \"domain\": \"agriculture\",\n \"messageId\": \"5e511275-d9ed-442a-964c-b4f4cc574c1b\",\n \"timestamp\": \"2026-10-07T03:23:01Z\",\n \"transactionId\": \"df14c457-d091-48ad-8bbc-6214001f0578\",\n \"version\": \"2.0.0\",\n \"senderId\": \"mahavistara.oan.local\"\n },\n \"message\": {\n \"intent\": {\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\" || @.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "B holds only its own weather. Run it after A has crawled to show B did NOT import A's mandi back." + }, + "response": [] + }, + { + "name": "3. Select weather (direct, no crossing)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{maha_adapter}}/provider/select", + "host": [ + "{{maha_adapter}}/provider/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"bharatvistar.oan.local\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-26T06:12:01.330Z\",\n \"senderId\": \"mahavistara.oan.local\",\n \"receiverId\": \"mahavistara.oan.local\",\n \"senderUri\": \"https://seeker-network-vistaar.da.gov.in/beckn\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n },\n \"validity\": {\n \"startsAt\": \"2026-08-26T00:00:00+05:30\",\n \"endsAt\": \"2026-08-30T23:59:59+05:30\"\n }\n },\n \"quantity\": {\n \"selected\": {\n \"measure\": {\n \"value\": \"1\",\n \"unit\": \"unit\"\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram.mahavistara\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "The same select as Bharat's #4, but sent straight to B. Same answer -- which is the point: A adds no special handling, it just forwards." + }, + "response": [] + } + ] + } + ] +} diff --git a/networkofnetwork/demo/publish-catalog-mandi.json b/networkofnetwork/demo/publish-catalog-mandi.json new file mode 100644 index 00000000..ac76b059 --- /dev/null +++ b/networkofnetwork/demo/publish-catalog-mandi.json @@ -0,0 +1,82 @@ +{ + "context": { + "action": "catalog/publish", + "bppUri": "https://bharatvistar.oan.local/provider", + "messageId": "bd83dc21-38f3-48fe-9be5-b82d4a213666", + "transactionId": "794abc60-4d9b-463f-ba1d-926724ade7f8", + "timestamp": "2026-10-07T03:50:03Z", + "version": "2.0.0", + "schemaContext": [ + "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld" + ], + "senderId": "bharatvistar.oan.local" + }, + "message": { + "catalogs": [ + { + "id": "catalog:mandi-price:agmarknet.bharatvistar", + "descriptor": { + "code": "MANDI_PRICE_MH", + "name": "Bharatvistar mandi prices", + "shortDesc": "On-demand commodity prices for agricultural markets.", + "longDesc": "Each Resource is one commodity at one market. Consumers discover by location or commodity and invoke the Provider for current prices." + }, + "isActive": true, + "bppId": "bharatvistar.oan.local", + "bppUri": "https://bharatvistar.oan.local/provider", + "provider": { + "id": "agmarknet.bharatvistar", + "descriptor": { + "code": "AGMARKNET", + "name": "Agmarknet Vistaar", + "shortDesc": "Live agricultural market price information." + } + }, + "resources": [ + { + "id": "res:agmarknet:nashik:onion", + "descriptor": { + "code": "MANDI_PRICE_NASHIK_ONION", + "name": "Onion prices at Nashik", + "shortDesc": "Current onion prices for Nashik, Maharashtra." + }, + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "subjectCategories": [ + "Market" + ], + "informationMode": "OnDemand", + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + } + } + } + ] + } + ], + "publishDirectives": [ + { + "catalogId": "catalog:mandi-price:agmarknet.bharatvistar", + "catalogType": "REGULAR", + "updateMode": "FULL", + "visibleTo": [ + "bharatvistar.oan.local" + ] + } + ] + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/publish-catalog.json b/networkofnetwork/demo/publish-catalog.json new file mode 100644 index 00000000..d844bde1 --- /dev/null +++ b/networkofnetwork/demo/publish-catalog.json @@ -0,0 +1,82 @@ +{ + "context": { + "action": "catalog/publish", + "bppUri": "https://mahavistara.oan.local/provider", + "messageId": "faa5ae21-67ac-45c7-a2f7-92bea0a2587e", + "transactionId": "0292de98-363b-4b6b-b61a-98ed5d7a4af5", + "timestamp": "2026-10-07T03:19:08Z", + "version": "2.0.0", + "schemaContext": [ + "https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld" + ], + "senderId": "mahavistara.oan.local" + }, + "message": { + "catalogs": [ + { + "id": "catalog:weather-observation:mausamgram.mahavistara", + "descriptor": { + "code": "MAUSAMGRAM_WX", + "name": "Mahavistara weather forecasts", + "shortDesc": "Daily weather forecasts for locations in Mahavistara.", + "longDesc": "Each Resource is a point forecast. Consumers discover by location and invoke the Provider for a three day forecast." + }, + "isActive": true, + "bppId": "mahavistara.oan.local", + "bppUri": "https://mahavistara.oan.local/provider", + "provider": { + "id": "mausamgram.mahavistara", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP", + "shortDesc": "Numerical weather prediction from IMD." + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "descriptor": { + "code": "WX_POINT_FORECAST", + "name": "Point weather forecast", + "shortDesc": "Three day forecast for a point location." + }, + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + } + } + } + ] + } + ], + "publishDirectives": [ + { + "catalogId": "catalog:weather-observation:mausamgram.mahavistara", + "catalogType": "REGULAR", + "updateMode": "FULL", + "visibleTo": [ + "mahavistara.oan.local", + "bharatvistar.oan.local" + ] + } + ] + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-cross-network.json b/networkofnetwork/demo/select-cross-network.json new file mode 100644 index 00000000..b3a1797d --- /dev/null +++ b/networkofnetwork/demo/select-cross-network.json @@ -0,0 +1,74 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z", + "bppUri": "http://oan-mahavistara-oan-local-adapter:9200/provider", + "senderId": "seeker-network-vistaar.da.gov.in", + "receiverId": "mahavistara.oan.local", + "senderUri": "https://seeker-network-vistaar.da.gov.in/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-08-26T00:00:00+05:30", + "endsAt": "2026-08-30T23:59:59+05:30" + } + }, + "quantity": { + "selected": { + "measure": { + "value": "1", + "unit": "unit" + } + } + } + } + ], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": [ + "res:mausamgram:point-forecast" + ], + "provider": { + "id": "mausamgram.mahavistara", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP" + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-local.json b/networkofnetwork/demo/select-local.json new file mode 100644 index 00000000..d661a2dc --- /dev/null +++ b/networkofnetwork/demo/select-local.json @@ -0,0 +1,73 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-08-26T06:12:01.330Z", + "senderId": "seeker-network-vistaar.da.gov.in", + "receiverId": "bharatvistar.oan.local", + "senderUri": "https://seeker-network-vistaar.da.gov.in/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "resources": [ + { + "id": "res:mausamgram:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": [ + "Weather" + ], + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-08-26T00:00:00+05:30", + "endsAt": "2026-08-30T23:59:59+05:30" + } + }, + "quantity": { + "selected": { + "measure": { + "value": "1", + "unit": "unit" + } + } + } + } + ], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": [ + "res:mausamgram:point-forecast" + ], + "provider": { + "id": "mausamgram.bharatvistar", + "descriptor": { + "code": "IMD-NWP-01", + "name": "IMD Mausamgram NWP" + } + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/demo/select-mandi-local.json b/networkofnetwork/demo/select-mandi-local.json new file mode 100644 index 00000000..8452866c --- /dev/null +++ b/networkofnetwork/demo/select-mandi-local.json @@ -0,0 +1,82 @@ +{ + "context": { + "version": "2.0.0", + "action": "select", + "networkId": "bharatvistar.oan.local", + "bppUri": "https://bharatvistar.oan.local/provider", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d23", + "timestamp": "2026-10-07T06:12:01.330Z", + "senderId": "seeker.bharatvistar.oan.local", + "receiverId": "bharatvistar.oan.local", + "senderUri": "https://seeker.bharatvistar.oan.local/beckn" + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { + "code": "ACTIVE", + "name": "active" + } + }, + "offer": { + "id": "offer:agmarknet:open-data", + "provider": { + "id": "agmarknet.bharatvistar", + "descriptor": { + "code": "AGMARKNET", + "name": "Agmarknet Vistaar" + } + }, + "resourceIds": [ + "res:agmarknet:nashik:onion" + ] + }, + "resources": [ + { + "id": "res:agmarknet:nashik:onion", + "quantity": { + "count": 1 + }, + "resourceAttributes": { + "@context": "https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "subjectCategories": [ + "Market" + ], + "informationMode": "OnDemand", + "location": { + "geo": { + "type": "Point", + "coordinates": [ + 73.7898, + 19.9975 + ] + } + }, + "validity": { + "startsAt": "2026-10-01T00:00:00+05:30", + "endsAt": "2026-12-31T23:59:59+05:30" + }, + "supportedCommodities": [ + { + "code": "149", + "name": "Onion" + } + ], + "market": { + "district": "96", + "state": "MH", + "marketCode": "2056", + "name": "Nashik" + } + } + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/networkofnetwork/docker-compose.yml b/networkofnetwork/docker-compose.yml new file mode 100644 index 00000000..e70fc1aa --- /dev/null +++ b/networkofnetwork/docker-compose.yml @@ -0,0 +1,326 @@ +# One OAN network, whole, on one machine. +# +# Separate from docker-compose.yml and deliberately so. That file is the +# three-adapter quick-start: one stack, fixed names, fixed ports, and an +# operator's own deployment. This one exists to be run MORE THAN ONCE on the +# same machine, as two or three networks that can see each other -- which is the +# only way to demonstrate federation without three laptops. +# +# What differs from the quick-start: +# +# ONE adapter, not three. A network is one participant; running the consumer, +# network and provider layers as three containers is a deployment choice, and +# the consolidated shape is the simpler one to reason about when there are two +# of everything already. +# +# A FRONT DOOR. A network id is a DNS domain, and a peer reaches a network at +# that domain over https. nginx terminates TLS, serves the two published +# documents, and proxies the rest inward. +# +# EVERYTHING PREFIXED. Container names and docker network names are global to +# the daemon rather than scoped to a compose project, so two networks running +# the same file would collide on both without this. +# +# Every value comes from the network's own .env, written by bin/network-env.py. +# Nothing here names a network. + +name: ${COMPOSE_PROJECT_NAME} + +networks: + # This network's own internals. Nothing outside the project reaches them. + internal: + name: ${COMPOSE_PROJECT_NAME}-internal + # The bridge every network on this machine shares. The ONLY crossing point, + # and only the front door and the adapter are on it -- which is what makes the + # boundary real rather than decorative. + federation: + name: oan-federation + external: true + +volumes: + registry-postgres-data: + discovery-postgres-data: + +x-logging: &logging + driver: json-file + options: { max-size: "10m", max-file: "3" } + +services: + + # ---------------------------------------------------------------- registry + + registry-postgres: + image: postgres:14 + container_name: ${COMPOSE_PROJECT_NAME}-registry-postgres + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + POSTGRES_DB: registry + POSTGRES_USER: ${REGISTRY_DB_USER} + POSTGRES_PASSWORD: ${REGISTRY_DB_PASSWORD} + volumes: + - registry-postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${REGISTRY_DB_USER} -d registry"] + interval: 5s + timeout: 5s + retries: 20 + + registry-keycloak: + image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest + container_name: ${COMPOSE_PROJECT_NAME}-registry-keycloak + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + DB_VENDOR: postgres + DB_ADDR: registry-postgres + DB_PORT: "5432" + DB_DATABASE: registry + DB_USER: ${REGISTRY_DB_USER} + DB_PASSWORD: ${REGISTRY_DB_PASSWORD} + KEYCLOAK_USER: ${KEYCLOAK_ADMIN_USER} + KEYCLOAK_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + KEYCLOAK_IMPORT: /opt/jboss/keycloak/imports/realm-export.json + PROXY_ADDRESS_FORWARDING: "true" + volumes: + # The realm this registry runs on. KEYCLOAK_IMPORT names it, and without + # the file keycloak restarts in a loop saying only that it added the admin + # user again. + - ./config/registry/imports:/opt/jboss/keycloak/imports + # Waited on rather than assumed: the registry service resolves its OAuth + # issuer at start-up, and a keycloak that is not listening yet fails it + # outright rather than retrying. + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9990/ || exit 1"] + interval: 10s + timeout: 10s + retries: 30 + depends_on: + registry-postgres: + condition: service_healthy + ports: + - "127.0.0.1:${KEYCLOAK_PORT}:8080" + + registry-service: + image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_IMAGE_TAG} + container_name: ${COMPOSE_PROJECT_NAME}-registry-service + restart: unless-stopped + logging: *logging + # internal ONLY, and that is a decision rather than an omission: the + # registry is this network's own record and nothing outside it has business + # reading it directly. + networks: [internal] + environment: + connectionInfo_uri: jdbc:postgresql://registry-postgres:5432/registry + connectionInfo_username: ${REGISTRY_DB_USER} + connectionInfo_password: ${REGISTRY_DB_PASSWORD} + search_providerName: dev.sunbirdrc.registry.service.NativeSearchService + authentication_enabled: "true" + sunbird_sso_realm: ${KEYCLOAK_REALM} + sunbird_sso_url: http://registry-keycloak:8080/auth + OAUTH2_RESOURCES_0_URI: http://registry-keycloak:8080/auth/realms/${KEYCLOAK_REALM} + OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH: realm_access.roles + identity_provider: dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + sunbird_sso_admin_client_id: ${KEYCLOAK_ADMIN_CLIENT_ID} + sunbird_sso_client_id: ${KEYCLOAK_CLIENT_ID} + sunbird_sso_admin_client_secret: ${KEYCLOAK_ADMIN_CLIENT_SECRET} + sunbird_keycloak_user_set_password: "true" + sunbird_keycloak_user_password: ${KEYCLOAK_DEFAULT_USER_PASSWORD} + encryption_enabled: "false" + event_enabled: "false" + idgen_enabled: "false" + claims_enabled: "false" + did_enabled: "false" + signature_enabled: "false" + certificate_enabled: "false" + filestorage_enabled: "false" + notification_enabled: "false" + notification_async_enabled: "false" + async_enabled: "false" + webhook_enabled: "false" + registry_base_apis_enable: "false" + manager_type: DefinitionsManager + expand_reference: "false" + swagger_title: OAN Registry + logging.level.root: INFO + volumes: + - ./config/registry/schemas:/home/sunbirdrc/config/public/_schemas:ro + depends_on: + registry-keycloak: + condition: service_healthy + ports: + - "127.0.0.1:${REGISTRY_PORT}:8081" + + # --------------------------------------------------------------- discovery + + discovery-postgres: + # pgvector rather than plain postgres: the discovery service's HNSW index + # options arrived in 0.8. + image: pgvector/pgvector:0.8.0-pg16 + container_name: ${COMPOSE_PROJECT_NAME}-discovery-postgres + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + # Literals, matching quick-start: these never varied per deployment, and a + # variable that is always the same value is a question nobody needs asked. + POSTGRES_DB: discovery + POSTGRES_USER: discovery + POSTGRES_PASSWORD: discovery + volumes: + - discovery-postgres-data:/var/lib/postgresql/data + # The crawler keeps its bookkeeping in a database of its own on this same + # server. Created here because a destroyed stack takes the volume with it, + # and the adapter restart-loops on a missing database. + - ./config/init-crawler-db.sh:/docker-entrypoint-initdb.d/10-crawler.sh:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U discovery -d discovery"] + interval: 5s + timeout: 5s + retries: 20 + + discovery-service: + image: ${DISCOVERY_IMAGE} + container_name: ${COMPOSE_PROJECT_NAME}-discovery-service + restart: unless-stopped + logging: *logging + networks: [internal] + environment: + APP_DOMAIN: ${DISCOVERY_DOMAIN:-Agriculture} + APP_NETWORK_ID: ${BECKN_NETWORK_ID} + APP_SUBSCRIBER_ID: ${DISCOVERY_SUBSCRIBER_ID} + DATABASE_AUTO_MIGRATE: "true" + DATABASE_URL: postgres://discovery:discovery@discovery-postgres:5432/discovery?sslmode=disable + GEO_MAX_GEOMETRIES_PER_CATALOG: ${DISCOVERY_MAX_GEOMETRIES_PER_CATALOG:-256} + OTEL_EXPORTER: ${DISCOVERY_OTEL_EXPORTER:-none} + RATE_LIMIT_BURST: ${DISCOVERY_RATE_LIMIT_BURST:-100000} + RATE_LIMIT_RPS: ${DISCOVERY_RATE_LIMIT_RPS:-100000} + SEARCH_ENABLE_TEXT_SEARCH: "false" + SERVER_PORT: "8080" + VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} + depends_on: + discovery-postgres: + condition: service_healthy + ports: + - "127.0.0.1:${DISCOVERY_PORT}:8080" + + # ------------------------------------------------------------------- mocks + + mock-imd: + image: ${MOCKIMD_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-mock-imd + restart: unless-stopped + logging: *logging + networks: [internal] + ports: + - "127.0.0.1:${MOCKIMD_PORT}:9100" + + mock-agmarknet: + image: ${MOCKAGMARKNET_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-mock-agmarknet + restart: unless-stopped + logging: *logging + networks: [internal] + ports: + - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" + + # ----------------------------------------------------------------- adapter + + # ONE adapter, serving every layer this network has. Its modules are mounted + # on separate paths of one port -- see config/network/adapter.yaml.tmpl. + # + # On the federation bridge as well as the internal one, because IT is what + # reaches another network: the crawl asking for catalogs, and a cross-network + # select going out. The front door is how others reach US; this is the other + # direction over the same bridge. + adapter: + image: ${ADAPTER_IMAGE} + pull_policy: missing + container_name: ${COMPOSE_PROJECT_NAME}-adapter + restart: unless-stopped + logging: *logging + networks: [internal, federation] + environment: + BECKN_NETWORK_ID: ${BECKN_NETWORK_ID} + # Go reads SSL_CERT_FILE for the system roots on Linux, so an outbound + # call verifies another network's certificate against the local CA with no + # code that knows anything about local certificates. + # + # Real verification against a different root, NOT verification turned off: + # a certificate this CA did not sign, or one whose name does not match, is + # still refused. + SSL_CERT_FILE: /etc/oan/ca.crt + CONFIG_FILE: /app/config/adapter.yaml + MAUSAMGRAM_AUTH: ${MAUSAMGRAM_AUTH} + AGMARKNET_ACCESS_NAME: ${AGMARKNET_ACCESS_NAME} + AGMARKNET_PASSWORD: ${AGMARKNET_PASSWORD} + VISTAAR_CLIENT_ID: ${VISTAAR_CLIENT_ID} + VISTAAR_CLIENT_SECRET: ${VISTAAR_CLIENT_SECRET} + volumes: + - ${NETWORK_DIR}/config/adapter.yaml:/app/config/adapter.yaml:ro + # Routing is NOT rendered per network -- nothing in it differs between + # networks, because every target is either this same adapter on another + # path or a service name inside this project. + - ./config/routing-consumer.yaml:/app/config/routing-consumer.yaml:ro + - ./config/routing-network.yaml:/app/config/routing-network.yaml:ro + - ./config/routing-publish.yaml:/app/config/routing-publish.yaml:ro + - ${NETWORK_DIR}/tls/ca.crt:/etc/oan/ca.crt:ro + depends_on: + registry-service: + condition: service_started + discovery-service: + condition: service_started + ports: + - "127.0.0.1:${ADAPTER_PORT}:9200" + + # --------------------------------------------------------------- front door + + # What another network talks to. It is the whole of this network's public + # surface: the two published documents, and a proxy inward for the Beckn + # actions. Nothing else here is reachable from the bridge. + wellknown: + image: nginx:1.27-alpine + container_name: ${COMPOSE_PROJECT_NAME}-wellknown + restart: unless-stopped + logging: *logging + environment: + # Read by envsubst into default.conf.template at start-up. + NETWORK_ID: ${BECKN_NETWORK_ID} + ADAPTER_HOST: adapter + ADAPTER_PORT: "9200" + MOCKIMD_HOST: mock-imd + MOCKIMD_INTERNAL_PORT: "9100" + MOCKAGMARKNET_HOST: mock-agmarknet + MOCKAGMARKNET_INTERNAL_PORT: "9101" + volumes: + - ${NETWORK_DIR}/.well-known:/usr/share/nginx/html/.well-known:ro + - ${NETWORK_DIR}/tls:/etc/nginx/tls:ro + - ./config/default.conf.template:/etc/nginx/templates/default.conf.template:ro + - ./config/proxy-headers.inc:/etc/nginx/conf.d/proxy-headers.inc:ro + networks: + internal: + # The upstream provider names, on the INTERNAL bridge because it is our + # own adapter that calls them. Without these the adapter's lookup of + # mausamgram. fails at DNS and the seeded baseUrl is a URL + # nothing can reach. + aliases: + - mausamgram.${BECKN_NETWORK_ID} + - agmarknet.${BECKN_NETWORK_ID} + federation: + # This is what makes the network id resolve. Another network's adapter + # looking up bharatvistar.oan.local reaches this container and nothing + # else behind it. + aliases: + - ${BECKN_NETWORK_ID} + depends_on: + adapter: + condition: service_started + ports: + # Published so the operator's own machine can fetch the documents. The + # network id only resolves inside the bridge, so a CLI reaches this port + # and carries the id as its Host header. + - "127.0.0.1:${FRONT_DOOR_PORT}:443"