feat: host the Network Descriptor and JWKS (network-adapter#74)

networkofnetwork/.env.example

@@ -0,0 +1,229 @@
+# Copy to .env and fill in every <angle-bracketed> value. bin/setup.py refuses
+# to touch the registry while any of them is still a placeholder -- the registry
+# is append-only, so a row seeded with a placeholder burns that id for good.
+#
+# One line per variable, and one meaning per name. Where a name carries a
+# prefix, the prefix says WHOSE the value is: MAUSAMGRAM_/AGMARKNET_/VISTAAR_/
+# POCRA_ are the four upstream entities, CONSUMER_/NETWORK_/PROVIDER_ are the
+# three adapters, REGISTRY_/KEYCLOAK_/DISCOVERY_ are the services.
+#
+# Every credential below is a shipped default, which means it is public. Change
+# all of them before the stack is reachable by anyone but you. The adapter
+# keypairs are the exception: setup.py generates those into keys/keys.json.
+
+# ---- images ----------------------------------------------------------------
+# Pinned, not latest, so what is deployed is in the diff. The adapter image and
+# the adapter configs in this repo move together: a config names plugins by id
+# and an id is a .so inside the image.
+ADAPTER_IMAGE=ghcr.io/openagrinet/network-adapter:0.0.1
+DISCOVERY_IMAGE=ghcr.io/openagrinet/discovery-service:0.0.1
+# Source dir for `docker compose build discovery`; nothing else reads it.
+DISCOVERY_CONTEXT=../../discovery-service
+# That build's base images. Compose already defaults them to public ones because
+# the Dockerfile's own dhi.io/* defaults 401 without a Hardened Images
+# subscription. Uncomment only if you have one.
+#DISCOVERY_BUILD_IMAGE=dhi.io/golang:1.27-alpine-dev
+#DISCOVERY_RUNTIME_IMAGE=dhi.io/static:20260611-musl-alpine
+# The two mock upstreams. No capability uses one; `make up` does not start them.
+MOCKIMD_IMAGE=ghcr.io/nisargabd/oan-mockimd:latest
+MOCKAGMARKNET_IMAGE=ghcr.io/nisargabd/oan-mockagmarknet:latest
+
+# ---- ports -----------------------------------------------------------------
+# All published on 127.0.0.1 except the reverse proxy's 80/443, and the loopback
+# binding is written literally in docker-compose.yml rather than taken from a
+# variable. Reach the rest over an SSH tunnel.
+REGISTRY_PORT=8081
+KEYCLOAK_PORT=8080
+KEYCLOAK_ADMIN_PORT=9990
+DISCOVERY_PORT=8090
+PROVIDER_ADAPTER_PORT=9200
+NETWORK_ADAPTER_PORT=9201
+CONSUMER_ADAPTER_PORT=9202
+# Only for looking at a mock directly; the adapter reaches it by service name.
+MOCKIMD_PORT=9100
+MOCKAGMARKNET_PORT=9101
+# The reverse-proxy profile (nginx-proxy-manager). 80/443 are public, admin is not.
+REVERSE_PROXY_HTTP_PORT=80
+REVERSE_PROXY_HTTPS_PORT=443
+NPM_ADMIN_PORT=81
+# The observability profile (hyperdx + clickhouse).
+HYPERDX_UI_PORT=8085
+HYPERDX_OTLP_GRPC_PORT=4317
+HYPERDX_OTLP_HTTP_PORT=4318
+CLICKHOUSE_HTTP_PORT=8123
+
+# ---- registry tier ---------------------------------------------------------
+# CHANGE every credential in this block before the stack is exposed.
+REGISTRY_IMAGE_TAG=v2.0.0
+# registry-db. Named REGISTRY_DB_ because discovery-db is a separate database.
+REGISTRY_DB_NAME=registry
+REGISTRY_DB_USER=postgres
+REGISTRY_DB_PASSWORD=postgres
+# The Keycloak login setup.py exchanges for the registry write token.
+REGISTRY_ADMIN_USER=no-user
+REGISTRY_ADMIN_PASSWORD=no-user-password
+KEYCLOAK_REALM=sunbird-rc
+KEYCLOAK_ADMIN_USER=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+# Two Keycloak clients: admin-api mints tokens, registry-frontend logs users in.
+KEYCLOAK_ADMIN_CLIENT_ID=admin-api
+KEYCLOAK_ADMIN_CLIENT_SECRET=1b46c0f2-6a1c-4a2f-8b0f-2b6b30bcd2b2
+KEYCLOAK_CLIENT_ID=registry-frontend
+# Password Keycloak assigns to registry users it provisions.
+KEYCLOAK_DEFAULT_USER_PASSWORD=abcd@123
+
+# ---- telemetry -------------------------------------------------------------
+# Both components ship to this collector; a bare host:port, no scheme.
+OTLP_ENDPOINT=hyperdx:4317
+# The deployment name on every stream, from both components.
+OTEL_ENVIRONMENT=dev
+# ADAPTERS ONLY, all three signals. false builds no exporter and never dials --
+# set it for `make up-core`, which does not start hyperdx.
+ADAPTER_OTEL_ENABLED=true
+# DISCOVERY ONLY, and a separate switch: `none` for `make up-core`.
+DISCOVERY_OTEL_EXPORTER=otlp
+
+# ---- discovery -------------------------------------------------------------
+# The network this deployment belongs to; pairs with the spec it validates against.
+BECKN_NETWORK_ID=oan-dev
+BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml
+# Required when DISCOVERY_OTEL_EXPORTER is otlp: it attributes discovery's spans.
+# Needs no keypair and no registry row -- discovery does not sign.
+DISCOVERY_SUBSCRIBER_ID=discovery.oan.dev
+# The sector. Checked against the build's sector list; every OAN component emits it.
+DISCOVERY_DOMAIN=Agriculture
+# GeoJSON shapes ONE catalog may add to the spatial index; past it, publish is
+# PARTIAL and the excess is unsearchable. Count shapes, not resources.
+#
+# 50000 against the image's default of 256, because that number counts SHAPES
+# while the cost it guards is incurred per H3 CELL -- and the two differ by up
+# to 8192 apiece. A Point is one cell; a state-sized polygon is the ceiling. So
+# a points-only catalog can raise this a long way and a polygon-heavy one
+# cannot. The mandi catalog is 272 markets carrying a Point under both
+# coverageAreas and market.location: 544 shapes, 544 cells, against the ~2.1M
+# cells the default stands in for. Measured 2026-09-12 -- 50000 Points is
+# roughly 3 s and 18 MB of index, so this is sized for where the catalog is
+# going rather than where it is.
+#
+# RAISE IT BEFORE A BULK PUBLISH, not after. While a catalog is over the
+# ceiling, republishing a resource that sits past the cut-off clears its
+# geometry rows and writes none back -- it drops off the map silently.
+#
+# Interim: discovery-service#32 moves the budget onto cells, where it stops
+# depending on what you publish and this line goes away.
+DISCOVERY_MAX_GEOMETRIES_PER_CATALOG=50000
+# Requests per second one client may make, and the bucket it may burst to.
+# Effectively off: the image defaults to 20/40, which is under any scripted
+# sweep, and a throttled request NACKs with AUT_RATE_LIMITED and no `catalogs`
+# -- so it reads as an empty result, not as backpressure. Cannot be 0: the
+# boot requires burst >= rps > 0.
+DISCOVERY_RATE_LIMIT_RPS=100000
+DISCOVERY_RATE_LIMIT_BURST=100000
+
+# ---- capability schemas ----------------------------------------------------
+# The ref in OpenAgriNet/network-specs that setup.py builds the four
+# SchemaRegistry rows from. Those rows say what a capability MEANS -- the pack
+# defining its attributes -- as against the *_MAPPING_URL values below, which
+# say how one upstream serves it. A branch, tag or commit; the path is derived
+# from the capability code, so schema/<Capability>/<version>/attributes.yaml.
+#
+# GET THIS RIGHT BEFORE THE FIRST RUN. The registry checks the URL's shape and
+# never fetches it, so a ref that does not serve the pack stores happily and
+# fails at whoever resolves it. It also cannot be corrected: SchemaRegistry
+# rejects updates and its delete is soft, keeping the unique index. Today main
+# does NOT serve these packs -- they are on schema-packs-v0.1 until that merges.
+SCHEMA_PACK_REF=main
+
+# ---- the three adapter identities ------------------------------------------
+# setup.py registers these three and generates a keypair for each. A participant
+# id IS the network identity a signature is verified against, so it must be
+# hostname-shaped. Pick them once: the registry cannot update or truly delete a
+# row, so an id is never reusable.
+CONSUMER_SUBSCRIBER_ID=consumer.oan.dev
+NETWORK_SUBSCRIBER_ID=network.oan.dev
+PROVIDER_SUBSCRIBER_ID=provider.oan.dev
+# The address PEERS are told to POST to, published in each registry row. Nothing
+# inside this stack resolves it -- routing here is by compose service name -- so
+# a wrong value costs nothing locally and everything on a shared network.
+CONSUMER_BASE_URL=https://consumer.oan.dev
+NETWORK_BASE_URL=https://network.oan.dev
+PROVIDER_BASE_URL=https://provider.oan.dev
+
+# ---- the four upstream providers -------------------------------------------
+# setup.py creates all four plus a capability binding each -- seven participants
+# and four bindings in all. The registry is unreachable from outside the stack,
+# so there is no second way to create them.
+#
+# THE BINDING KEY IS <PARTICIPANT_ID>|<CAPABILITY>. The provider adapter builds
+# it from the incoming payload and matches it against its own config; setup.py
+# renders that config AND seeds the registry from these same values, which is
+# what keeps the two from disagreeing. Change one, re-run setup.py.
+# A mismatch is a 404, either NET_ENTITY_NOT_FOUND or "no active record".
+#
+# All four are REAL external APIs -- the stack needs egress to each.
+
+# WeatherObservation, from IMD Mausamgram. One ready-made header value, "Basic "
+# prefix included, sent verbatim (authScheme header) -- IMD issues a single
+# opaque token, not a user/password pair.
+MAUSAMGRAM_PARTICIPANT_ID=mausamgram
+MAUSAMGRAM_CAPABILITY=openagrinet:WeatherObservation
+MAUSAMGRAM_BASE_URL=https://mausamgram.imd.gov.in
+MAUSAMGRAM_PATH=/nwpapi/get-daily
+MAUSAMGRAM_AUTH=<the whole header value IMD gave you, e.g. Basic eyJhb...>
+MAUSAMGRAM_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/mausamgram/weather-observation.select.yaml
+
+# MandiPrice, from Agmarknet. authScheme tokenQuery: the adapter POSTs the
+# access name and password to the token URL and sends the token it gets back as
+# a query parameter. Those two reach the adapter as environment variables only.
+# TOKEN_URL IS ABSOLUTE, not a path: it is rendered into the config verbatim and
+# POSTed as given, so a bare path fails with unsupported protocol scheme "".
+# KNOWN AND UNFIXED: "no records" comes back as HTTP 400, which the adapter
+# treats as permanent -- so an empty market reads as ERROR, not "no prices".
+AGMARKNET_PARTICIPANT_ID=agmarknet
+AGMARKNET_CAPABILITY=openagrinet:MandiPrice
+AGMARKNET_BASE_URL=<the Agmarknet base URL>
+AGMARKNET_PATH=/v1/fetch-agmarknet-vistaar
+AGMARKNET_TOKEN_URL=<the Agmarknet token endpoint, absolute: http://host:port/path>
+AGMARKNET_ACCESS_NAME=<the Agmarknet access name>
+AGMARKNET_PASSWORD=<the Agmarknet password>
+AGMARKNET_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/agmarknet/mandi-price.select.yaml
+
+# KnowledgeAdvisory, from Bharat Vistaar -- the entity that runs the corpus, not
+# the docs-pipeline that filled it. authScheme oauth2 client credentials, and a
+# POST. The two secrets reach the adapter as environment variables only.
+VISTAAR_PARTICIPANT_ID=bharat-vistaar
+VISTAAR_CAPABILITY=openagrinet:KnowledgeAdvisory
+VISTAAR_BASE_URL=<the Bharat Vistaar base URL>
+VISTAAR_PATH=/docs-pipeline-api/search
+VISTAAR_TOKEN_URL=<the Bharat Vistaar OAuth2 token endpoint>
+VISTAAR_CLIENT_ID=<the service account client id>
+VISTAAR_CLIENT_SECRET=<the service account secret>
+VISTAAR_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/knowledge/knowledge-advisory.select.yaml
+
+# AgricultureFacility, from POCRA. A POST, and the only capability needing no
+# credential -- its registry row publishes authScheme none.
+# TWO MEASURED BEHAVIOURS: it takes exactly ONE facility type per call and
+# caches on message_id for ten minutes while IGNORING location; and after ~20
+# calls it answers 200 with an empty catalogue, indistinguishable from "nothing
+# nearby". That is why searchConcurrency is pinned to 1. Do not raise it.
+POCRA_PARTICIPANT_ID=pocra
+POCRA_CAPABILITY=openagrinet:AgricultureFacility
+POCRA_BASE_URL=https://middleware-bap-client.mahapocra.gov.in
+POCRA_PATH=/search
+POCRA_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/main/quick-start/config/mappings/pocra/agriculture-facility.select.yaml
+FEDERATION_INTENT={"filters": {"type": "jsonpath", "expression": "$.catalogs[*] ? (@.isActive == true)"}}
+
+# on|off. off means this network crawls nobody -- it still ANSWERS another
+# network's crawl, which is the network module's job and unaffected.
+FEDERATION_CRAWL_PEERS=on
+
+# How long a peer may serve a copy of our catalogues, in seconds. It is also
+# what paces their re-crawl: the crawler refreshes at projectionTtl/3, floored
+# at 30s, so 30 here means a peer re-crawls us every 30 seconds. Short on
+# purpose -- it is what makes a catalog visibly cross during a demo.
+FEDERATION_PROJECTION_TTL=30
+
+# Upper bound on the re-crawl interval. Recomputed from the peers found on
+# each pass, so this is also how long a deployment with no peers at startup
+# waits before looking again.
+FEDERATION_REFRESH_SECONDS=30

networkofnetwork/.gitignore

@@ -0,0 +1,8 @@
+# One directory per network, and all of it generated: .env, the keypair, TLS
+# material, the rendered adapter config and the signed documents.
+#
+# Ignored as a WHOLE rather than by pattern -- a rule naming .env and keys/
+# would not catch networks/<any-network>/tls/server.key, and the cost of
+# missing one is a private key in a public repository.
+networks/
+__pycache__/

networkofnetwork/Makefile

@@ -0,0 +1,76 @@
+# Network of networks, locally.
+#
+# Runs a whole OAN network by name, as many times as you like on one machine --
+# each with its own registry, its own discovery, its own adapter and a front
+# door the others reach it at. Two of them is federation with nothing stubbed.
+#
+#     make network NETWORK_ID=bharatvistar.oan.local
+#     make network NETWORK_ID=mahavistara.oan.local
+#     make ls
+#
+# Self-contained: its own .env.example, registry schemas, setup and compose
+# file. Nothing here reads or writes anything outside this directory.
+#
+# Anchored to this file's own directory rather than $(PWD), so
+# `make -C networkofnetwork network ...` works from the repo root.
+
+NET := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/network.sh
+
+.PHONY: help network setup host-descriptor e2e-descriptor admit down destroy teardown ps logs ls
+
+# Default target: running a bare `make` in a directory that can delete a
+# Postgres volume should print the menu, not pick something from it.
+help:
+	@$(NET)
+
+# ------------------------------------------------------------- one network
+
+# Everything: ports, certificates, .env, containers, registry seed, front door.
+# Safe to re-run -- it is also how you apply a config change.
+network:  ; @$(NET) up      $(NETWORK_ID)
+
+# Re-seed the registry and re-render the adapter config only.
+setup:    ; @$(NET) setup   $(NETWORK_ID)
+
+# Generate and sign the two documents this network HOSTS at its own domain --
+# the descriptor and its jwks -- from its registry entry. Run it after
+# `network`, and again whenever the entry changes.
+#
+# Named for the documents, not for the verb: "publish" on its own reads as
+# publishing a CATALOG, which is a different thing entirely and goes to the
+# adapter's /publish/ mount.
+host-descriptor: ; @$(NET) host-descriptor $(NETWORK_ID)
+
+# End-to-end check of the hosted descriptor and jwks: publishes a new revision,
+# fetches it over TLS as PEER would, tampers with it, and restores what was
+# served. Writes nothing to either registry.
+#
+#     make e2e-descriptor NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local
+e2e-descriptor: ; @python3 $(dir $(NET))e2e_descriptor.py $(NETWORK_ID) $(PEER)
+
+# Record our decision to deal with another network: fetch its descriptor over
+# TLS, verify the signature against the key it publishes, and store it.
+#
+# Two networks, so two variables. NETWORK_ID is US -- whose registry is written
+# -- and PEER is them. It is one direction only; run it both ways for a pair.
+#
+#     make admit NETWORK_ID=bharatvistar.oan.local PEER=mahavistara.oan.local
+admit:    ; @$(NET) admit   $(NETWORK_ID) $(PEER)
+
+down:     ; @$(NET) down    $(NETWORK_ID)
+
+# Deletes the data AND releases the ports, so the block can be reused.
+destroy:  ; @$(NET) destroy $(NETWORK_ID)
+
+# Destroy EVERY deployed network. Asks first; TEARDOWN_YES=1 skips the prompt.
+#
+#     make teardown
+teardown: ; @$(NET) teardown
+
+ps:       ; @$(NET) ps      $(NETWORK_ID)
+logs:     ; @$(NET) logs    $(NETWORK_ID) $(SVC)
+
+# ----------------------------------------------------------- every network
+
+# What is deployed on this machine, and which ports each one holds.
+ls:       ; @$(NET) ls