diff --git a/api-collection/OpenAgriNet.api-collection.json b/api-collection/OpenAgriNet.api-collection.json index 8ef86f0e..d36cf264 100644 --- a/api-collection/OpenAgriNet.api-collection.json +++ b/api-collection/OpenAgriNet.api-collection.json @@ -168,7 +168,7 @@ }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\"\n },\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-09T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mandi-price:open-data\",\n \"resourceIds\": [\n \"resource:mandi-price:market:1634\"\n ],\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"oan-dev\",\n \"receiverId\": \"oan-dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:mandi-price:market:1634\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"23\",\n \"name\": \"Onion\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Rahuri APMC\",\n \"district\": \"338\",\n \"state\": \"MH\"\n },\n \"validity\": {\n \"startsAt\": \"2026-09-01T00:00:00+05:30\",\n \"endsAt\": \"2026-09-09T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:mandi-price:open-data\",\n \"resourceIds\": [\n \"resource:mandi-price:market:1634\"\n ],\n \"provider\": {\n \"id\": \"agmarknet\",\n \"descriptor\": {\n \"code\": \"AGMARKNET\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -324,7 +324,7 @@ }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:weather-observation:mausamgram:open-data\",\n \"resourceIds\": [\n \"resource:weather-observation:mausamgram:daily-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"oan-dev\",\n \"receiverId\": \"oan-dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:weather-observation:mausamgram:daily-forecast\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 74.4,\n 18.4\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:weather-observation:mausamgram:open-data\",\n \"resourceIds\": [\n \"resource:weather-observation:mausamgram:daily-forecast\"\n ],\n \"provider\": {\n \"id\": \"mausamgram\",\n \"descriptor\": {\n \"code\": \"MAUSAMGRAM\",\n \"name\": \"IMD Mausamgram\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -480,7 +480,7 @@ }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"topics\": [\n \"cotton\",\n \"pink bollworm\",\n \"pest management\"\n ]\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:knowledge-advisory:knowledge:open-data\",\n \"resourceIds\": [\n \"resource:knowledge-advisory:knowledge:retrieval\"\n ],\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"oan-dev\",\n \"receiverId\": \"oan-dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld#openagrinet:KnowledgeAdvisory\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:knowledge-advisory:knowledge:retrieval\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/KnowledgeAdvisory/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:KnowledgeAdvisory\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Crop\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"topics\": [\n \"cotton\",\n \"pink bollworm\",\n \"pest management\"\n ]\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:knowledge-advisory:knowledge:open-data\",\n \"resourceIds\": [\n \"resource:knowledge-advisory:knowledge:retrieval\"\n ],\n \"provider\": {\n \"id\": \"bharat-vistaar\",\n \"descriptor\": {\n \"code\": \"KNOWLEDGE_PROVIDER\",\n \"name\": \"Knowledge retrieval provider\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -636,7 +636,7 @@ }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"consumer.oan.dev\",\n \"receiverId\": \"provider.oan.dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"supportedFacilityTypes\": [\n \"Warehouse\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agriculture-facility:pocra:open-data\",\n \"resourceIds\": [\n \"resource:agriculture-facility:pocra:facility-search\"\n ],\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"domain\": \"agriculture\",\n \"action\": \"select\",\n \"version\": \"2.0.0\",\n \"networkId\": \"oan-dev\",\n \"senderId\": \"oan-dev\",\n \"receiverId\": \"oan-dev\",\n \"transactionId\": \"{{transactionId}}\",\n \"messageId\": \"{{messageId}}\",\n \"timestamp\": \"{{timestamp}}\",\n \"schemaContext\": [\n \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld#openagrinet:AgricultureFacility\"\n ]\n },\n \"message\": {\n \"contract\": {\n \"status\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n },\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"ACTIVE\",\n \"name\": \"active\"\n }\n },\n \"resources\": [\n {\n \"id\": \"resource:agriculture-facility:pocra:facility-search\",\n \"quantity\": {\n \"count\": 1\n },\n \"resourceAttributes\": {\n \"@context\": \"https://openagrinet.github.io/network-specs/schema/AgricultureFacility/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:AgricultureFacility\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Facility\"\n ],\n \"supportedFacilityTypes\": [\n \"Warehouse\"\n ],\n \"location\": {\n \"geo\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agriculture-facility:pocra:open-data\",\n \"resourceIds\": [\n \"resource:agriculture-facility:pocra:facility-search\"\n ],\n \"provider\": {\n \"id\": \"pocra\",\n \"descriptor\": {\n \"code\": \"POCRA\",\n \"name\": \"PoCRA, Government of Maharashtra\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" diff --git a/quick-start/.env.example b/quick-start/.env.example index 3b6d9a10..843d324a 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -4,12 +4,13 @@ # # One line per variable, and one meaning per name. Where a name carries a # prefix, the prefix says WHOSE the value is: MAUSAMGRAM_/AGMARKNET_/VISTAAR_/ -# POCRA_ are the four upstream entities, CONSUMER_/NETWORK_/PROVIDER_ are the -# three adapters, REGISTRY_/KEYCLOAK_/DISCOVERY_ are the services. +# POCRA_ are the four upstream entities, BECKN_NETWORK_/NETWORK_ are the network +# itself -- one identity all three adapters share -- and REGISTRY_/KEYCLOAK_/ +# DISCOVERY_ are the services. # # Every credential below is a shipped default, which means it is public. Change -# all of them before the stack is reachable by anyone but you. The adapter -# keypairs are the exception: setup.py generates those into keys/keys.json. +# all of them before the stack is reachable by anyone but you. The network's +# keypair is the exception: setup.py generates it into keys/keys.json. # ---- images ---------------------------------------------------------------- # Pinned, not latest, so what is deployed is in the diff. The adapter image and @@ -84,8 +85,13 @@ ADAPTER_OTEL_ENABLED=true DISCOVERY_OTEL_EXPORTER=otlp # ---- discovery ------------------------------------------------------------- -# The network this deployment belongs to; pairs with the spec it validates against. -BECKN_NETWORK_ID=oan-dev +# The network's identity, and the network discovery serves. setup.py registers +# it as ONE registry entry that all three adapters share, and the Network +# Descriptor is published at https:///.well-known/openagrinet -- so it +# must be a DNS name (it needs a dot; the registry refuses a node id without +# one). Use the network's real domain once it is fixed. Pick it once: the +# registry cannot update or truly delete a row, so an id is never reusable. +BECKN_NETWORK_ID=oan-dev.local BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml # Required when DISCOVERY_OTEL_EXPORTER is otlp: it attributes discovery's spans. # Needs no keypair and no registry row -- discovery does not sign. @@ -120,39 +126,28 @@ DISCOVERY_MAX_GEOMETRIES_PER_CATALOG=50000 DISCOVERY_RATE_LIMIT_RPS=100000 DISCOVERY_RATE_LIMIT_BURST=100000 -# ---- capability schemas ---------------------------------------------------- -# The ref in OpenAgriNet/network-specs that setup.py builds the four -# SchemaRegistry rows from. Those rows say what a capability MEANS -- the pack -# defining its attributes -- as against the *_MAPPING_URL values below, which -# say how one upstream serves it. A branch, tag or commit; the path is derived -# from the capability code, so schema///attributes.yaml. +# ---- the network's identity and federation --------------------------------- +# One registry entry for the whole network, id BECKN_NETWORK_ID above, shared by +# all three adapters with ONE keypair. The key's osid is the network's key id: +# what the adapters name in the Authorization header, and what +# `make host-descriptor` publishes as the descriptor's keyId and the JWKS kid. # -# GET THIS RIGHT BEFORE THE FIRST RUN. The registry checks the URL's shape and -# never fetches it, so a ref that does not serve the pack stores happily and -# fails at whoever resolves it. It also cannot be corrected: SchemaRegistry -# rejects updates and its delete is soft, keeping the unique index. Today main -# does NOT serve these packs -- they are on schema-packs-v0.1 until that merges. -SCHEMA_PACK_REF=main - -# ---- the three adapter identities ------------------------------------------ -# setup.py registers these three and generates a keypair for each. A participant -# id IS the network identity a signature is verified against, so it must be -# hostname-shaped. Pick them once: the registry cannot update or truly delete a -# row, so an id is never reusable. -CONSUMER_SUBSCRIBER_ID=consumer.oan.dev -NETWORK_SUBSCRIBER_ID=network.oan.dev -PROVIDER_SUBSCRIBER_ID=provider.oan.dev -# The address PEERS are told to POST to, published in each registry row. Nothing -# inside this stack resolves it -- routing here is by compose service name -- so -# a wrong value costs nothing locally and everything on a shared network. -CONSUMER_BASE_URL=https://consumer.oan.dev -NETWORK_BASE_URL=https://network.oan.dev -PROVIDER_BASE_URL=https://provider.oan.dev +# A readable name for the entry and the Network Descriptor. Optional; the id is +# used when it is unset. +BECKN_NETWORK_NAME=OAN Dev +# Where the network layer is reached -- published in the registry entry, and +# the descriptor's services.discovery is this plus /discover. Nothing inside +# this stack resolves it, so a wrong value costs nothing locally and everything +# on a shared network. +NETWORK_BASE_URL=https://oan-dev.local +# Seconds another network may keep what it crawls from this one, published in +# the Network Descriptor. Optional; omitted from the descriptor when unset. +FEDERATION_PROJECTION_TTL=30 # ---- the four upstream providers ------------------------------------------- -# setup.py creates all four plus a capability binding each -- seven participants -# and four bindings in all. The registry is unreachable from outside the stack, -# so there is no second way to create them. +# setup.py creates all four plus a capability binding each -- five participants +# (the network itself and these four) and four bindings in all. The registry is +# unreachable from outside the stack, so there is no second way to create them. # # THE BINDING KEY IS |. The provider adapter builds # it from the incoming payload and matches it against its own config; setup.py diff --git a/quick-start/.gitignore b/quick-start/.gitignore index ac5fe84d..dfd0af7f 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -4,6 +4,9 @@ # Private keys. Generated by bin/setup.py, never committed. keys/ +# Federation documents, generated by `make host-descriptor`. +.well-known/ + # Python's bytecode cache, left behind by running bin/setup.py. bin/__pycache__/ diff --git a/quick-start/Makefile b/quick-start/Makefile index 1aae6475..78dab5a1 100644 --- a/quick-start/Makefile +++ b/quick-start/Makefile @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core down destroy setup reverse-proxy observability pull restart restart-edge ps logs +.PHONY: help up up-core down destroy setup host-descriptor reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -39,6 +39,9 @@ observability: ; @$(STACK) observability setup: ; @$(STACK) setup +# Generate and serve the federation documents. Re-run when the entry changes. +host-descriptor: ; @$(STACK) host-descriptor + # NPM chowns the files it is given, so a plain `git pull` fails on them. pull: ; @$(STACK) pull diff --git a/quick-start/README.md b/quick-start/README.md index 1165f014..b489f7e8 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -96,8 +96,10 @@ Three tiers, in the only order that works: registry and discovery, then `bin/setup.py`, then the mocks and the three adapters. Allow up to five minutes the first time — Keycloak on a cold volume. -`setup.py` generates a keypair per adapter, registers seven participants and four -capability bindings, and renders the three adapter configs. Nothing needs +`setup.py` generates the network's one keypair, registers five participants -- +one entry for the network (`BECKN_NETWORK_ID`, shared by all three adapters) and +four upstreams -- and four capability bindings, and renders the three adapter +configs. Nothing needs creating by hand. → Appendix C for why the order matters, Appendix D for what it wrote. @@ -110,7 +112,7 @@ make ps Answers `select`, and the only layer that calls an upstream. Serves all four capabilities from one adapter. -`.env` keys: `PROVIDER_SUBSCRIBER_ID`, and the four pairs that become binding +`.env` keys: the four pairs that become binding keys — `MAUSAMGRAM_PARTICIPANT_ID` + `MAUSAMGRAM_CAPABILITY`, `AGMARKNET_PARTICIPANT_ID` + `AGMARKNET_CAPABILITY`, `VISTAAR_PARTICIPANT_ID` + `VISTAAR_CAPABILITY`, `POCRA_PARTICIPANT_ID` + `POCRA_CAPABILITY`. @@ -136,8 +138,11 @@ All four capability steps should be listed by name. Fronts discovery: verifies the caller, passes `discover` and `publish` on, and re-signs as itself. -`.env` keys: `NETWORK_SUBSCRIBER_ID`. `BECKN_NETWORK_ID` belongs to the discovery -service behind it — a `discover` naming a different network finds nothing. +`.env` keys: `BECKN_NETWORK_ID` and `NETWORK_BASE_URL`. `BECKN_NETWORK_ID` is the +network's one identity -- all three adapters sign as it -- and is also the +discovery service's network, so a `discover` naming a different network finds +nothing. `NETWORK_BASE_URL` is where this layer is reached; the Network +Descriptor publishes its `discover` endpoint from it. ```sh docker compose logs network-adapter | grep 'Server listening' @@ -148,7 +153,7 @@ docker compose logs network-adapter | grep 'Server listening' The consumer's edge. Sends `discover` to the network layer and `select` straight to the provider layer, per `config/adapters/routing-consumer.yaml`. -`.env` keys: `CONSUMER_SUBSCRIBER_ID`. +`.env` keys: none of its own; it signs as `BECKN_NETWORK_ID`. ```sh docker compose logs consumer-adapter | grep 'Server listening' @@ -461,7 +466,7 @@ an `allow`. | File | How it applies | |---|---| | `npm-custom/http_top.conf` | **Automatic**, top of the `http` block. The `consumer` rate-limit zone and `limit_req_status 429`. | -| `npm-custom/server_proxy.conf` | **Automatic**, every server block. The `/publish` deny. | +| `npm-custom/server_proxy.conf` | **Automatic**, every server block. The `/publish` deny, and the Network Descriptor, JWKS and schema packs from `.well-known/` (see Appendix D). | | `npm-advanced/consumer.conf` | **Manual** — paste into the consumer host's Advanced tab. `limit_req` for that host only; a 10 r/s ceiling on signed peer traffic would throttle for no gain. | | `npm-advanced/hyperdx.conf` | **Manual** — paste into the HyperDX host's Advanced tab. 300s timeouts for ClickHouse scans, buffering off for live tail. Carries no credential; the Access List does. | @@ -513,7 +518,7 @@ loopback. A registry route would depend on that silently. ``` 1. registry and discovery (also sunbird-registry-postgres, sunbird-registry-keycloak, discovery-postgres) -2. bin/setup.py keys, seven participants, four bindings, three configs +2. bin/setup.py keys, five participants, four bindings, three configs 3. mock upstreams, then the three adapters 4. nginx-proxy-manager the public edge — 80 and 443, all interfaces 5. hyperdx ClickStack @@ -550,10 +555,24 @@ evidence `server_proxy.conf` is still mounted. `setup.py` wrote all of it. Nothing here is a step; it is what to look at when something does not match. -**Three `node` rows, one per adapter** — an id, a role (`consumer`, `provider`, -`network`) and the public halves of a keypair. Private halves stay in -`keys/keys.json` and are never in the registry. Keys are published as bare -base64, no encoding label. +**One `node` row, for the network** — `BECKN_NETWORK_ID` as its id, every layer +it serves as its role (`["consumer", "network", "provider"]`), and the public +half of ONE keypair, shared by all three adapters. The private half stays in +`keys/keys.json` and is never in the registry. Keys are published as bare +base64, no encoding label. The key's own `osid` is the network's key id: what +the adapters sign with, and what the Network Descriptor and JWKS publish. + +**The Network Descriptor and JWKS** are generated FROM that row, not stored in +the registry. `make host-descriptor` signs them with the network's key and writes +`.well-known/openagrinet` and `.well-known/jwks.json`, plus +`.well-known/schema-packs.json` -- the list of schema packs this network speaks, +which the descriptor's `schemaPacks` links to as one URL rather than listing. +NPM then serves all three (`npm-custom/server_proxy.conf`); the command checks +they are live, which needs a proxy host for the networkId (Appendix B). Re-run +it whenever the row changes -- each run publishes the next revision, and another +network refuses one that does not increase. `FEDERATION_PROJECTION_TTL` in `.env`, if +set, is published as how long another network may keep what it crawls from +this one. **Four `upstream` rows, one per provider** — an ordinary HTTP API this deployment calls. It signs nothing, so it needs no role and no keys. Holds a @@ -1003,7 +1022,7 @@ config/ reverse-proxy/ npm-custom/ mounted to /data/nginx/custom; NPM includes these http_top.conf on its own -- rate-limit zone, and the /publish - server_proxy.conf deny every proxy host gets + server_proxy.conf deny and the .well-known documents every proxy host gets npm-advanced/ NOT loaded. Paste into a host's Advanced tab; kept consumer.conf here because a textarea in a database is not hyperdx.conf reviewable. consumer = rate limit; hyperdx = ClickHouse @@ -1051,7 +1070,7 @@ for v in sunbird-registry-postgres-data discovery-postgres-data nginx-proxy-mana done ``` -Then `make up`, and confirm the seven participants and your proxy hosts before +Then `make up`, and confirm the five participants and your proxy hosts before deleting anything. Keycloak shares `sunbird-registry-postgres-data`, so its realm travels with that volume — and equally does not survive if you skip it. diff --git a/quick-start/bin/oanfed.py b/quick-start/bin/oanfed.py new file mode 100644 index 00000000..ac7ebbb0 --- /dev/null +++ b/quick-start/bin/oanfed.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""oanfed -- publish and verify a network's federation documents. + +Served at https://{networkId}/.well-known/: openagrinet (signed descriptor), +jwks.json and schema-packs.json, all generated from our registry entry. + + oanfed.py publish generate and sign this network's documents + oanfed.py verify fetch a network's documents and check them +""" + +import argparse, base64, binascii, http.client, json, os, pathlib, re, ssl, sys, urllib.error, urllib.parse, urllib.request +from datetime import datetime, timezone + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives import serialization as ser + +# Fixed: a peer derives them from the networkId alone. +WELL_KNOWN_DESCRIPTOR = "/.well-known/openagrinet" +WELL_KNOWN_JWKS = "/.well-known/jwks.json" +WELL_KNOWN_SCHEMA_PACKS = "/.well-known/schema-packs.json" + +# A peer is not trusted yet when fetched; cap what it can make us read. +MAX_DOCUMENT_BYTES = 1 << 20 + +# JOSE name per RFC 9864; anything else is refused. +ALGORITHM = "Ed25519" + +DESCRIPTOR_VERSION = "1.0" +DESCRIPTOR_STATUSES = ("Active", "Suspended", "Retired") + +# How far a peer's issuedAt may run ahead of our clock. +MAX_CLOCK_SKEW_SECONDS = 300 + +class FederationError(Exception): + """Anything a caller is expected to act on rather than crash over.""" + + +# --- canonical form ------------------------------------------------------- + +# JCS numbers are exact only up to 2**53. +MAX_SAFE_INTEGER = 2 ** 53 - 1 + + +def canonical(descriptor): + """The bytes that get signed: RFC 8785 (JCS) of everything but the signature.""" + without_signature = {k: v for k, v in descriptor.items() if k != "signature"} + return _jcs(without_signature).encode("utf-8") + + +def _jcs(value): + """One JCS value. Floats are refused: their formatting differs across implementations.""" + if value is None or isinstance(value, bool): + return json.dumps(value) + if isinstance(value, int): + if abs(value) > MAX_SAFE_INTEGER: + raise FederationError(f"integer {value} is outside the range JCS signs exactly") + return str(value) + if isinstance(value, float): + raise FederationError(f"non-integer number {value!r} cannot be signed; use an integer") + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError as err: + raise FederationError("string is not valid Unicode (lone surrogate)") from err + # Python's escaping matches JCS. + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list): + return "[" + ",".join(_jcs(v) for v in value) + "]" + if isinstance(value, dict): + # JCS sorts by UTF-16 code units, not code points. + keys = sorted(value, key=lambda k: k.encode("utf-16-be")) + return "{" + ",".join(f"{_jcs(k)}:{_jcs(value[k])}" for k in keys) + "}" + raise FederationError(f"{type(value).__name__} cannot be signed") + + +# --- keys ----------------------------------------------------------------- + +def b64url(raw): + """base64url without padding, as a JWKS carries it.""" + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def unb64url(encoded): + if not isinstance(encoded, str) or not re.fullmatch(r"[A-Za-z0-9_-]*", encoded): + raise FederationError("value is not base64url") + return base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + + +def jwk(public, kid): + """The JWK for a raw Ed25519 public key.""" + return {"kty": "OKP", "crv": "Ed25519", "kid": kid, "x": b64url(public)} + + +def load_signing_key(path): + """This network's private key, from setup.py's keys.json; refused if others can read it.""" + path = pathlib.Path(path) + if not path.is_file(): + raise FederationError(f"{path} does not exist; run setup.py first") + if path.stat().st_mode & 0o077: + raise FederationError( + f"{path} is readable by group or others; chmod 600 it before signing with it") + try: + raw = base64.b64decode(json.loads(path.read_text())["signingPrivate"], validate=True) + except (ValueError, KeyError, binascii.Error) as err: + raise FederationError(f"{path} holds no readable signingPrivate: {err}") from err + if len(raw) != 32: + raise FederationError(f"{path} signingPrivate is {len(raw)} bytes, want 32") + return Ed25519PrivateKey.from_private_bytes(raw) + + +def public_of(private): + return private.public_key().public_bytes( + encoding=ser.Encoding.Raw, format=ser.PublicFormat.Raw + ) + + +# --- the published documents: shape --------------------------------------- + +def _is_https_url(value): + if not isinstance(value, str): + return False + parsed = urllib.parse.urlsplit(value) + return parsed.scheme == "https" and bool(parsed.hostname) + + +def validate_schema_packs(packs): + """The schemaPacks list: distinct https URLs.""" + if not (isinstance(packs, list) and all(_is_https_url(p) for p in packs) + and len(set(packs)) == len(packs)): + raise FederationError("schema packs is not a list of distinct https URLs") + return packs + + +def _parse_time(value): + if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", value): + raise FederationError(f"issuedAt {value!r} is not a UTC timestamp like 2026-09-17T10:00:00Z") + try: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + except ValueError as err: + raise FederationError(f"issuedAt {value!r} is not a real date") from err + + +def _nonempty_str(value): + return isinstance(value, str) and value.strip() != "" + + +def validate_keyset(keyset): + """A checked JWKS, as {kid: raw public key} for its Ed25519 keys. Duplicate kids are refused.""" + if not isinstance(keyset, dict) or not isinstance(keyset.get("keys"), list): + raise FederationError("JWKS is not an object with a keys array") + found, seen = {}, set() + for entry in keyset["keys"]: + if not isinstance(entry, dict) or not _nonempty_str(entry.get("kid")): + raise FederationError("JWKS holds a key without a kid") + kid = entry["kid"] + if kid in seen: + raise FederationError(f"JWKS holds two keys with kid {kid!r}") + seen.add(kid) + if entry.get("kty") != "OKP" or entry.get("crv") != "Ed25519": + continue + if entry.get("alg", ALGORITHM) != ALGORITHM: + raise FederationError(f"key {kid!r} declares alg {entry.get('alg')!r}") + if entry.get("use", "sig") != "sig": + raise FederationError(f"key {kid!r} is not a signing key") + raw = unb64url(entry.get("x", "")) + if len(raw) != 32: + raise FederationError(f"key {kid!r} is {len(raw)} bytes, want 32") + found[kid] = raw + return found + + +def validate_descriptor(descriptor, now=None): + """The Network Descriptor contract, checked before signing and after fetching.""" + if not isinstance(descriptor, dict): + raise FederationError("descriptor is not a JSON object") + + def need(ok, message): + if not ok: + raise FederationError(message) + + need(descriptor.get("version") == DESCRIPTOR_VERSION, + f"descriptor version {descriptor.get('version')!r} is not {DESCRIPTOR_VERSION!r}") + network_id = descriptor.get("networkId") + need(isinstance(network_id, str) + and re.fullmatch(r"(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}", + network_id) is not None, + f"networkId {network_id!r} is not a lowercase DNS domain") + need(_nonempty_str(descriptor.get("name")), "name is missing") + operator = descriptor.get("operator") + need(isinstance(operator, dict) and _nonempty_str(operator.get("name")), + "operator.name is missing") + need("operatorId" not in operator or _nonempty_str(operator["operatorId"]), + "operator.operatorId is empty") + revision = descriptor.get("revision") + need(isinstance(revision, int) and not isinstance(revision, bool) + and 1 <= revision <= MAX_SAFE_INTEGER, + f"revision {revision!r} is not a positive integer") + need(descriptor.get("status") in DESCRIPTOR_STATUSES, + f"status {descriptor.get('status')!r} is not one of {', '.join(DESCRIPTOR_STATUSES)}") + issued = _parse_time(descriptor.get("issuedAt")) + now = now or datetime.now(timezone.utc) + need((issued - now).total_seconds() <= MAX_CLOCK_SKEW_SECONDS, + f"issuedAt {descriptor['issuedAt']} is in the future") + profile = descriptor.get("operatingProfile") + need(isinstance(profile, list) and profile and all(_nonempty_str(p) for p in profile), + "operatingProfile is not a non-empty list of versions") + # Unsigned, so it must sit on the network's own domain. + packs = descriptor.get("schemaPacks") + need(_is_https_url(packs) + and urllib.parse.urlsplit(packs).hostname == str(network_id).lower(), + "schemaPacks is not an https URL on the network's own domain") + services = descriptor.get("services") + need(isinstance(services, dict) and _is_https_url(services.get("discovery")), + "services.discovery is not an https URL") + contacts = descriptor.get("contacts") + need(contacts is None or (isinstance(contacts, dict) + and all(_nonempty_str(v) for v in contacts.values())), + "contacts is not an object of strings") + ttl = descriptor.get("projectionTtl") + need(ttl is None or (isinstance(ttl, int) and not isinstance(ttl, bool) and ttl >= 0), + f"projectionTtl {ttl!r} is not a non-negative integer") + + +# --- sign and verify ------------------------------------------------------ + +def sign(descriptor, private, key_id): + """A copy of the descriptor carrying its signature.""" + signed = {k: v for k, v in descriptor.items() if k != "signature"} + signed["signature"] = { + "keyId": key_id, + "algorithm": ALGORITHM, + "value": b64url(private.sign(canonical(signed))), + } + return signed + + +def verify(descriptor, keyset, now=None): + """Check a descriptor's shape and signature against the JWKS. Proves origin, not trust.""" + validate_descriptor(descriptor, now) + keys = validate_keyset(keyset) + + signature = descriptor.get("signature") + if not isinstance(signature, dict): + raise FederationError("descriptor carries no signature") + if signature.get("algorithm") != ALGORITHM: + raise FederationError(f"unsupported signature algorithm: {signature.get('algorithm')!r}") + key_id = signature.get("keyId") + if key_id not in keys: + raise FederationError(f"no published Ed25519 key matches the signature's keyId {key_id!r}") + + try: + Ed25519PublicKey.from_public_bytes(keys[key_id]).verify( + unb64url(signature.get("value", "")), canonical(descriptor) + ) + except Exception as err: + raise FederationError("signature does not verify") from err + +# --- fetching a peer's documents ------------------------------------------ + +class ResolvingHTTPSConnection(http.client.HTTPSConnection): + """Like curl --resolve: connects to an address, still verifying the certificate's name.""" + + def __init__(self, server_name, address, **kwargs): + host, _, port = address.partition(":") + super().__init__(host, int(port or 443), **kwargs) + self._server_name = server_name + + def connect(self): + super(http.client.HTTPSConnection, self).connect() + self.sock = self._context.wrap_socket(self.sock, server_hostname=self._server_name) + + +class Fetcher: + """Retrieves and verifies another network's documents. scheme is https outside local tests.""" + + def __init__(self, scheme="https", ca_file=None, resolve=None, timeout=10): + self.scheme, self.resolve, self.timeout = scheme or "https", resolve, timeout + # A different root CA, never verification turned off. + self.context = ssl.create_default_context(cafile=ca_file) if ca_file else None + + def fetch(self, network_id): + """Fetch a network's descriptor, JWKS and schema packs, verified against each other.""" + if not network_id: + raise FederationError("no networkId given") + + descriptor = self._get(network_id, WELL_KNOWN_DESCRIPTOR) + keyset = self._get(network_id, WELL_KNOWN_JWKS) + + if descriptor.get("networkId") != network_id: + raise FederationError( + f"{network_id} publishes a descriptor for {descriptor.get('networkId')!r}") + verify(descriptor, keyset) + where = urllib.parse.urlsplit(descriptor["schemaPacks"]) + served = self._get(network_id, where.path + (f"?{where.query}" if where.query else "")) + if not isinstance(served, dict): + raise FederationError(f"{network_id} serves schema packs that are not a JSON object") + packs = validate_schema_packs(served.get("schemaPacks")) + return descriptor, keyset, packs + + def _get(self, network_id, path): + where = f"{self.scheme}://{network_id}{path}" + try: + if self.scheme == "https": + connection = ( + ResolvingHTTPSConnection(network_id, self.resolve, + context=self.context, timeout=self.timeout) + if self.resolve else + http.client.HTTPSConnection(network_id, context=self.context, + timeout=self.timeout) + ) + else: + host = self.resolve or network_id + connection = http.client.HTTPConnection(host, timeout=self.timeout) + + connection.request("GET", path, headers={"Accept": "application/json", + "Host": network_id}) + response = connection.getresponse() + if response.status != 200: + raise FederationError(f"{where} returned {response.status} {response.reason}") + body = response.read(MAX_DOCUMENT_BYTES + 1) + if len(body) > MAX_DOCUMENT_BYTES: + raise FederationError(f"{where} is larger than {MAX_DOCUMENT_BYTES} bytes") + except FederationError: + raise + except Exception as err: + raise FederationError(f"fetching {where}: {err}") from err + finally: + try: + connection.close() + except Exception: + pass + + try: + return json.loads(body) + except json.JSONDecodeError as err: + raise FederationError(f"parsing {where}: {err}") from err + + +class ReadOnlyRegistry: + """Registry searches, which need no credentials.""" + + def __init__(self, url, timeout=30): + self.url, self.timeout = url.rstrip("/"), timeout + + def _search(self, entity, filters): + request = urllib.request.Request( + f"{self.url}/api/v1/{entity}/search", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + found = json.load(response) + except urllib.error.HTTPError as err: + raise FederationError( + f"registry search {entity} returned {err.code}: " + f"{err.read().decode()[:200]}") from err + except Exception as err: + raise FederationError(f"reaching the registry at {self.url}: {err}") from err + return found.get("data", found) if isinstance(found, dict) else found + + def participant(self, participant_id): + rows = self._search("Participant", {"participantId": {"eq": participant_id}}) + return rows[0] if rows else None + + def schema_packs(self): + """The active schema pack URLs this deployment speaks.""" + rows = self._search("SchemaRegistry", {"status": {"eq": "active"}}) + return sorted({r["schemaUrl"] for r in rows if r.get("schemaUrl")}) + + +def _registry_time(value): + """A registry timestamp with any fraction length (Python < 3.11 reads 3 or 6). No zone is UTC.""" + text = re.sub(r"\.(\d+)", lambda m: "." + m.group(1)[:6].ljust(6, "0"), + str(value).replace("Z", "+00:00"), count=1) + parsed = datetime.fromisoformat(text) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def registered_signing_keys(entry, now=None): + """The entry's active, unexpired Ed25519 keys, including ones not valid yet.""" + now = now or datetime.now(timezone.utc) + usable = [] + for k in entry.get("keys") or []: + if str(k.get("alg", "")).lower() != "ed25519" or k.get("status", "active") != "active": + continue + name = k.get("osid", "?") + until = k.get("validUntil") + if until: + try: + expiry = _registry_time(until) + except ValueError as err: + raise FederationError(f"registry key {name} has an unreadable validUntil {until!r}") from err + if expiry <= now: + continue + try: + raw = base64.b64decode(str(k.get("key", "")).removeprefix("base64:"), validate=True) + except binascii.Error as err: + raise FederationError(f"registry key {name} is not base64") from err + if len(raw) != 32: + raise FederationError(f"registry key {name} is not a 32-byte Ed25519 key") + if raw not in usable: + usable.append(raw) + return usable + + +def last_published_revision(path): + """The revision already on disk, or 0. Unreadable is an error, so a revision never goes back.""" + if not path.exists(): + return 0 + try: + return int(json.loads(path.read_text())["revision"]) + except Exception as err: + raise FederationError(f"cannot read the revision in {path}: {err}") from err + + +def write_atomically(path, document): + """Replace a published file in one step, so a reader never sees half of it.""" + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n") + os.replace(tmp, path) + + +def cmd_publish(args): + """Generate and sign this network's documents from its registry entry. + + --keys must hold the entry's active key: it signs both the descriptor and traffic. + """ + out = pathlib.Path(args.out) / ".well-known" + descriptor_path, jwks_path = out / "openagrinet", out / "jwks.json" + packs_path = out / "schema-packs.json" + + held = last_published_revision(descriptor_path) + if args.revision <= held: + raise FederationError( + f"revision {args.revision} does not increase on the published {held}; " + f"every peer holding {held} would refuse it") + + registry = ReadOnlyRegistry(args.registry_url, args.registry_timeout) + entry = registry.participant(args.network_id) + if entry is None: + raise FederationError( + f"{args.network_id} is not in this registry -- seed it first, " + f"then publish from it") + + registered = registered_signing_keys(entry) + if not registered: + raise FederationError( + f"{args.network_id} has no active Ed25519 key in the registry to sign with") + + network_key = load_signing_key(args.keys) + if public_of(network_key) not in registered: + raise FederationError( + f"{args.keys} is not an active key registered for {args.network_id}; " + f"publishing it would sign the descriptor with one key and traffic with another") + + base_url = entry.get("baseUrl", "").rstrip("/") + # Matched by bytes: during a rotation the entry holds several keys. + ours = [k for k in entry.get("keys") or [] + if str(k.get("key", "")).removeprefix("base64:") == base64.b64encode(public_of(network_key)).decode()] + key_id = (ours[0].get("osid") if ours else None) or "" + if not key_id: + raise FederationError(f"{args.network_id}'s registry key has no osid") + network_jwk = jwk(public_of(network_key), key_id) + + descriptor = { + "version": DESCRIPTOR_VERSION, + "networkId": entry["participantId"], + "name": entry.get("name") or entry["participantId"], + "operator": {"name": args.operator or entry.get("name") or entry["participantId"]}, + "revision": args.revision, + "status": args.status, + "issuedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "operatingProfile": [p.strip() for p in args.operating_profile.split(",") if p.strip()], + "schemaPacks": f"https://{entry['participantId']}{WELL_KNOWN_SCHEMA_PACKS}", + "services": {"discovery": f"{base_url}/discover"}, + } + if args.projection_ttl is not None: + descriptor["projectionTtl"] = args.projection_ttl + + packs = validate_schema_packs(registry.schema_packs()) + signed = sign(descriptor, network_key, key_id) + keyset = {"keys": [network_jwk]} + # Checked as a peer would, before anything is written. + verify(signed, keyset) + + # Descriptor last, so it never names a key not yet published. + out.mkdir(parents=True, exist_ok=True) + write_atomically(packs_path, {"schemaPacks": packs}) + write_atomically(jwks_path, keyset) + write_atomically(descriptor_path, signed) + + print(f"{args.network_id}: descriptor written at revision {args.revision}") + print(f" descriptor {descriptor_path}") + print(f" keys {jwks_path}") + print(f" packs {packs_path} ({len(packs)})") + print(f" key {network_jwk['kid']}") + return 0 + + +def cmd_verify(args): + """Fetch another network's documents and check them.""" + descriptor, keyset, packs = fetcher_from(args).fetch(args.network_id) + print(f"verified {descriptor['networkId']} at revision {descriptor.get('revision')}, " + f"status {descriptor.get('status')}") + print(f" discovery {(descriptor.get('services') or {}).get('discovery')}") + print(f" keys {len(keyset.get('keys', []))} published") + print(f" packs {len(packs)} at {descriptor['schemaPacks']}") + print(" this proves the documents are self-consistent, NOT that the network " + "should be trusted -- admission is that decision") + return 0 + + +def fetcher_from(args): + return Fetcher(scheme=args.scheme, ca_file=args.ca_file or None, + resolve=args.resolve or None, timeout=args.timeout) + + +# --- flags ---------------------------------------------------------------- + +def add_peer_access(parser): + """How an operator reaches a peer's documents.""" + parser.add_argument("--scheme", default="https", + help="http only for a stack with no certificates at all") + parser.add_argument("--ca-file", default="", + help="verify the peer's certificate against the CAs in this PEM file") + parser.add_argument("--resolve", default="", + help="reach the peer at this host:port instead of resolving its domain") + parser.add_argument("--timeout", type=float, default=10, help="per-request timeout") + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="oanfed.py", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + publish = commands.add_parser("publish", help="generate and sign this network's documents") + publish.add_argument("--network-id", required=True, + help="this network's DNS domain, which is its identity") + publish.add_argument("--registry-url", required=True, + help="this network's own registry, which holds the entry " + "this descriptor is generated FROM") + publish.add_argument("--registry-timeout", type=float, default=30, + help="per-request timeout for the registry") + publish.add_argument("--operator", default="", + help="organisation answering for this network; " + "defaults to the registered name") + publish.add_argument("--revision", type=int, required=True, + help="descriptor revision; must increase on every republish") + publish.add_argument("--operating-profile", default="OAN/v0.1,Beckn/v2", + help="comma-separated version stack") + publish.add_argument("--status", default="Active", choices=DESCRIPTOR_STATUSES, + help="what this network declares itself to be") + publish.add_argument("--projection-ttl", type=int, default=None, + help="seconds a peer may keep what it crawls; omit to let peers decide") + publish.add_argument("--out", required=True, help="directory to write .well-known/ into") + publish.add_argument("--keys", required=True, + help="the keys.json setup.py writes; its signing key signs the " + "descriptor AND the network's traffic, so it must be the " + "key registered on the entry") + publish.set_defaults(run=cmd_publish) + + verify_cmd = commands.add_parser("verify", help="fetch and check another network's documents") + verify_cmd.add_argument("--network-id", required=True, help="the peer to check") + add_peer_access(verify_cmd) + verify_cmd.set_defaults(run=cmd_verify) + + args = parser.parse_args(argv) + try: + return args.run(args) + except FederationError as err: + print(f"oanfed {args.command}: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 541ef06e..bf80a9d7 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -8,11 +8,11 @@ it renders are bind-mounted files, and an adapter started before they exist leaves a directory in their place. -WHAT IT WRITES. Seven participants, four capability bindings, four schemas: +WHAT IT WRITES. Five participants, four capability bindings, four schemas: - 3 x node one per adapter -- consumer, network, provider -- each with the - public halves of a keypair. The private halves stay in - keys/keys.json and never reach the registry. + 1 x node the network itself, BECKN_NETWORK_ID, shared by all three + adapters, with the public half of ONE keypair. The private + half stays in keys/keys.json. 4 x upstream the APIs this deployment calls. ALL FOUR are real external hosts now, whose base URLs come from .env -- no mock is in use by any capability. An upstream signs nothing, so it needs @@ -110,11 +110,10 @@ def is_placeholder(value): # # Checked together, up front, for two reasons. seed() runs BEFORE render(), so a # placeholder in a render-only key like VISTAAR_TOKEN_URL would otherwise be -# found only after seven rows already existed -- unfixable, per the note above. +# found only after five rows already existed -- unfixable, per the note above. # And one list beats being sent back to .env ten times in a row. REQUIRED = [ - "CONSUMER_SUBSCRIBER_ID", "NETWORK_SUBSCRIBER_ID", "PROVIDER_SUBSCRIBER_ID", - "CONSUMER_BASE_URL", "NETWORK_BASE_URL", "PROVIDER_BASE_URL", + "BECKN_NETWORK_ID", "NETWORK_BASE_URL", "MAUSAMGRAM_PARTICIPANT_ID", "MAUSAMGRAM_CAPABILITY", "MAUSAMGRAM_BASE_URL", "MAUSAMGRAM_AUTH", "MAUSAMGRAM_MAPPING_URL", "AGMARKNET_PARTICIPANT_ID", "AGMARKNET_CAPABILITY", "AGMARKNET_BASE_URL", @@ -166,37 +165,32 @@ def x25519_pair(): def load_or_generate_keys(): - """Keys persist across runs: the registry already holds the public halves.""" - roles = (("consumer", env("CONSUMER_SUBSCRIBER_ID")), - ("network", env("NETWORK_SUBSCRIBER_ID")), - ("provider", env("PROVIDER_SUBSCRIBER_ID"))) + """The network's one identity. Keys persist across runs.""" + participant = env("BECKN_NETWORK_ID") if KEYS.exists(): print("keys: reusing keys/keys.json") - identities = json.loads(KEYS.read_text()) + identity = json.loads(KEYS.read_text()) # The keypair persists; the id is read from .env each run. This registry - # cannot update a record, so changing an id in .env seeds a NEW + # cannot update a record, so changing the id in .env seeds a NEW # participant rather than editing one -- and the same keypair moving to # the new id is what makes that a rename rather than a rekey. - for role, participant in roles: - if identities[role]["participantId"] != participant: - print(f" {role}: id is now {participant}, keeping the keypair") - identities[role]["participantId"] = participant - KEYS.write_text(json.dumps(identities, indent=2)) - return identities + if identity["participantId"] != participant: + print(f" id is now {participant}, keeping the keypair") + identity["participantId"] = participant + KEYS.write_text(json.dumps(identity, indent=2)) + return identity print("keys: generating") - identities = {} - for role, participant in roles: - sign_private, sign_public = ed25519_pair() - encr_private, encr_public = x25519_pair() - identities[role] = {"participantId": participant, - "signingPrivate": sign_private, "signingPublic": sign_public, - "encrPrivate": encr_private, "encrPublic": encr_public} + sign_private, sign_public = ed25519_pair() + encr_private, encr_public = x25519_pair() + identity = {"participantId": participant, + "signingPrivate": sign_private, "signingPublic": sign_public, + "encrPrivate": encr_private, "encrPublic": encr_public} KEYS.parent.mkdir(exist_ok=True) - KEYS.write_text(json.dumps(identities, indent=2)) + KEYS.write_text(json.dumps(identity, indent=2)) KEYS.chmod(0o600) - return identities + return identity # ----------------------------------------------------------------- registry @@ -283,9 +277,8 @@ def signing_key_block(public_key): to a base64 decoder is the value as published, and a label left on fails every verification with a decode error pointing nowhere near the registry. - No friendly key id, because nothing could look one up: the registry assigns - an osid on write and that is what a sender names in the Authorization - header. No use either -- alg carries the purpose, ed25519 signs.""" + No key id of our own: the registry assigns the key an osid, which is what a + sender names. No use either -- alg carries the purpose, ed25519 signs.""" return [{"alg": "ed25519", "key": public_key, "status": "active", "validFrom": "2026-01-01T00:00:00Z", "validUntil": "2030-01-01T00:00:00Z"}] @@ -367,7 +360,7 @@ def ensure_binding(bearer, participant_id, capability, path, mapping_url, ) -def schema_url(capability, version, ref): +def schema_url(capability, version): """Build a pack URL from the capability code, which already contains the pack directory's name: openagrinet:WeatherObservation is served out of schema/WeatherObservation. Deriving it means the two cannot drift. @@ -375,19 +368,22 @@ def schema_url(capability, version, ref): The registry pattern-checks the result against exactly this shape, so a capability whose pack is laid out differently is refused rather than stored -- which is the behaviour to want, since all eight packs in - network-specs are //attributes.yaml today.""" - return ("https://raw.githubusercontent.com/OpenAgriNet/network-specs/" - f"{ref}/schema/{capability.partition(':')[2]}/{version}" + network-specs are //attributes.yaml today. + + openagrinet.github.io, which catalogs carry in their @context; a crawl + derived from a raw.githubusercontent URL matches nothing.""" + return ("https://openagrinet.github.io/network-specs/" + f"schema/{capability.partition(':')[2]}/{version}" "/attributes.yaml") -def ensure_capability_schema(bearer, capability, name, version, ref): +def ensure_capability_schema(bearer, capability, name, version): """Create a capability's schema row only when absent. THE REGISTRY NEVER FETCHES schemaUrl -- it only checks its shape. So a row pointing at a ref that does not serve the pack stores perfectly happily and - fails later, at whoever resolves it. Getting SCHEMA_PACK_REF right matters - more than the write succeeding. + fails later, at whoever resolves it. Getting the URL right matters more + than the write succeeding. And it cannot be corrected afterwards: SchemaRegistry.json sets additionalProperties false, so an update is rejected over the system fields @@ -398,7 +394,7 @@ def ensure_capability_schema(bearer, capability, name, version, ref): return result = post("SchemaRegistry", { "capabilityCode": capability, "name": name, "version": version, - "schemaUrl": schema_url(capability, version, ref), + "schemaUrl": schema_url(capability, version), "status": "active"}, bearer) print(f" {capability}: {result['params']['status']} " f"{result['params'].get('errmsg', '')[:160]}") @@ -415,26 +411,23 @@ def ensure_participant(bearer, participant_id, payload): print(f" {participant_id}: {status} {result['params'].get('errmsg', '')[:160]}") -def seed(identities): +def seed(identity): wait_for_registry() bearer = token() - # Seven participants and four capability bindings, all of it from here. + # Five participants and four capability bindings, all of it from here. # # The registry is not reachable from outside this stack -- no published # port beyond loopback and no proxy host in front of it -- so there is no # second way to create these. Everything the network needs to answer a # request has to exist by the time this returns. - print("registry: three adapter identities") - for role, name, network_role in ( - ("consumer", "OAN consumer layer adapter", "consumer"), - ("network", "OAN network layer adapter", "network"), - ("provider", "OAN provider layer adapter", "provider")): - identity = identities[role] - ensure_participant(bearer, identity["participantId"], - node(identity["participantId"], name, network_role, - env(f"{role.upper()}_BASE_URL"), - identity["signingPublic"])) + # One entry for the network, shared by all three adapters. + print("registry: one network identity") + ensure_participant(bearer, identity["participantId"], + node(identity["participantId"], + env("BECKN_NETWORK_NAME", identity["participantId"]), + list(ROLES), env("NETWORK_BASE_URL"), + identity["signingPublic"])) # All four upstreams are real external providers now -- no mock is in use # by any capability, so every base URL comes from .env and the stack needs @@ -500,63 +493,48 @@ def seed(identities): # # SEPARATE FROM THE MAPPING URLS ABOVE, which live in helmcharts and # describe one upstream's wire format. These are network-specs and describe - # the network's own vocabulary, so they move on their own ref. + # the network's own vocabulary, published on its own site. print("registry: four capability schemas") - ref = env("SCHEMA_PACK_REF", "main") for var, name, version in CAPABILITY_SCHEMAS: - ensure_capability_schema(bearer, env(var), name, version, ref) - + ensure_capability_schema(bearer, env(var), name, version) -def key_osids(identities): - """Read back each key's osid, and check the registry still holds the public - half we have the private half for. - The Authorization header names a key by its osid rather than by the friendly - id, so the adapters have to be configured with the value the registry - assigned. +def key_osid(identity): + """Check the registry still holds the public half we have the private half + for, and read back the key's osid: the key id the adapters sign with. The mismatch check matters because this registry cannot update a record and its delete is soft: a participant seeded against an earlier keys.json keeps that public key forever. Signing with a new private half would then produce signatures nobody can verify -- and the failure would surface much later, as an authentication error with no obvious cause.""" - for role, identity in identities.items(): - records = search("Participant", {"participantId": {"eq": identity["participantId"]}}) - keys = (records[0].get("keys") or []) if records else [] - if not keys: - sys.exit(f"setup: {identity['participantId']} has no published key") - - # Bare base64 now, but an older row may still carry the label, and a - # confusing mismatch error is worse than one tolerant line. - published = keys[0]["key"].removeprefix("base64:") - if published != identity["signingPublic"]: - sys.exit( - f"setup: {identity['participantId']} is registered with a different key.\n" - f" This registry cannot update a record, and its delete is soft and keeps\n" - f" the unique index, so the id cannot be reused. Either restore the\n" - f" matching keys/keys.json, or pick a new id for {role.upper()}_SUBSCRIBER_ID\n" - f" in .env and re-run.") - identity["keyOsid"] = keys[0]["osid"] - return identities + records = search("Participant", {"participantId": {"eq": identity["participantId"]}}) + keys = (records[0].get("keys") or []) if records else [] + if not keys: + sys.exit(f"setup: {identity['participantId']} has no published key") + + # Bare base64 now, but an older row may still carry the label, and a + # confusing mismatch error is worse than one tolerant line. + published = keys[0]["key"].removeprefix("base64:") + if published != identity["signingPublic"]: + sys.exit( + f"setup: {identity['participantId']} is registered with a different key.\n" + f" This registry cannot update a record, and its delete is soft and keeps\n" + f" the unique index, so the id cannot be reused. Either restore the\n" + f" matching keys/keys.json, or pick a new BECKN_NETWORK_ID in .env and\n" + f" re-run.") + identity["keyOsid"] = keys[0]["osid"] + return identity # ------------------------------------------------------------------ configs -# The role key is three things at once: the entry in keys/keys.json, the -# __ROLE_* placeholder prefix, and the config filename. They were spelled -# differently once -- role "exp" writing experience.yaml -- and a lookup table -# kept them apart. They are the same word now, so the table is gone. -# -# CHANGING A ROLE KEY IS NOT A RENAME. It makes this script generate a FRESH -# keypair for a participant the registry has already published a public key -# for, which it cannot update and whose delete is soft -- so the id cannot be -# reused either. The adapter then signs with a key nobody can verify, and it -# surfaces much later as an authentication error with no obvious cause. Renaming -# exp -> consumer therefore came with `make destroy` and a new keys.json. +# Each role is a __ROLE_* placeholder prefix, a config filename, and a layer in +# the network entry's role list. ROLES = ("consumer", "network", "provider") -def render(identities): +def render(identity): print("configs:") mausamgram_binding = (f"{env('MAUSAMGRAM_PARTICIPANT_ID')}" f"|{env('MAUSAMGRAM_CAPABILITY')}") @@ -565,8 +543,8 @@ def render(identities): vistaar_binding = (f"{env('VISTAAR_PARTICIPANT_ID')}" f"|{env('VISTAAR_CAPABILITY')}") pocra_binding = f"{env('POCRA_PARTICIPANT_ID')}|{env('POCRA_CAPABILITY')}" + # All three adapters share the network's id and key. for role in ROLES: - identity = identities[role] template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() prefix = role.upper() for placeholder, value in ( @@ -623,13 +601,13 @@ def render(identities): if __name__ == "__main__": load_dotenv() preflight() - identities = load_or_generate_keys() - seed(identities) - identities = key_osids(identities) - KEYS.write_text(json.dumps(identities, indent=2)) - render(identities) + identity = load_or_generate_keys() + seed(identity) + identity = key_osid(identity) + KEYS.write_text(json.dumps(identity, indent=2)) + render(identity) print(f""" -ready. The registry holds seven participants and four capability bindings, and +ready. The registry holds five participants and four capability bindings, and the adapter configs are rendered, so nothing further has to be created by hand. {env('MAUSAMGRAM_PARTICIPANT_ID')}|{env('MAUSAMGRAM_CAPABILITY')} diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 1e7ac6e1..8fe39ae4 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -83,6 +83,7 @@ up() { # 0.0.0.0:80 and :443, deliberately not scoped, because Let's Encrypt # validates HTTP-01 from its own servers. step 4 5 "nginx-proxy-manager -- the public edge (80 and 443, all interfaces)" + mkdir -p .well-known # before Docker can create it root-owned docker compose --profile reverse-proxy up -d nginx-proxy-manager # ClickStack. Heaviest thing here by a wide margin: ClickHouse alone wants @@ -124,12 +125,12 @@ up_registry_tier() { docker compose up -d sunbird-registry-service discovery-service } -# Generates the adapter keypairs, registers the three adapter identities, and +# Generates the network's keypair, registers its one identity, and # renders config/adapters/{provider,network,consumer}.yaml from the .tmpl files # beside them. Safe to re-run: keys come from keys/keys.json once it exists, # and participants already registered are left alone. up_setup() { - step 2 "$1" "bin/setup.py -- keys, seven registry participants, adapter configs" + step 2 "$1" "bin/setup.py -- keys, five registry participants, adapter configs" python3 bin/setup.py } @@ -215,6 +216,7 @@ WARN reverse_proxy() { preflight step 1 1 "nginx-proxy-manager -- publishes 80 and 443 on ALL interfaces" + mkdir -p .well-known # before Docker can create it root-owned docker compose --profile reverse-proxy up -d nginx-proxy-manager cat <<'NEXT' @@ -320,6 +322,54 @@ setup() { python3 bin/setup.py } +# Generate the federation documents into .well-known/ and serve them through NPM +# (npm-custom/server_proxy.conf). Each run publishes the next revision. +host_descriptor() { + preflight + step 1 3 "bin/oanfed.py publish -- the Network Descriptor, JWKS and schema-pack list" + local revision=1 + if [ -f .well-known/openagrinet ]; then + revision=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["revision"] + 1)' \ + .well-known/openagrinet) + fi + # TTL last: it may be empty. + local values + values=$(python3 -c 'import os, sys; sys.path.insert(0, "bin"); import setup +setup.load_dotenv() +print(setup.env("BECKN_NETWORK_ID"), setup.registry_url(), + setup.env("REVERSE_PROXY_HTTPS_PORT", "443"), os.environ.get("FEDERATION_PROJECTION_TTL", ""))') + read -r network_id registry https_port ttl <<<"$values" + python3 bin/oanfed.py publish \ + --network-id "$network_id" --registry-url "$registry" \ + --keys keys/keys.json --revision "$revision" \ + ${ttl:+--projection-ttl "$ttl"} --out . + + step 2 3 "nginx-proxy-manager -- serves .well-known/ on every proxy host" + local was_running + was_running=$(docker inspect -f '{{.State.Running}}' nginx-proxy-manager 2>/dev/null || true) + docker compose --profile reverse-proxy up -d nginx-proxy-manager + if [ "$was_running" = true ]; then + docker exec nginx-proxy-manager nginx -s reload >/dev/null 2>&1 \ + || warn "NPM did not reload; run make restart-edge if the check below fails" + else + info "NPM just started. Change its default login before anything else: make reverse-proxy says how." + fi + + step 3 3 "checking https://$network_id/.well-known/openagrinet through NPM" + # Checks routing only (-k); a peer checks the certificate. + local served + served=$(curl -fsSk --max-time 10 --resolve "$network_id:$https_port:127.0.0.1" \ + "https://$network_id:$https_port/.well-known/openagrinet" 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' 2>/dev/null || true) + if [ "$served" = "$revision" ]; then + info "served at revision $revision -- peers can fetch it now" + else + warn "written, but NPM is not serving it for $network_id yet${served:+ (it serves revision $served)}" + info "NPM needs a proxy host, with a certificate, for $network_id -- README Appendix B." + info "Create it, then re-run make host-descriptor." + fi +} + usage() { cat <<'USAGE' bin/stack.sh @@ -330,6 +380,8 @@ bin/stack.sh down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only + host-descriptor sign the descriptor, JWKS and schema packs into .well-known/ + and serve them through nginx-proxy-manager (public, 80/443) reverse-proxy start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own pull git pull, fixing the npm-custom ownership first @@ -347,6 +399,7 @@ case "${1:-}" in down) down ;; destroy) destroy ;; setup) setup ;; + host-descriptor) host_descriptor ;; reverse-proxy) reverse_proxy ;; observability) observability ;; pull) pull ;; diff --git a/quick-start/bin/test_oanfed.py b/quick-start/bin/test_oanfed.py new file mode 100644 index 00000000..b17b7a0a --- /dev/null +++ b/quick-start/bin/test_oanfed.py @@ -0,0 +1,319 @@ +"""Checks for publishing and verifying the Network Descriptor and JWKS. + + python3 -m unittest quick-start/bin/test_oanfed.py + +No network, no registry: the registry is a stub returning one entry. +""" + +import base64, copy, json, os, pathlib, sys, tempfile, unittest +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +import oanfed as o +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +NETWORK = "network-a.example.org" +# The registry entry's own osid. Nothing names the key by it. +ENTRY_OSID = "1-dd2e40a3-8d4d-427e-bc77-266befb01d00" +# The key's own osid: the network's key id. +KEY_ID = "1-6b9f77c2-139a-4c26-a278-bbb73384ea8a" + + +def raw_public(private): + return o.public_of(private) + + +class StubRegistry: + entry = None + + def __init__(self, *_): + pass + + def participant(self, participant_id): + return self.entry if participant_id == self.entry["participantId"] else None + + def schema_packs(self): + return ["https://openagrinet.github.io/network-specs/schema/MandiPrice/v0.1"] + + +class Base(unittest.TestCase): + def setUp(self): + self.dir = pathlib.Path(tempfile.mkdtemp()) + self.traffic = Ed25519PrivateKey.generate() + StubRegistry.entry = { + "participantId": NETWORK, "name": "Network A", "osid": ENTRY_OSID, + "baseUrl": f"https://{NETWORK}/network", + "keys": [self.registry_key(self.traffic)], + } + self._real = o.ReadOnlyRegistry + o.ReadOnlyRegistry = StubRegistry + self.key_file = self.write_key(self.traffic, self.dir / "keys" / "keys.json") + + def tearDown(self): + o.ReadOnlyRegistry = self._real + + @staticmethod + def write_key(private, path): + """A keys.json the way setup.py writes it: standard base64 seeds.""" + seed = private.private_bytes(o.ser.Encoding.Raw, o.ser.PrivateFormat.Raw, + o.ser.NoEncryption()) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"participantId": NETWORK, + "signingPrivate": base64.b64encode(seed).decode()})) + path.chmod(0o600) + return path + + @staticmethod + def registry_key(private, **extra): + return {"alg": "ed25519", "status": "active", "validFrom": "2026-01-01T00:00:00Z", + "key": base64.b64encode(raw_public(private)).decode(), "osid": KEY_ID, **extra} + + def publish(self, revision=1, **overrides): + args = SimpleNamespace( + network_id=NETWORK, registry_url="http://registry", registry_timeout=1, + operator="", revision=revision, operating_profile="OAN/v0.1,Beckn/v2", + status="Active", projection_ttl=None, out=str(self.dir), + keys=str(self.key_file)) + vars(args).update(overrides) + o.cmd_publish(args) + well_known = self.dir / ".well-known" + return (json.loads((well_known / "openagrinet").read_text()), + json.loads((well_known / "jwks.json").read_text())) + + +class Canonical(unittest.TestCase): + def test_keys_sort_by_utf16_code_units(self): + # RFC 8785 section 3.2.3: U+1F600 sorts BEFORE U+FB33 in UTF-16. + keys = ["€", "\r", "דּ", "1", "\U0001F600", "\u0080", "ö"] + out = o._jcs({k: 0 for k in keys}) + order = [json.loads("{" + p + "}").popitem()[0] for p in out[1:-1].split(",")] + self.assertEqual(order, ["\r", "1", "\u0080", "ö", "€", "\U0001F600", "דּ"]) + + def test_escaping_and_no_whitespace(self): + self.assertEqual(o._jcs({"b": "a/\u001f\né", "a": [1, True, None]}), + '{"a":[1,true,null],"b":"a/\\u001f\\né"}') + + def test_floats_huge_ints_and_lone_surrogates_are_refused(self): + for bad in (1.5, 2 ** 53, "\ud800"): + with self.assertRaises(o.FederationError): + o._jcs({"x": bad}) + + def test_signature_is_excluded(self): + d = {"a": 1} + self.assertEqual(o.canonical(d), o.canonical({**d, "signature": {"value": "x"}})) + + +class Publish(Base): + def test_documents_verify_with_the_one_registered_key(self): + descriptor, keyset = self.publish() + o.verify(descriptor, keyset) + # Named by the key's osid, never the entry's. + self.assertEqual(descriptor["signature"]["keyId"], KEY_ID) + self.assertEqual([k["kid"] for k in keyset["keys"]], [KEY_ID]) + self.assertNotIn(ENTRY_OSID, json.dumps([descriptor, keyset])) + self.assertEqual(keyset["keys"][0], {"kty": "OKP", "crv": "Ed25519", "kid": KEY_ID, + "x": o.b64url(raw_public(self.traffic))}) + self.assertEqual(descriptor["services"]["discovery"], f"https://{NETWORK}/network/discover") + self.assertNotIn("operationalKeys", descriptor) + + def test_schema_packs_is_one_url_serving_the_list(self): + descriptor, _ = self.publish() + self.assertEqual(descriptor["schemaPacks"], f"https://{NETWORK}/.well-known/schema-packs.json") + served = json.loads((self.dir / ".well-known" / "schema-packs.json").read_text()) + self.assertEqual(served, {"schemaPacks": StubRegistry().schema_packs()}) + for field in ("version", "networkId", "name", "operator", "revision", "status", + "issuedAt", "operatingProfile", "schemaPacks", "services"): + self.assertIn(field, descriptor) + self.assertEqual(descriptor["operatingProfile"], ["OAN/v0.1", "Beckn/v2"]) + + def test_a_peer_reads_the_schema_packs_we_publish(self): + self.publish() + fetcher = o.Fetcher() + fetcher._get = lambda _, path: json.loads((self.dir / path.lstrip("/")).read_text()) + self.assertEqual(fetcher.fetch(NETWORK)[2], StubRegistry().schema_packs()) + # A bare list, without the schemaPacks wrapper, is not the agreed shape. + packs_file = self.dir / ".well-known" / "schema-packs.json" + packs_file.write_text(json.dumps(StubRegistry().schema_packs())) + with self.assertRaises(o.FederationError): + fetcher.fetch(NETWORK) + + def test_entry_without_an_osid_still_publishes(self): + # A registry other than Sunbird RC assigns no osid; nothing here needs one. + del StubRegistry.entry["osid"] + descriptor, keyset = self.publish() + o.verify(descriptor, keyset) + + def test_registry_key_without_an_osid_is_refused(self): + del StubRegistry.entry["keys"][0]["osid"] + with self.assertRaisesRegex(o.FederationError, "has no osid"): + self.publish() + self.assertFalse((self.dir / ".well-known" / "openagrinet").exists()) + + def test_key_id_is_taken_from_our_key_during_a_rotation(self): + # Two keys on the entry: only the one we sign with names the documents. + other = Ed25519PrivateKey.generate() + StubRegistry.entry["keys"].insert(0, self.registry_key(other, osid="1-next-key")) + descriptor, keyset = self.publish() + self.assertEqual(descriptor["signature"]["keyId"], KEY_ID) + self.assertEqual([k["kid"] for k in keyset["keys"]], [KEY_ID]) + + def test_readable_key_is_refused(self): + os.chmod(self.key_file, 0o644) + with self.assertRaisesRegex(o.FederationError, "chmod 600"): + self.publish() + + def test_unregistered_key_is_refused(self): + other = self.write_key(Ed25519PrivateKey.generate(), self.dir / "keys" / "other.json") + with self.assertRaisesRegex(o.FederationError, "not an active key registered"): + self.publish(keys=str(other)) + + def test_missing_or_unreadable_keys_file_fails_cleanly(self): + with self.assertRaisesRegex(o.FederationError, "run setup.py first"): + self.publish(keys=str(self.dir / "absent.json")) + bad = self.dir / "keys" / "bad.json" + bad.write_text("{}") + bad.chmod(0o600) + with self.assertRaisesRegex(o.FederationError, "no readable signingPrivate"): + self.publish(keys=str(bad)) + + def test_revision_must_increase(self): + self.publish(3) + for revision in (3, 2): + with self.assertRaisesRegex(o.FederationError, "does not increase"): + self.publish(revision) + + def test_corrupt_published_descriptor_is_not_read_as_revision_zero(self): + self.publish(5) + (self.dir / ".well-known" / "openagrinet").write_text("{") + with self.assertRaises(o.FederationError): + self.publish(1) + + def test_only_usable_registry_keys_can_sign(self): + revoked, expired, upcoming = (Ed25519PrivateKey.generate() for _ in range(3)) + future = (datetime.now(timezone.utc) + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + StubRegistry.entry["keys"] += [ + self.registry_key(revoked, status="revoked"), + self.registry_key(expired, validUntil="2020-01-01T00:00:00Z"), + self.registry_key(upcoming, validFrom=future), + {"alg": "x25519", "status": "active", "key": base64.b64encode(b"e" * 32).decode()}, + ] + self.assertEqual(o.registered_signing_keys(StubRegistry.entry), + [raw_public(self.traffic), raw_public(upcoming)]) + for private in (revoked, expired): + with self.subTest(private=private), \ + self.assertRaisesRegex(o.FederationError, "not an active key registered"): + self.publish(keys=str(self.write_key(private, self.dir / "keys" / "other.json"))) + + def test_malformed_registry_keys_fail_cleanly(self): + for bad in ({"validUntil": "not-a-date"}, {"key": "!!!not base64!!!"}, + {"key": base64.b64encode(b"short").decode()}): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, **bad)] + with self.subTest(bad=bad), self.assertRaises(o.FederationError): + self.publish() + + def test_registry_expiry_without_timezone_is_read_as_utc(self): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, + validUntil="2020-01-01T00:00:00")] + self.assertEqual(o.registered_signing_keys(StubRegistry.entry), []) + + def test_the_same_registry_key_twice_still_signs(self): + StubRegistry.entry["keys"].append(self.registry_key(self.traffic)) + self.assertEqual(o.registered_signing_keys(StubRegistry.entry), [raw_public(self.traffic)]) + _, keyset = self.publish() + self.assertEqual(len(keyset["keys"]), 1) + + def test_registry_expiry_with_any_fraction_length_is_read(self): + for until in ("2099-01-01T00:00:00.5Z", "2099-01-01T00:00:00.123456789Z", + "2099-01-01T00:00:00.1234567+05:30"): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, validUntil=until)] + with self.subTest(until=until): + self.assertEqual(len(o.registered_signing_keys(StubRegistry.entry)), 1) + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, + validUntil="2020-01-01T00:00:00.123456789Z")] + self.assertEqual(o.registered_signing_keys(StubRegistry.entry), []) + + def test_key_path_that_is_a_folder_says_so(self): + folder = self.dir / "keys-folder" + folder.mkdir() + with self.assertRaisesRegex(o.FederationError, "run setup.py first"): + self.publish(keys=str(folder)) + + def test_no_usable_traffic_key_is_refused(self): + StubRegistry.entry["keys"] = [self.registry_key(self.traffic, status="revoked")] + with self.assertRaisesRegex(o.FederationError, "no active Ed25519 key"): + self.publish() + + def test_invalid_descriptor_is_never_written(self): + StubRegistry.entry["baseUrl"] = "http://insecure/network" + with self.assertRaisesRegex(o.FederationError, "services.discovery"): + self.publish() + self.assertFalse((self.dir / ".well-known" / "openagrinet").exists()) + + def test_unregistered_network_is_refused(self): + with self.assertRaisesRegex(o.FederationError, "not in this registry"): + self.publish(network_id="other.example.org") + + +class Verify(Base): + def setUp(self): + super().setUp() + self.descriptor, self.keyset = self.publish() + + def refuses(self, descriptor=None, keyset=None, match=""): + with self.assertRaisesRegex(o.FederationError, match): + o.verify(self.descriptor if descriptor is None else descriptor, + self.keyset if keyset is None else keyset) + + def test_any_change_to_a_signed_field_is_caught(self): + for field, value in (("status", "Suspended"), ("revision", 99), + ("services", {"discovery": "https://evil.example/d"})): + tampered = {**self.descriptor, field: value} + with self.subTest(field=field): + self.refuses(tampered) + + def test_malformed_signatures_fail_cleanly(self): + for signature in (None, {}, {"algorithm": "EdDSA"}, + {**self.descriptor["signature"], "keyId": "nope"}, + {**self.descriptor["signature"], "value": "!!not-base64!!"}, + {**self.descriptor["signature"], "value": ""}): + with self.subTest(signature=signature): + self.refuses({**self.descriptor, "signature": signature}) + + def test_malformed_jwks(self): + dup = copy.deepcopy(self.keyset) + dup["keys"].append(dict(dup["keys"][0])) + short = copy.deepcopy(self.keyset) + short["keys"][0]["x"] = o.b64url(b"short") + for keyset in ({}, {"keys": "x"}, {"keys": [{"kty": "OKP"}]}, dup, short): + with self.subTest(keyset=keyset): + self.refuses(keyset=keyset) + + def test_contract_fields(self): + future = (datetime.now(timezone.utc) + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + for field, value in (("version", "0.1"), ("networkId", "Not A Domain"), + ("name", ""), ("operator", {}), ("revision", 0), + ("revision", True), ("status", "active"), ("issuedAt", future), + ("issuedAt", "2026-02-30T00:00:00Z"), ("operatingProfile", []), + ("schemaPacks", ["https://x/y"]), + ("schemaPacks", f"http://{NETWORK}/.well-known/schema-packs.json"), + ("schemaPacks", "https://evil.example/.well-known/schema-packs.json"), + ("projectionTtl", -1)): + with self.subTest(field=field, value=value): + with self.assertRaises(o.FederationError): + o.validate_descriptor({**self.descriptor, field: value}) + + def test_schema_pack_list(self): + self.assertEqual(o.validate_schema_packs(["https://a/b"]), ["https://a/b"]) + for packs in ("https://a/b", ["http://a/b"], ["https://a/b", "https://a/b"], {}): + with self.subTest(packs=packs), self.assertRaises(o.FederationError): + o.validate_schema_packs(packs) + + def test_key_named_by_a_different_kid_is_refused(self): + renamed = copy.deepcopy(self.keyset) + renamed["keys"][0]["kid"] = "1-00000000-0000-0000-0000-000000000000" + self.refuses(keyset=renamed, match="no published") + + +if __name__ == "__main__": + unittest.main() diff --git a/quick-start/config/registry/schemas/SchemaRegistry.json b/quick-start/config/registry/schemas/SchemaRegistry.json index c398d63c..ca12a3a1 100644 --- a/quick-start/config/registry/schemas/SchemaRegistry.json +++ b/quick-start/config/registry/schemas/SchemaRegistry.json @@ -16,7 +16,7 @@ "name": { "type": "string", "minLength": 1, "maxLength": 200, "pattern": "\\S" }, "version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+$" }, "schemaUrl": { "type": "string", "maxLength": 2000, - "pattern": "^https://raw\\.githubusercontent\\.com/OpenAgriNet/network-specs/[A-Za-z0-9_-]+/schema/[A-Za-z0-9]+/v[0-9]+\\.[0-9]+/[A-Za-z0-9_-]+\\.yaml$" }, + "pattern": "^https://(raw\\.githubusercontent\\.com/OpenAgriNet/network-specs/[A-Za-z0-9_-]+|openagrinet\\.github\\.io/network-specs)/schema/[A-Za-z0-9]+/v[0-9]+\\.[0-9]+/[A-Za-z0-9_-]+\\.yaml$" }, "status": { "$ref": "#/definitions/Status" } } }, diff --git a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf index 35f956d6..8c976506 100644 --- a/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf +++ b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf @@ -28,3 +28,23 @@ # location = /publish { # deny all; # } + +# Federation documents from `make host-descriptor`. Public, so every host may +# serve them; exact matches leave /.well-known/acme-challenge/ alone. +location = /.well-known/openagrinet { + alias /srv/well-known/openagrinet; + default_type application/json; + add_header Cache-Control "no-cache"; +} + +location = /.well-known/jwks.json { + alias /srv/well-known/jwks.json; + default_type application/json; + add_header Cache-Control "no-cache"; +} + +location = /.well-known/schema-packs.json { + alias /srv/well-known/schema-packs.json; + default_type application/json; + add_header Cache-Control "no-cache"; +} diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index 9568abc3..9aa3579b 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -199,6 +199,9 @@ services: image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest container_name: sunbird-registry-keycloak restart: unless-stopped + # An interrupted boot leaves keycloak-add-user.json behind, and every restart + # then fails on the duplicate admin. Clear it first. + entrypoint: ["/bin/sh", "-c", "rm -f /opt/jboss/keycloak/standalone/configuration/keycloak-add-user.json && exec /opt/jboss/tools/docker-entrypoint.sh \"$$@\"", "--"] logging: *logging networks: [oan-internal] volumes: @@ -249,7 +252,7 @@ services: # silently depend on it. # # What this costs: nothing outside the VM can write a registry row, which - # is why bin/setup.py seeds all seven participants and all four bindings from + # is why bin/setup.py seeds all five participants and all four bindings from # inside. It is also why the Postman collection has no registry request -- # there is no way for one to work. # @@ -921,6 +924,9 @@ services: # ownership of the files, which is cosmetic and not tracked by git. - ./config/reverse-proxy/npm-custom:/data/nginx/custom + # Federation documents from `make host-descriptor`, served by server_proxy.conf. + - ./.well-known:/srv/well-known:ro + depends_on: # Ordering only; the adapter image has no healthcheck. #