From 8e73635de7a521a6bd68ee8f5d0d490b3723c220 Mon Sep 17 00:00:00 2001 From: coodos Date: Tue, 15 Sep 2026 03:49:48 +0800 Subject: [PATCH 1/2] fix(awareness): make delivery durable and restart-safe --- .env.example | 18 +- .../awareness/awareness-outbox-dispatcher.ts | 288 +++++++++ .../evault-core/src/core/db/db.service.ts | 570 ++++++++++++------ .../add-awareness-outbox-indexes.ts | 23 + .../src/core/protocol/graphql-server.ts | 349 +++-------- infrastructure/evault-core/src/index.ts | 142 ++++- .../src/services/BindingDocumentService.ts | 158 +++-- pnpm-lock.yaml | 6 + services/awareness-service/api/package.json | 5 +- services/awareness-service/api/src/config.ts | 32 +- .../api/src/controllers/AdminController.ts | 16 +- .../api/src/controllers/ConsumerController.ts | 6 +- .../api/src/controllers/IngestController.ts | 7 +- .../api/src/controllers/QueryController.ts | 51 +- .../api/src/controllers/SystemController.ts | 111 ++++ .../api/src/database/data-source.ts | 7 + .../src/database/entities/AwarenessEvent.ts | 47 ++ .../api/src/database/entities/DeadLetter.ts | 1 + .../api/src/database/entities/Delivery.ts | 44 +- .../api/src/database/entities/Packet.ts | 4 +- .../src/database/entities/WorkerHeartbeat.ts | 20 + .../database/migrations/1715200000000-Init.ts | 2 +- .../1780000000000-AddDeliveryContentHash.ts | 47 -- .../1780404367748-AddDeliveryContentHash.ts | 11 +- .../1780404367749-AddDeliveryPayload.ts | 6 +- ...0-AddDeliveriesSubscriptionCreatedIndex.ts | 2 +- .../1789430400000-DurableAwarenessEvents.ts | 178 ++++++ services/awareness-service/api/src/index.ts | 31 +- services/awareness-service/api/src/openapi.ts | 99 ++- .../api/src/scripts/backfill-neo4j.ts | 68 ++- .../api/src/services/DeliveryEngine.ts | 425 ++++++++----- .../api/src/services/IngestService.ts | 243 +++++--- .../api/src/services/SeedService.ts | 94 ++- .../api/src/services/SubscriptionMatcher.ts | 9 +- services/awareness-service/api/src/types.ts | 7 +- .../api/src/utils/contentHash.ts | 5 +- 36 files changed, 2252 insertions(+), 880 deletions(-) create mode 100644 infrastructure/evault-core/src/core/awareness/awareness-outbox-dispatcher.ts create mode 100644 infrastructure/evault-core/src/core/db/migrations/add-awareness-outbox-indexes.ts create mode 100644 services/awareness-service/api/src/controllers/SystemController.ts create mode 100644 services/awareness-service/api/src/database/entities/AwarenessEvent.ts create mode 100644 services/awareness-service/api/src/database/entities/WorkerHeartbeat.ts delete mode 100644 services/awareness-service/api/src/database/migrations/1780000000000-AddDeliveryContentHash.ts create mode 100644 services/awareness-service/api/src/database/migrations/1789430400000-DurableAwarenessEvents.ts diff --git a/.env.example b/.env.example index 35429aec9..3fd2827cc 100644 --- a/.env.example +++ b/.env.example @@ -154,9 +154,23 @@ DO_SPACES_BUCKET="your-spaces-bucket" DO_SPACES_CDN_URL="" # Secret used to sign AaaS portal session JWTs AAAS_JWT_SECRET="replace-with-a-strong-secret" -# Webhook delivery tuning -AWARENESS_MAX_ATTEMPTS=3 +# Webhook delivery worker runs in the same AaaS process as the HTTP API. AWARENESS_DELIVERY_POLL_MS=2000 +AWARENESS_DELIVERY_LEASE_MS=30000 +AWARENESS_DELIVERY_BATCH_TIMEOUT_MS=25000 +# Retry downstream webhooks for 24 hours before dead-lettering. +AWARENESS_DELIVERY_RETRY_WINDOW_MS=86400000 +AWARENESS_WORKER_HEARTBEAT_MS=10000 +AWARENESS_WORKER_STALE_MS=30000 +# Bound every Postgres operation so one poisoned connection cannot wedge AaaS. +AWARENESS_DB_STATEMENT_TIMEOUT_MS=10000 +AWARENESS_DB_QUERY_TIMEOUT_MS=12000 +AWARENESS_DB_LOCK_TIMEOUT_MS=5000 +# Durable eVault -> AaaS outbox tuning. Ingest retries never expire. +AWARENESS_OUTBOX_POLL_MS=1000 +AWARENESS_OUTBOX_LEASE_MS=30000 +AWARENESS_OUTBOX_DB_TIMEOUT_MS=10000 +AWARENESS_OUTBOX_RETENTION_MS=604800000 # The one-time Neo4j backfill reuses the standard NEO4J_URI / NEO4J_USER / # NEO4J_PASSWORD vars at the top of this file - it reads evault-core's graph # directly, so there are no AaaS-specific Neo4j vars. diff --git a/infrastructure/evault-core/src/core/awareness/awareness-outbox-dispatcher.ts b/infrastructure/evault-core/src/core/awareness/awareness-outbox-dispatcher.ts new file mode 100644 index 000000000..3c64e0c1d --- /dev/null +++ b/infrastructure/evault-core/src/core/awareness/awareness-outbox-dispatcher.ts @@ -0,0 +1,288 @@ +import axios from "axios"; +import { randomUUID } from "node:crypto"; +import neo4j, { type Driver } from "neo4j-driver"; + +interface ClaimedEvent { + eventId: string; + packetId: string; + schemaId: string; + w3id: string; + evaultPublicKey: string | null; + dataJson: string; + operation: "create" | "update" | "delete"; + requestingPlatform: string | null; + occurredAt: string; + streamVersion: number; + attempts: number; + leaseToken: string; +} + +const RETRY_SCHEDULE = [ + 1_000, 5_000, 30_000, 60_000, 120_000, 300_000, 900_000, 3_600_000, + 21_600_000, 86_400_000, +]; + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function numberValue(value: any): number { + return typeof value?.toNumber === "function" + ? value.toNumber() + : Number(value); +} + +function positiveInteger(raw: string | undefined, fallback: number): number { + const value = Number(raw); + return Number.isSafeInteger(value) && value > 0 ? value : fallback; +} + +/** + * Drains Neo4j awareness outbox rows until AaaS durably acknowledges them. + * Unlike the old resolver-level POST, failures survive process restarts. + */ +export class AwarenessOutboxDispatcher { + private stopping = false; + private loop?: Promise; + private readonly workerId = + `${process.env.HOSTNAME ?? "evault"}-${process.pid}`; + private readonly pollMs = positiveInteger( + process.env.AWARENESS_OUTBOX_POLL_MS, + 1_000, + ); + private readonly leaseMs = positiveInteger( + process.env.AWARENESS_OUTBOX_LEASE_MS, + 30_000, + ); + private readonly dbTimeoutMs = positiveInteger( + process.env.AWARENESS_OUTBOX_DB_TIMEOUT_MS, + 10_000, + ); + private lastCycleAt: Date | null = null; + private lastError: string | null = null; + + constructor(private readonly driver: Driver) {} + + start(): void { + if (this.loop) return; + this.stopping = false; + this.loop = this.run(); + console.log(`[awareness-outbox] dispatcher ${this.workerId} started`); + } + + async stop(): Promise { + this.stopping = true; + await this.loop; + this.loop = undefined; + } + + health(): { + configured: boolean; + running: boolean; + workerId: string; + lastCycleAt: Date | null; + lastError: string | null; + } { + return { + configured: Boolean(process.env.AWARENESS_SERVICE_URL), + running: Boolean(this.loop) && !this.stopping, + workerId: this.workerId, + lastCycleAt: this.lastCycleAt, + lastError: this.lastError, + }; + } + + private async run(): Promise { + while (!this.stopping) { + try { + const events = await this.claim(50); + const results = await Promise.allSettled( + events.map((event) => this.deliver(event)), + ); + const rejected = results.filter( + (result) => result.status === "rejected", + ); + if (rejected.length > 0) { + const first = rejected[0] as PromiseRejectedResult; + throw new Error( + `${rejected.length} outbox completion(s) failed; leases will be reclaimed: ${first.reason instanceof Error ? first.reason.message : String(first.reason)}`, + ); + } + await this.cleanupAcknowledged(); + this.lastError = null; + } catch (error) { + this.lastError = + error instanceof Error ? error.message : String(error); + console.error( + "[awareness-outbox] dispatch cycle failed:", + error, + ); + } finally { + this.lastCycleAt = new Date(); + } + if (!this.stopping) await delay(this.pollMs); + } + } + + private async claim(limit: number): Promise { + if (!process.env.AWARENESS_SERVICE_URL) return []; + const now = Date.now(); + const leaseToken = randomUUID(); + const session = this.driver.session(); + try { + const result = await session.executeWrite( + (tx) => + tx.run( + `MATCH (a:AwarenessOutbox) + WHERE (a.status IN ['pending', 'failed'] AND a.nextAttemptAt <= $now) + OR (a.status = 'delivering' AND a.leaseExpiresAt <= $now) + WITH a + WHERE NOT EXISTS { + MATCH (earlier:AwarenessOutbox) + WHERE earlier.packetId = a.packetId + AND earlier.status IN ['pending', 'failed', 'delivering'] + AND earlier.streamVersion < a.streamVersion + } + WITH a ORDER BY a.nextAttemptAt, a.createdAt LIMIT $limit + SET a.status = 'delivering', + a.leaseOwner = $workerId, + a.leaseToken = $leaseToken, + a.leaseExpiresAt = $leaseExpiresAt + RETURN a`, + { + now, + limit: neo4j.int(limit), + workerId: this.workerId, + leaseToken, + leaseExpiresAt: now + this.leaseMs, + }, + ), + { timeout: this.dbTimeoutMs }, + ); + return result.records.map((record) => { + const p = record.get("a").properties; + return { + eventId: p.eventId, + packetId: p.packetId, + schemaId: p.schemaId, + w3id: p.w3id, + evaultPublicKey: p.evaultPublicKey ?? null, + dataJson: p.dataJson, + operation: p.operation, + requestingPlatform: p.requestingPlatform ?? null, + occurredAt: p.occurredAt, + streamVersion: numberValue(p.streamVersion), + attempts: numberValue(p.attempts), + leaseToken, + }; + }); + } finally { + await session.close(); + } + } + + private async deliver(event: ClaimedEvent): Promise { + try { + const ingestUrl = new URL( + "/ingest", + process.env.AWARENESS_SERVICE_URL!, + ).toString(); + await axios.post( + ingestUrl, + { + eventId: event.eventId, + id: event.packetId, + w3id: event.w3id, + evaultPublicKey: event.evaultPublicKey, + data: JSON.parse(event.dataJson), + schemaId: event.schemaId, + operation: event.operation, + requestingPlatform: event.requestingPlatform, + occurredAt: event.occurredAt, + streamVersion: event.streamVersion, + }, + { + headers: { + "Content-Type": "application/json", + "x-ingest-secret": + process.env.AWARENESS_INGEST_SECRET ?? "", + }, + timeout: 5000, + }, + ); + } catch (error: any) { + await this.finish( + event, + false, + `${error?.response?.status ?? error?.code ?? "error"}: ${error?.message ?? String(error)}`, + ); + return; + } + await this.finish(event, true, null); + } + + private async finish( + event: ClaimedEvent, + delivered: boolean, + error: string | null, + ): Promise { + const attempts = event.attempts + 1; + const base = + RETRY_SCHEDULE[Math.min(attempts - 1, RETRY_SCHEDULE.length - 1)]; + const jitter = base * (Math.random() * 0.2 - 0.1); + const session = this.driver.session(); + try { + await session.executeWrite( + (tx) => + tx.run( + `MATCH (a:AwarenessOutbox { eventId: $eventId, leaseToken: $leaseToken }) + SET a.status = $status, + a.attempts = $attempts, + a.lastError = $error, + a.nextAttemptAt = $nextAttemptAt, + a.acknowledgedAt = $acknowledgedAt, + a.leaseOwner = null, + a.leaseToken = null, + a.leaseExpiresAt = null`, + { + eventId: event.eventId, + leaseToken: event.leaseToken, + status: delivered ? "delivered" : "failed", + attempts, + error, + nextAttemptAt: delivered + ? Date.now() + : Date.now() + base + jitter, + acknowledgedAt: delivered ? Date.now() : null, + }, + ), + { timeout: this.dbTimeoutMs }, + ); + } finally { + await session.close(); + } + } + + private async cleanupAcknowledged(): Promise { + const retentionMs = positiveInteger( + process.env.AWARENESS_OUTBOX_RETENTION_MS, + 7 * 24 * 60 * 60 * 1000, + ); + const session = this.driver.session(); + try { + await session.executeWrite( + (tx) => + tx.run( + `MATCH (a:AwarenessOutbox { status: 'delivered' }) + WHERE a.acknowledgedAt < $before + WITH a LIMIT 500 + DELETE a`, + { before: Date.now() - retentionMs }, + ), + { timeout: this.dbTimeoutMs }, + ); + } finally { + await session.close(); + } + } +} diff --git a/infrastructure/evault-core/src/core/db/db.service.ts b/infrastructure/evault-core/src/core/db/db.service.ts index bead136f0..ceadca000 100644 --- a/infrastructure/evault-core/src/core/db/db.service.ts +++ b/infrastructure/evault-core/src/core/db/db.service.ts @@ -1,4 +1,5 @@ import neo4j, { type Driver } from "neo4j-driver"; +import { randomUUID } from "node:crypto"; import { W3IDBuilder } from "w3id"; import { timed } from "../utils/timing"; import { parseStoredAclBlock, serializeAclBlock } from "../acl"; @@ -21,6 +22,56 @@ import type { StoreMetaEnvelopeResult, } from "./types"; +export interface AwarenessWriteContext { + evaultPublicKey: string | null; + requestingPlatform?: string | null; + skipAwareness?: boolean; +} + +function awarenessOutboxParams( + packetId: string, + schemaId: string, + eName: string, + data: unknown, + operation: "create" | "update" | "delete", + context: AwarenessWriteContext, +): Record { + return { + awarenessEventId: randomUUID(), + awarenessPacketId: packetId, + awarenessSchemaId: schemaId, + awarenessW3id: eName, + awarenessEvaultPublicKey: context.evaultPublicKey, + awarenessDataJson: JSON.stringify(data ?? null), + awarenessOperation: operation, + awarenessRequestingPlatform: context.requestingPlatform ?? null, + awarenessOccurredAt: new Date().toISOString(), + awarenessNow: Date.now(), + }; +} + +const CREATE_AWARENESS_OUTBOX = ` + WITH m + MERGE (s:AwarenessStream { packetId: $awarenessPacketId }) + SET s.version = coalesce(s.version, 0) + 1 + CREATE (a:AwarenessOutbox { + eventId: $awarenessEventId, + packetId: $awarenessPacketId, + schemaId: $awarenessSchemaId, + w3id: $awarenessW3id, + evaultPublicKey: $awarenessEvaultPublicKey, + dataJson: $awarenessDataJson, + operation: $awarenessOperation, + requestingPlatform: $awarenessRequestingPlatform, + occurredAt: $awarenessOccurredAt, + streamVersion: s.version, + status: 'pending', + attempts: 0, + nextAttemptAt: $awarenessNow, + createdAt: $awarenessNow + }) +`; + /** * Service for managing meta-envelopes and their associated envelopes in Neo4j. * Provides functionality for storing, retrieving, searching, and updating data @@ -78,40 +129,41 @@ export class DbService { meta: Omit, "id">, acl: string[], eName: string, + awareness?: AwarenessWriteContext, ): Promise> { - return timed("db.storeMetaEnvelope", async () => { - if (!eName) { - throw new Error("eName is required for storing meta-envelopes"); - } + return timed("db.storeMetaEnvelope", async () => { + if (!eName) { + throw new Error("eName is required for storing meta-envelopes"); + } - const w3id = await timed("db.storeMetaEnvelope.buildMetaId", () => - new W3IDBuilder().build(), - ); + const w3id = await timed("db.storeMetaEnvelope.buildMetaId", () => + new W3IDBuilder().build(), + ); - const cypher: string[] = [ - `CREATE (m:MetaEnvelope { id: $metaId, ontology: $ontology, acl: $acl, aclBlock: $aclBlock, eName: $eName })`, - ]; + const cypher: string[] = [ + `CREATE (m:MetaEnvelope { id: $metaId, ontology: $ontology, acl: $acl, aclBlock: $aclBlock, eName: $eName })`, + ]; - const envelopeParams: Record = { - metaId: w3id.id, - ontology: meta.ontology, - acl: acl, - aclBlock: serializeAclBlock(meta._acl), - eName: eName, - }; + const envelopeParams: Record = { + metaId: w3id.id, + ontology: meta.ontology, + acl: acl, + aclBlock: serializeAclBlock(meta._acl), + eName: eName, + }; - const createdEnvelopes: Envelope[] = []; - let counter = 0; + const createdEnvelopes: Envelope[] = []; + let counter = 0; - for (const [key, value] of Object.entries(meta.payload)) { - const envW3id = await new W3IDBuilder().build(); - const envelopeId = envW3id.id; - const alias = `e${counter}`; + for (const [key, value] of Object.entries(meta.payload)) { + const envW3id = await new W3IDBuilder().build(); + const envelopeId = envW3id.id; + const alias = `e${counter}`; - const { value: storedValue, type: valueType } = - serializeValue(value); + const { value: storedValue, type: valueType } = + serializeValue(value); - cypher.push(` + cypher.push(` CREATE (${alias}:Envelope { id: $${alias}_id, ontology: $${alias}_ontology, @@ -122,35 +174,50 @@ export class DbService { MERGE (m)-[:LINKS_TO]->(${alias}) `); - envelopeParams[`${alias}_id`] = envelopeId; - envelopeParams[`${alias}_ontology`] = key; - envelopeParams[`${alias}_value`] = storedValue; - envelopeParams[`${alias}_type`] = valueType; + envelopeParams[`${alias}_id`] = envelopeId; + envelopeParams[`${alias}_ontology`] = key; + envelopeParams[`${alias}_value`] = storedValue; + envelopeParams[`${alias}_type`] = valueType; - createdEnvelopes.push({ - id: envelopeId, - ontology: key, - value: value as T[keyof T], - valueType, - }); + createdEnvelopes.push({ + id: envelopeId, + ontology: key, + value: value as T[keyof T], + valueType, + }); - counter++; - } + counter++; + } - await timed("db.storeMetaEnvelope.runQuery", () => - this.runQueryInternal(cypher.join("\n"), envelopeParams), - ); + if (awareness && !awareness.skipAwareness) { + cypher.push(CREATE_AWARENESS_OUTBOX); + Object.assign( + envelopeParams, + awarenessOutboxParams( + w3id.id, + meta.ontology, + eName, + meta.payload, + "create", + awareness, + ), + ); + } - return { - metaEnvelope: { - id: w3id.id, - ontology: meta.ontology, - acl: acl, - _acl: meta._acl, - }, - envelopes: createdEnvelopes, - }; - }); + await timed("db.storeMetaEnvelope.runQuery", () => + this.runQueryInternal(cypher.join("\n"), envelopeParams), + ); + + return { + metaEnvelope: { + id: w3id.id, + ontology: meta.ontology, + acl: acl, + _acl: meta._acl, + }, + envelopes: createdEnvelopes, + }; + }); } /** @@ -169,6 +236,7 @@ export class DbService { acl: string[], eName: string, id?: string, + awareness?: AwarenessWriteContext, ): Promise> { if (!eName) { throw new Error("eName is required for storing meta-envelopes"); @@ -223,6 +291,21 @@ export class DbService { counter++; } + if (awareness && !awareness.skipAwareness) { + cypher.push(CREATE_AWARENESS_OUTBOX); + Object.assign( + envelopeParams, + awarenessOutboxParams( + metaId, + meta.ontology, + eName, + meta.payload, + "create", + awareness, + ), + ); + } + await this.runQueryInternal(cypher.join("\n"), envelopeParams); return { @@ -541,17 +624,53 @@ export class DbService { * @param id - The ID of the meta-envelope to delete * @param eName - The eName identifier for multi-tenant isolation */ - async deleteMetaEnvelope(id: string, eName: string): Promise { + async deleteMetaEnvelope( + id: string, + eName: string, + awareness?: AwarenessWriteContext, + ): Promise { if (!eName) { throw new Error("eName is required for deleting meta-envelopes"); } + const params: Record = { id, eName }; + const outbox = awareness && !awareness.skipAwareness; + if (outbox) { + Object.assign( + params, + awarenessOutboxParams(id, "", eName, null, "delete", awareness), + ); + } await this.runQueryInternal( ` - MATCH (m:MetaEnvelope { id: $id, eName: $eName })-[:LINKS_TO]->(e:Envelope) - DETACH DELETE m, e + MATCH (m:MetaEnvelope { id: $id, eName: $eName }) + OPTIONAL MATCH (m)-[:LINKS_TO]->(e:Envelope) + WITH m, collect(e) AS envelopes + ${ + outbox + ? `MERGE (s:AwarenessStream { packetId: $awarenessPacketId }) + SET s.version = coalesce(s.version, 0) + 1 + CREATE (a:AwarenessOutbox { + eventId: $awarenessEventId, + packetId: $awarenessPacketId, + schemaId: m.ontology, + w3id: $awarenessW3id, + evaultPublicKey: $awarenessEvaultPublicKey, + dataJson: $awarenessDataJson, + operation: $awarenessOperation, + requestingPlatform: $awarenessRequestingPlatform, + occurredAt: $awarenessOccurredAt, + streamVersion: s.version, + status: 'pending', attempts: 0, + nextAttemptAt: $awarenessNow, createdAt: $awarenessNow + }) + WITH m, envelopes` + : "" + } + FOREACH (node IN envelopes | DETACH DELETE node) + DETACH DELETE m `, - { id, eName }, + params, ); } @@ -565,6 +684,7 @@ export class DbService { envelopeId: string, newValue: T, eName: string, + awareness?: AwarenessWriteContext, ): Promise { if (!eName) { throw new Error("eName is required for updating envelope values"); @@ -573,14 +693,44 @@ export class DbService { const { value: storedValue, type: valueType } = serializeValue(newValue); - // First verify the envelope belongs to a meta-envelope with the correct eName - await this.runQueryInternal( - ` - MATCH (m:MetaEnvelope { eName: $eName })-[:LINKS_TO]->(e:Envelope { id: $envelopeId }) - SET e.value = $newValue, e.valueType = $valueType - `, - { envelopeId, newValue: storedValue, valueType, eName }, - ); + const session = this.driver.session(); + try { + await session.executeWrite(async (tx) => { + const result = await tx.run( + ` + MATCH (m:MetaEnvelope { eName: $eName })-[:LINKS_TO]->(e:Envelope { id: $envelopeId }) + SET e.value = $newValue, e.valueType = $valueType + WITH m + MATCH (m)-[:LINKS_TO]->(allEnvelope:Envelope) + RETURN m.id AS id, m.ontology AS ontology, collect(allEnvelope) AS envelopes + `, + { envelopeId, newValue: storedValue, valueType, eName }, + ); + const record = result.records[0]; + if (!record || !awareness || awareness.skipAwareness) return; + const payload: Record = {}; + for (const node of record.get("envelopes")) { + payload[node.properties.ontology] = deserializeValue( + node.properties.value, + node.properties.valueType, + ); + } + await tx.run( + `MATCH (m:MetaEnvelope { id: $awarenessPacketId, eName: $awarenessW3id }) + ${CREATE_AWARENESS_OUTBOX}`, + awarenessOutboxParams( + record.get("id"), + record.get("ontology"), + eName, + payload, + "update", + awareness, + ), + ); + }); + } finally { + await session.close(); + } } /** @@ -598,22 +748,25 @@ export class DbService { meta: Omit, "id">, acl: string[], eName: string, + awareness?: AwarenessWriteContext, ): Promise> { - return timed("db.updateMetaEnvelopeById", async () => { - if (!eName) { - throw new Error("eName is required for updating meta-envelopes"); - } + return timed("db.updateMetaEnvelopeById", async () => { + if (!eName) { + throw new Error( + "eName is required for updating meta-envelopes", + ); + } - // The whole read-modify-write cycle runs inside a single Neo4j write - // transaction. The opening MERGE+SET acquires a write lock on the - // MetaEnvelope node, so concurrent updates to the same id serialize - // here — without this, request B's "delete stale envelopes" step - // could clobber fields that request A just wrote. - const session = this.driver.session(); - try { - return await session.executeWrite(async (tx) => { - const findResult = await tx.run( - ` + // The whole read-modify-write cycle runs inside a single Neo4j write + // transaction. The opening MERGE+SET acquires a write lock on the + // MetaEnvelope node, so concurrent updates to the same id serialize + // here — without this, request B's "delete stale envelopes" step + // could clobber fields that request A just wrote. + const session = this.driver.session(); + try { + return await session.executeWrite(async (tx) => { + const findResult = await tx.run( + ` MERGE (m:MetaEnvelope { id: $id, eName: $eName }) ON CREATE SET m.ontology = $ontology, m.acl = $acl, m.aclBlock = $aclBlock ON MATCH SET m.ontology = $ontology, m.acl = $acl, m.aclBlock = coalesce($aclBlock, m.aclBlock) @@ -621,149 +774,164 @@ export class DbService { OPTIONAL MATCH (m)-[:LINKS_TO]->(e:Envelope) RETURN collect(e) AS envelopes `, - { - id, - eName, - ontology: meta.ontology, - acl, - aclBlock: serializeAclBlock(meta._acl), - }, - ); - - const envelopeNodes: any[] = ( - findResult.records[0]?.get("envelopes") ?? [] - ).filter((n: any) => n !== null && n !== undefined); + { + id, + eName, + ontology: meta.ontology, + acl, + aclBlock: serializeAclBlock(meta._acl), + }, + ); - let workingEnvelopes: Envelope[] = - envelopeNodes.map((node: any) => ({ - id: node.properties.id, - ontology: node.properties.ontology, - value: deserializeValue( - node.properties.value, - node.properties.valueType, - ) as T[keyof T], - valueType: node.properties.valueType, - })); - - // Deduplicate envelopes — if multiple Envelope nodes share the - // same ontology, keep the first and delete the rest. - const seen = new Map(); - const dupsToDelete: string[] = []; - for (const env of workingEnvelopes) { - if (seen.has(env.ontology)) { - dupsToDelete.push(env.id); - } else { - seen.set(env.ontology, env.id); + const envelopeNodes: any[] = ( + findResult.records[0]?.get("envelopes") ?? [] + ).filter((n: any) => n !== null && n !== undefined); + + let workingEnvelopes: Envelope[] = + envelopeNodes.map((node: any) => ({ + id: node.properties.id, + ontology: node.properties.ontology, + value: deserializeValue( + node.properties.value, + node.properties.valueType, + ) as T[keyof T], + valueType: node.properties.valueType, + })); + + // Deduplicate envelopes — if multiple Envelope nodes share the + // same ontology, keep the first and delete the rest. + const seen = new Map(); + const dupsToDelete: string[] = []; + for (const env of workingEnvelopes) { + if (seen.has(env.ontology)) { + dupsToDelete.push(env.id); + } else { + seen.set(env.ontology, env.id); + } + } + if (dupsToDelete.length > 0) { + console.warn( + `[eVault] Cleaning ${dupsToDelete.length} duplicate envelope(s) for MetaEnvelope ${id}`, + ); + await tx.run( + `MATCH (e:Envelope) WHERE e.id IN $ids DETACH DELETE e`, + { ids: dupsToDelete }, + ); + workingEnvelopes = workingEnvelopes.filter( + (e) => !dupsToDelete.includes(e.id), + ); } - } - if (dupsToDelete.length > 0) { - console.warn( - `[eVault] Cleaning ${dupsToDelete.length} duplicate envelope(s) for MetaEnvelope ${id}`, - ); - await tx.run( - `MATCH (e:Envelope) WHERE e.id IN $ids DETACH DELETE e`, - { ids: dupsToDelete }, - ); - workingEnvelopes = workingEnvelopes.filter( - (e) => !dupsToDelete.includes(e.id), - ); - } - const createdEnvelopes: Envelope[] = []; + const createdEnvelopes: Envelope[] = []; - for (const [key, value] of Object.entries(meta.payload)) { - const { value: storedValue, type: valueType } = - serializeValue(value); - const existingEnvelope = workingEnvelopes.find( - (e) => e.ontology === key, - ); + for (const [key, value] of Object.entries(meta.payload)) { + const { value: storedValue, type: valueType } = + serializeValue(value); + const existingEnvelope = workingEnvelopes.find( + (e) => e.ontology === key, + ); - if (existingEnvelope) { - await tx.run( - ` + if (existingEnvelope) { + await tx.run( + ` MATCH (e:Envelope { id: $envelopeId }) SET e.value = $newValue, e.valueType = $valueType `, - { - envelopeId: existingEnvelope.id, - newValue: storedValue, + { + envelopeId: existingEnvelope.id, + newValue: storedValue, + valueType, + }, + ); + createdEnvelopes.push({ + id: existingEnvelope.id, + ontology: key, + value: value as T[keyof T], valueType, - }, - ); - createdEnvelopes.push({ - id: existingEnvelope.id, - ontology: key, - value: value as T[keyof T], - valueType, - }); - } else { - const envW3id = await new W3IDBuilder().build(); - const envelopeId = envW3id.id; - await tx.run( - ` + }); + } else { + const envW3id = await new W3IDBuilder().build(); + const envelopeId = envW3id.id; + await tx.run( + ` MATCH (m:MetaEnvelope { id: $metaId, eName: $eName }) MERGE (m)-[:LINKS_TO]->(e:Envelope { ontology: $ontology }) ON CREATE SET e.id = $envelopeId, e.value = $newValue, e.valueType = $valueType ON MATCH SET e.value = $newValue, e.valueType = $valueType `, - { - metaId: id, - eName, - envelopeId, + { + metaId: id, + eName, + envelopeId, + ontology: key, + newValue: storedValue, + valueType, + }, + ); + createdEnvelopes.push({ + id: envelopeId, ontology: key, - newValue: storedValue, + value: value as T[keyof T], valueType, - }, - ); - createdEnvelopes.push({ - id: envelopeId, - ontology: key, - value: value as T[keyof T], - valueType, - }); + }); + } } - } - // PATCH semantics: fields absent from the new payload are - // left alone. Callers (notably web3-adapter) project partial - // platform updates through toGlobal — if the platform only - // touched one column, only one ontology reaches us, and - // deleting "stale" envelopes here would clobber every other - // field on the meta-envelope (e.g. wiping participantIds when - // a read-receipt update arrives). - - // Build the full post-write state by merging the pre-write - // envelope set with everything we just wrote. Used by - // resolvers to fan out webhooks containing the complete - // merged state — receivers overwrite their local row with - // whatever the webhook carries, so a partial diff would - // make them lose every untouched field. - const mergedPayload: Record = {}; - for (const env of workingEnvelopes) { - mergedPayload[env.ontology] = env.value; - } - for (const env of createdEnvelopes) { - mergedPayload[env.ontology] = env.value; - } + // PATCH semantics: fields absent from the new payload are + // left alone. Callers (notably web3-adapter) project partial + // platform updates through toGlobal — if the platform only + // touched one column, only one ontology reaches us, and + // deleting "stale" envelopes here would clobber every other + // field on the meta-envelope (e.g. wiping participantIds when + // a read-receipt update arrives). + + // Build the full post-write state by merging the pre-write + // envelope set with everything we just wrote. Used by + // resolvers to fan out webhooks containing the complete + // merged state — receivers overwrite their local row with + // whatever the webhook carries, so a partial diff would + // make them lose every untouched field. + const mergedPayload: Record = {}; + for (const env of workingEnvelopes) { + mergedPayload[env.ontology] = env.value; + } + for (const env of createdEnvelopes) { + mergedPayload[env.ontology] = env.value; + } - return { - metaEnvelope: { - id, - ontology: meta.ontology, - acl, - _acl: meta._acl, - }, - envelopes: createdEnvelopes, - mergedPayload, - }; - }); - } catch (error) { - console.error("Error in updateMetaEnvelopeById:", error); - throw error; - } finally { - await session.close(); - } - }); + if (awareness && !awareness.skipAwareness) { + await tx.run( + `MATCH (m:MetaEnvelope { id: $awarenessPacketId, eName: $awarenessW3id }) + ${CREATE_AWARENESS_OUTBOX}`, + awarenessOutboxParams( + id, + meta.ontology, + eName, + mergedPayload, + "update", + awareness, + ), + ); + } + + return { + metaEnvelope: { + id, + ontology: meta.ontology, + acl, + _acl: meta._acl, + }, + envelopes: createdEnvelopes, + mergedPayload, + }; + }); + } catch (error) { + console.error("Error in updateMetaEnvelopeById:", error); + throw error; + } finally { + await session.close(); + } + }); } /** diff --git a/infrastructure/evault-core/src/core/db/migrations/add-awareness-outbox-indexes.ts b/infrastructure/evault-core/src/core/db/migrations/add-awareness-outbox-indexes.ts new file mode 100644 index 000000000..447bd574d --- /dev/null +++ b/infrastructure/evault-core/src/core/db/migrations/add-awareness-outbox-indexes.ts @@ -0,0 +1,23 @@ +import type { Driver } from "neo4j-driver"; + +export async function createAwarenessOutboxIndexes( + driver: Driver, +): Promise { + const session = driver.session(); + try { + await session.run( + "CREATE CONSTRAINT awareness_stream_packet_id IF NOT EXISTS FOR (s:AwarenessStream) REQUIRE s.packetId IS UNIQUE", + ); + await session.run( + "CREATE CONSTRAINT awareness_outbox_event_id IF NOT EXISTS FOR (a:AwarenessOutbox) REQUIRE a.eventId IS UNIQUE", + ); + await session.run( + "CREATE INDEX awareness_outbox_due IF NOT EXISTS FOR (a:AwarenessOutbox) ON (a.status, a.nextAttemptAt)", + ); + await session.run( + "CREATE INDEX awareness_outbox_stream IF NOT EXISTS FOR (a:AwarenessOutbox) ON (a.packetId, a.status, a.streamVersion)", + ); + } finally { + await session.close(); + } +} diff --git a/infrastructure/evault-core/src/core/protocol/graphql-server.ts b/infrastructure/evault-core/src/core/protocol/graphql-server.ts index 20fd1476e..fc9235ed2 100644 --- a/infrastructure/evault-core/src/core/protocol/graphql-server.ts +++ b/infrastructure/evault-core/src/core/protocol/graphql-server.ts @@ -5,13 +5,15 @@ import { Permission, resolveAclBlock, } from "../acl"; -import axios from "axios"; import type { GraphQLSchema } from "graphql"; import { createSchema, createYoga } from "graphql-yoga"; import { getJWTHeader } from "w3id"; -import { BindingDocumentService, BINDING_DOCUMENT_ONTOLOGY } from "../../services/BindingDocumentService"; +import { + BindingDocumentService, + BINDING_DOCUMENT_ONTOLOGY, +} from "../../services/BindingDocumentService"; import { hashAnswer } from "../utils/security-answer"; -import type { DbService } from "../db/db.service"; +import type { AwarenessWriteContext, DbService } from "../db/db.service"; import { computeEnvelopeHash, computeEnvelopeHashForDelete, @@ -39,7 +41,8 @@ export class GraphQLServer { private evaultPublicKey: string | null; private evaultW3ID: string | null; private evaultInstance: any; // Reference to the eVault instance - private messageNotificationService: MessageNotificationService | null = null; + private messageNotificationService: MessageNotificationService | null = + null; private securityQuestionService: SecurityQuestionService | null = null; constructor( @@ -87,81 +90,16 @@ export class GraphQLServer { return this.securityQuestionService; } - /** - * Forwards an awareness packet to Awareness as a Service (AaaS). - * - * AaaS has replaced eVault's built-in webhook fanout: instead of querying - * the registry and POSTing to every platform here, we make a single POST - * to AaaS, which owns subscription matching, retry/dead-letter delivery and - * the catch-all fanout that preserves the previous behaviour. - * - * @param webhookPayload - The awareness packet { id, w3id, evaultPublicKey, - * data, schemaId } - * @param requestingPlatform - The platform that triggered the change, if - * known. AaaS uses it to skip delivering the packet - * back to its origin (prevents webhook ping-pong). - */ - private async notifyAwareness( - webhookPayload: any, - requestingPlatform: string | null = null, - ): Promise { - // One log line per dispatch — this remains the source of truth for - // "what eVault claims it sent"; correlate against AaaS ingest logs. - try { - const payloadJson = JSON.stringify(webhookPayload); - console.log( - `[webhook] id=${webhookPayload?.id} schemaId=${webhookPayload?.schemaId} w3id=${webhookPayload?.w3id} payload=${payloadJson}`, - ); - } catch { - console.log( - `[webhook] id=${webhookPayload?.id} schemaId=${webhookPayload?.schemaId} payload=`, - ); - } - - if (!process.env.AWARENESS_SERVICE_URL) { - console.log("[webhook] AWARENESS_SERVICE_URL not set, skipping"); - return; - } - - const ingestUrl = new URL( - "/ingest", - process.env.AWARENESS_SERVICE_URL, - ).toString(); - const maxAttempts = 3; - - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - try { - const response = await axios.post( - ingestUrl, - { ...webhookPayload, requestingPlatform }, - { - headers: { - "Content-Type": "application/json", - "x-ingest-secret": - process.env.AWARENESS_INGEST_SECRET ?? "", - }, - timeout: 5000, - }, - ); - console.log( - `[webhook] AaaS accepted id=${webhookPayload?.id} status=${response.status} attempt=${attempt}`, - ); - return; - } catch (error: any) { - const status = error?.response?.status ?? "no-response"; - const code = error?.code ?? "unknown"; - const message = error?.message ?? "unknown error"; - console.error( - `[webhook] AaaS ingest failed id=${webhookPayload?.id} status=${status} code=${code} attempt=${attempt}/${maxAttempts}: ${message}`, - ); - - if (attempt < maxAttempts) { - await new Promise((resolve) => - setTimeout(resolve, 250 * 2 ** (attempt - 1)), - ); - } - } - } + /** Metadata persisted with the mutation's transactional outbox event. */ + private awarenessContext( + context: VaultContext, + skipAwareness = false, + ): AwarenessWriteContext { + return { + evaultPublicKey: this.evaultPublicKey, + requestingPlatform: context.tokenPayload?.platform ?? null, + skipAwareness, + }; } /** @@ -406,6 +344,7 @@ export class GraphQLServer { }, input.acl, context.eName, + this.awarenessContext(context), ); // Build parsed from actual written envelopes, not input @@ -423,26 +362,18 @@ export class GraphQLServer { parsed: parsedFromEnvelopes, }; - // Forward the awareness packet for create operation - const webhookPayload = { - id: result.metaEnvelope.id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: input.payload, - schemaId: input.ontology, - operation: "create" as const, - }; - - // Fire-and-forget ingest to AaaS - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ); - // Send push notifications for new messages - console.log(`[NOTIF] createMetaEnvelope ontology: "${input.ontology}"`); - if (MessageNotificationService.isMessageSchema(input.ontology)) { - console.log(`[NOTIF] Message schema detected, triggering notification for envelope ${result.metaEnvelope.id}`); + console.log( + `[NOTIF] createMetaEnvelope ontology: "${input.ontology}"`, + ); + if ( + MessageNotificationService.isMessageSchema( + input.ontology, + ) + ) { + console.log( + `[NOTIF] Message schema detected, triggering notification for envelope ${result.metaEnvelope.id}`, + ); this.getMessageNotificationService() .notifyParticipants({ messageGlobalId: result.metaEnvelope.id, @@ -451,7 +382,10 @@ export class GraphQLServer { acl: input.acl, }) .catch((err) => - console.error("Message notification failed:", err), + console.error( + "Message notification failed:", + err, + ), ); } @@ -547,6 +481,7 @@ export class GraphQLServer { }, input.acl, context.eName, + this.awarenessContext(context), ); // Build parsed from actual written envelopes, not input @@ -564,29 +499,6 @@ export class GraphQLServer { parsed: parsedFromEnvelopes, }; - // Deliver webhooks for update operation. - // Use the FULL post-write state, not input.payload — - // input.payload is the partial diff the caller sent, - // and receivers overwrite their local row with - // whatever the webhook carries. Sending the diff - // would make the receiver lose every untouched - // field (e.g. a read-receipt update would wipe - // participantIds on the receiver side). - const webhookPayload = { - id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: result.mergedPayload ?? input.payload, - schemaId: input.ontology, - operation: "update" as const, - }; - - // Fire-and-forget ingest to AaaS - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ); - // Log envelope operation best-effort const platform = context.tokenPayload?.platform ?? null; @@ -676,7 +588,11 @@ export class GraphQLServer { }; } - await this.db.deleteMetaEnvelope(id, context.eName); + await this.db.deleteMetaEnvelope( + id, + context.eName, + this.awarenessContext(context), + ); // Log after delete succeeds, best-effort const platform = @@ -765,7 +681,7 @@ export class GraphQLServer { const isEmoverMigration = skipWebhooks && context.tokenPayload?.platform === - process.env.EMOVER_API_URL; + process.env.EMOVER_API_URL; // Only allow webhook skipping for authorized migration platforms const shouldSkipWebhooks = isEmoverMigration; @@ -791,6 +707,10 @@ export class GraphQLServer { input.acl, context.eName, input.id, // Preserve ID if provided + this.awarenessContext( + context, + shouldSkipWebhooks, + ), ); results.push({ @@ -799,26 +719,6 @@ export class GraphQLServer { }); successCount++; - // Forward awareness packet if not skipping - if (!shouldSkipWebhooks) { - const webhookPayload = { - id: result.metaEnvelope.id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: input.payload, - schemaId: input.ontology, - operation: "create" as const, - }; - - // Fire-and-forget ingest to AaaS - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ).catch((err) => { - console.error(`[WEBHOOK] AaaS ingest failed for bulk-create envelope ${result.metaEnvelope.id}:`, err); - }); - } - // Log envelope operation best-effort const platform = context.tokenPayload?.platform ?? null; @@ -947,6 +847,7 @@ export class GraphQLServer { ownerSignature: input.ownerSignature, }, context.eName, + this.awarenessContext(context), ); const metaEnvelopeId = result.id; @@ -954,9 +855,12 @@ export class GraphQLServer { context.tokenPayload?.platform ?? null; const envelopeHash = computeEnvelopeHash({ id: metaEnvelopeId, - ontology: - BINDING_DOCUMENT_ONTOLOGY, - payload: result.bindingDocument as unknown as Record, + ontology: BINDING_DOCUMENT_ONTOLOGY, + payload: + result.bindingDocument as unknown as Record< + string, + unknown + >, }); this.db @@ -967,8 +871,7 @@ export class GraphQLServer { operation: "create", platform, timestamp: new Date().toISOString(), - ontology: - BINDING_DOCUMENT_ONTOLOGY, + ontology: BINDING_DOCUMENT_ONTOLOGY, }) .catch((err) => console.error( @@ -977,20 +880,6 @@ export class GraphQLServer { ), ); - const webhookPayload = { - id: metaEnvelopeId, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: result.bindingDocument, - schemaId: - BINDING_DOCUMENT_ONTOLOGY, - operation: "create" as const, - }; - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ); - return { bindingDocument: result.bindingDocument, metaEnvelopeId, @@ -1058,15 +947,18 @@ export class GraphQLServer { signature: input.signature, }, context.eName, + this.awarenessContext(context), ); const platform = context.tokenPayload?.platform ?? null; const envelopeHash = computeEnvelopeHash({ id: input.bindingDocumentId, - ontology: - BINDING_DOCUMENT_ONTOLOGY, - payload: result as unknown as Record, + ontology: BINDING_DOCUMENT_ONTOLOGY, + payload: result as unknown as Record< + string, + unknown + >, }); this.db @@ -1077,8 +969,7 @@ export class GraphQLServer { operation: "update", platform, timestamp: new Date().toISOString(), - ontology: - BINDING_DOCUMENT_ONTOLOGY, + ontology: BINDING_DOCUMENT_ONTOLOGY, }) .catch((err) => console.error( @@ -1087,20 +978,6 @@ export class GraphQLServer { ), ); - const webhookPayload = { - id: input.bindingDocumentId, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: result, - schemaId: - BINDING_DOCUMENT_ONTOLOGY, - operation: "update" as const, - }; - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ); - return { bindingDocument: result, errors: [], @@ -1193,9 +1070,8 @@ export class GraphQLServer { } try { - const result = await this - .getSecurityQuestionService() - .validate( + const result = + await this.getSecurityQuestionService().validate( context.eName, input.metaEnvelopeId, input.candidate, @@ -1264,6 +1140,7 @@ export class GraphQLServer { }, input.acl, context.eName, + this.awarenessContext(context), ); // Add parsed field to metaEnvelope for GraphQL response @@ -1272,28 +1149,18 @@ export class GraphQLServer { parsed: input.payload, }; - // Forward the awareness packet for create operation. - // The requesting platform is passed so AaaS can skip - // delivering the packet back to its origin — the same - // ping-pong guard the old fanout enforced here. - const webhookPayload = { - id: result.metaEnvelope.id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: input.payload, - schemaId: input.ontology, - operation: "create" as const, - }; - - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, - ); - // Send push notifications for new messages - console.log(`[NOTIF] storeMetaEnvelope ontology: "${input.ontology}"`); - if (MessageNotificationService.isMessageSchema(input.ontology)) { - console.log(`[NOTIF] Message schema detected in storeMetaEnvelope, triggering notification for envelope ${result.metaEnvelope.id}`); + console.log( + `[NOTIF] storeMetaEnvelope ontology: "${input.ontology}"`, + ); + if ( + MessageNotificationService.isMessageSchema( + input.ontology, + ) + ) { + console.log( + `[NOTIF] Message schema detected in storeMetaEnvelope, triggering notification for envelope ${result.metaEnvelope.id}`, + ); this.getMessageNotificationService() .notifyParticipants({ messageGlobalId: result.metaEnvelope.id, @@ -1302,7 +1169,10 @@ export class GraphQLServer { acl: input.acl, }) .catch((err) => - console.error("[NOTIF] Message notification failed:", err), + console.error( + "[NOTIF] Message notification failed:", + err, + ), ); } @@ -1398,7 +1268,8 @@ export class GraphQLServer { errors: [ { field: "content", - message: "File content is empty or not valid base64", + message: + "File content is empty or not valid base64", code: "INVALID_CONTENT", }, ], @@ -1409,7 +1280,9 @@ export class GraphQLServer { const MAX_FILE_BYTES = 250 * 1024 * 1024; // 250 MB if (buffer.length > MAX_FILE_BYTES) { - const maxMb = Math.round(MAX_FILE_BYTES / (1024 * 1024)); + const maxMb = Math.round( + MAX_FILE_BYTES / (1024 * 1024), + ); return { errors: [ { @@ -1458,35 +1331,7 @@ export class GraphQLServer { }, input.acl, context.eName, - ); - - // Forward the awareness packet, exactly as every - // other write path does. Without this an uploaded - // blob is invisible to AaaS, which forces consumers - // to mirror it as a second envelope under a - // different ontology just to observe the upload. - // - // `data` is the stored payload verbatim so the - // packet matches what a consumer reads back via - // metaEnvelope(id) or GET /api/packets. - // - // Fire-and-forget: the envelope is already - // committed, so an AaaS outage must not fail the - // upload. Awaiting here would drop into the catch - // block below and delete a blob that is still - // referenced by a live envelope. - const webhookPayload = { - id: result.metaEnvelope.id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: payload, - schemaId: FILE_SCHEMA_ID, - operation: "create" as const, - }; - - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, + this.awarenessContext(context), ); // Log envelope operation best-effort (do not fail mutation) @@ -1582,26 +1427,7 @@ export class GraphQLServer { }, input.acl, context.eName, - ); - - // Deliver webhooks with the FULL post-write state. - // See the long comment on the new updateMetaEnvelope - // resolver above — sending input.payload (the - // partial diff) would make receivers clobber their - // own untouched fields. - const webhookPayload = { - id: id, - w3id: context.eName, - evaultPublicKey: this.evaultPublicKey, - data: result.mergedPayload ?? input.payload, - schemaId: input.ontology, - operation: "update" as const, - }; - - // Fire-and-forget ingest to AaaS - this.notifyAwareness( - webhookPayload, - context.tokenPayload?.platform || null, + this.awarenessContext(context), ); // Log envelope operation best-effort (do not fail mutation) @@ -1653,7 +1479,11 @@ export class GraphQLServer { id, context.eName, ); - await this.db.deleteMetaEnvelope(id, context.eName); + await this.db.deleteMetaEnvelope( + id, + context.eName, + this.awarenessContext(context), + ); // Log after delete succeeds, best-effort const platform = context.tokenPayload?.platform ?? null; const envelopeHash = computeEnvelopeHashForDelete(id); @@ -1698,6 +1528,7 @@ export class GraphQLServer { envelopeId, newValue, context.eName, + this.awarenessContext(context), ); if (metaInfo) { const platform = diff --git a/infrastructure/evault-core/src/index.ts b/infrastructure/evault-core/src/index.ts index 1378395f2..42424316c 100644 --- a/infrastructure/evault-core/src/index.ts +++ b/infrastructure/evault-core/src/index.ts @@ -3,6 +3,7 @@ import path from "path"; import cors from "cors"; import dotenv from "dotenv"; import express, { type Request, type Response } from "express"; +import type { Server as HttpServer } from "node:http"; import { AppDataSource } from "./config/database"; import { NotificationController } from "./controllers/NotificationController"; import { ProvisioningController } from "./controllers/ProvisioningController"; @@ -29,6 +30,7 @@ import { connectWithRetry } from "./core/db/retry-neo4j"; import { registerHttpRoutes } from "./core/http/server"; import { GraphQLServer } from "./core/protocol/graphql-server"; import { LogService } from "./core/w3id/log-service"; +import { AwarenessOutboxDispatcher } from "./core/awareness/awareness-outbox-dispatcher"; dotenv.config({ path: path.resolve(__dirname, "../../../.env") }); @@ -41,7 +43,13 @@ expressApp.use( cors({ origin: "*", methods: ["GET", "POST", "OPTIONS", "PATCH"], - allowedHeaders: ["Content-Type", "Authorization", "X-ENAME", "X-ON-BEHALF-OF", "x-shared-secret"], + allowedHeaders: [ + "Content-Type", + "Authorization", + "X-ENAME", + "X-ON-BEHALF-OF", + "x-shared-secret", + ], credentials: true, }), ); @@ -74,6 +82,8 @@ let graphqlServer: GraphQLServer; let logService: LogService; let driver: Driver; let provisioningService: ProvisioningService | undefined; +let awarenessOutboxDispatcher: AwarenessOutboxDispatcher | undefined; +let expressServer: HttpServer | undefined; // Initialize eVault Core const initializeEVault = async ( @@ -146,6 +156,15 @@ const initializeEVault = async ( console.warn("Failed to create EnvelopeOperationLog indexes:", error); } + try { + const { createAwarenessOutboxIndexes } = await import( + "./core/db/migrations/add-awareness-outbox-indexes" + ); + await createAwarenessOutboxIndexes(driver); + } catch (error) { + console.warn("Failed to create awareness outbox indexes:", error); + } + // One-time backfill: create operation logs for existing metaenvelopes (platform inferred from ontology) try { const { backfillEnvelopeOperationLogs } = await import( @@ -157,6 +176,8 @@ const initializeEVault = async ( } const dbService = new DbService(driver); + awarenessOutboxDispatcher = new AwarenessOutboxDispatcher(driver); + awarenessOutboxDispatcher.start(); const protectedZoneService = new ProtectedZoneService(driver); logService = new LogService(driver); const publicKey = process.env.EVAULT_PUBLIC_KEY || null; @@ -188,7 +209,13 @@ const initializeEVault = async ( await fastifyServer.register(fastifyCors, { origin: true, // Allow all origins methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - allowedHeaders: ["Content-Type", "Authorization", "X-ENAME", "X-ON-BEHALF-OF", "x-shared-secret"], + allowedHeaders: [ + "Content-Type", + "Authorization", + "X-ENAME", + "X-ON-BEHALF-OF", + "x-shared-secret", + ], credentials: true, }); @@ -264,7 +291,9 @@ const initializeEVault = async ( // Provisioner JWKs — must be on Express (provisioner URL port) for signer URL resolution expressApp.get("/.well-known/jwks.json", (_req: Request, res: Response) => { try { - const { getProvisionerJwk } = require("./core/utils/provisioner-signer"); + const { + getProvisionerJwk, + } = require("./core/utils/provisioner-signer"); res.json({ keys: [getProvisionerJwk()] }); } catch { res.json({ keys: [] }); @@ -276,6 +305,95 @@ expressApp.get("/health", (req: Request, res: Response) => { res.json({ status: "ok" }); }); +expressApp.get("/ready", async (_req: Request, res: Response) => { + try { + await driver.getServerInfo(); + const dispatcher = awarenessOutboxDispatcher?.health(); + const dispatcherReady = Boolean( + dispatcher?.configured && dispatcher.running, + ); + const session = driver.session(); + try { + const result = await session.run(` + MATCH (a:AwarenessOutbox) + WHERE a.status IN ['pending', 'failed', 'delivering'] + RETURN count(a) AS queued, + coalesce(max(timestamp() - a.createdAt), 0) AS oldestAgeMs + `); + const record = result.records[0]; + return res.status(dispatcherReady ? 200 : 503).json({ + status: dispatcherReady ? "ready" : "not-ready", + neo4j: "ok", + awarenessDispatcher: dispatcher ?? { + configured: false, + running: false, + }, + awarenessOutbox: { + queued: record.get("queued").toNumber(), + oldestAgeMs: record.get("oldestAgeMs").toNumber(), + }, + }); + } finally { + await session.close(); + } + } catch (error) { + return res.status(503).json({ + status: "not-ready", + neo4j: "unavailable", + error: error instanceof Error ? error.message : String(error), + }); + } +}); + +expressApp.get("/metrics", async (_req: Request, res: Response) => { + try { + const dispatcher = awarenessOutboxDispatcher?.health(); + const lastCycleAgeSeconds = dispatcher?.lastCycleAt + ? Math.max( + 0, + (Date.now() - dispatcher.lastCycleAt.getTime()) / 1000, + ) + : -1; + const session = driver.session(); + try { + const result = await session.run(` + MATCH (a:AwarenessOutbox) + RETURN count(CASE WHEN a.status IN ['pending', 'failed', 'delivering'] THEN 1 END) AS queued, + count(CASE WHEN a.status = 'failed' THEN 1 END) AS failed, + coalesce(max(CASE WHEN a.status IN ['pending', 'failed', 'delivering'] THEN timestamp() - a.createdAt ELSE 0 END), 0) AS oldestAgeMs + `); + const record = result.records[0]; + return res + .type("text/plain; version=0.0.4") + .send( + [ + "# TYPE evault_awareness_outbox_events gauge", + `evault_awareness_outbox_events{status=\"active\"} ${record.get("queued").toNumber()}`, + `evault_awareness_outbox_events{status=\"failed\"} ${record.get("failed").toNumber()}`, + "# TYPE evault_awareness_outbox_oldest_seconds gauge", + `evault_awareness_outbox_oldest_seconds ${record.get("oldestAgeMs").toNumber() / 1000}`, + "# TYPE evault_awareness_dispatcher_running gauge", + `evault_awareness_dispatcher_running ${dispatcher?.running ? 1 : 0}`, + "# TYPE evault_awareness_dispatcher_configured gauge", + `evault_awareness_dispatcher_configured ${dispatcher?.configured ? 1 : 0}`, + "# TYPE evault_awareness_dispatcher_last_cycle_age_seconds gauge", + `evault_awareness_dispatcher_last_cycle_age_seconds ${lastCycleAgeSeconds}`, + "", + ].join("\n"), + ); + } finally { + await session.close(); + } + } catch (error) { + return res + .status(503) + .type("text/plain") + .send( + `# metrics unavailable: ${error instanceof Error ? error.message : String(error)}\n`, + ); + } +}); + // Start the server const start = async () => { try { @@ -316,7 +434,7 @@ const start = async () => { await initializeEVault(provisioningService); // Start Express server for provisioning (after Fastify is ready) - expressApp.listen(expressPort, () => { + expressServer = expressApp.listen(expressPort, () => { console.log( `Express server (Provisioning API) running on port ${expressPort}`, ); @@ -328,3 +446,19 @@ const start = async () => { }; start(); + +async function shutdown(signal: string): Promise { + console.log(`${signal} received, shutting down eVault`); + await awarenessOutboxDispatcher?.stop(); + await fastifyServer?.close(); + if (expressServer) { + await new Promise((resolve) => + expressServer!.close(() => resolve()), + ); + } + await driver?.close(); + if (AppDataSource.isInitialized) await AppDataSource.destroy(); +} + +process.once("SIGTERM", () => void shutdown("SIGTERM")); +process.once("SIGINT", () => void shutdown("SIGINT")); diff --git a/infrastructure/evault-core/src/services/BindingDocumentService.ts b/infrastructure/evault-core/src/services/BindingDocumentService.ts index 2f675e267..73f5350af 100644 --- a/infrastructure/evault-core/src/services/BindingDocumentService.ts +++ b/infrastructure/evault-core/src/services/BindingDocumentService.ts @@ -2,8 +2,11 @@ import axios from "axios"; import { createHash } from "node:crypto"; import nacl from "tweetnacl"; import { verifySignature } from "signature-validator"; -import type { DbService } from "../core/db/db.service"; -import type { FindMetaEnvelopesPaginatedOptions, MetaEnvelopeConnection } from "../core/db/types"; +import type { AwarenessWriteContext, DbService } from "../core/db/db.service"; +import type { + FindMetaEnvelopesPaginatedOptions, + MetaEnvelopeConnection, +} from "../core/db/types"; import { computeBindingDocumentHash, getCanonicalBindingDocumentBytes, @@ -49,10 +52,14 @@ function validateBindingDocumentData( typeof d.name !== "string" ) { throw new ValidationError( - 'id_document data must have string fields: vendor, reference, name', + "id_document data must have string fields: vendor, reference, name", ); } - return { vendor: d.vendor, reference: d.reference, name: d.name } as BindingDocumentIdDocumentData; + return { + vendor: d.vendor, + reference: d.reference, + name: d.name, + } as BindingDocumentIdDocumentData; } case "photograph": { if (typeof d.photoBlob !== "string") { @@ -62,37 +69,46 @@ function validateBindingDocumentData( } // description is optional and back-compat: older photos may not // have it. Validate the type only when present. - if (d.description !== undefined && typeof d.description !== "string") { + if ( + d.description !== undefined && + typeof d.description !== "string" + ) { throw new ValidationError( "photograph data field 'description' must be a string when provided", ); } - const out: BindingDocumentPhotographData = { photoBlob: d.photoBlob }; - if (typeof d.description === "string") out.description = d.description; + const out: BindingDocumentPhotographData = { + photoBlob: d.photoBlob, + }; + if (typeof d.description === "string") + out.description = d.description; return out; } case "social_connection": { if (typeof d.name !== "string") { throw new ValidationError( - 'social_connection data must have string field: name', + "social_connection data must have string field: name", ); } - const denseParties = Array.isArray(d.parties) ? Array.from(d.parties) : null; + const denseParties = Array.isArray(d.parties) + ? Array.from(d.parties) + : null; if ( !denseParties || denseParties.length !== 2 || !denseParties.every( - (p: unknown) => typeof p === "string" && (p as string).startsWith("@"), + (p: unknown) => + typeof p === "string" && (p as string).startsWith("@"), ) || denseParties[0] === denseParties[1] ) { throw new ValidationError( - 'social_connection data must have parties: array of 2 distinct eNames prefixed with @', + "social_connection data must have parties: array of 2 distinct eNames prefixed with @", ); } if (typeof d.relation_description !== "string") { throw new ValidationError( - 'social_connection data must have string field: relation_description', + "social_connection data must have string field: relation_description", ); } return { @@ -122,7 +138,10 @@ function validateBindingDocumentData( } as BindingDocumentPersonalParametersData; } case "security_question": { - if (typeof d.question !== "string" || d.question.trim().length === 0) { + if ( + typeof d.question !== "string" || + d.question.trim().length === 0 + ) { throw new ValidationError( "security_question data must have non-empty string field: question", ); @@ -152,11 +171,16 @@ function validateBindingDocumentData( case "deployment_key": { if ( d.kind !== "deployment_key" || - typeof d.deploymentName !== "string" || !d.deploymentName.trim() || - typeof d.environment !== "string" || !d.environment.trim() || - typeof d.deployerEname !== "string" || !d.deployerEname.startsWith("@") || - typeof d.platformEname !== "string" || !d.platformEname.startsWith("@") || - typeof d.publicKey !== "string" || !d.publicKey.startsWith("z") || + typeof d.deploymentName !== "string" || + !d.deploymentName.trim() || + typeof d.environment !== "string" || + !d.environment.trim() || + typeof d.deployerEname !== "string" || + !d.deployerEname.startsWith("@") || + typeof d.platformEname !== "string" || + !d.platformEname.startsWith("@") || + typeof d.publicKey !== "string" || + !d.publicKey.startsWith("z") || d.algorithm !== "ECDSA_P256" ) { throw new ValidationError("deployment_key data is invalid"); @@ -174,11 +198,16 @@ function validateBindingDocumentData( case "software_version": { if ( d.kind !== "software_version" || - typeof d.platformEname !== "string" || !d.platformEname.startsWith("@") || - typeof d.versionEname !== "string" || !d.versionEname.startsWith("@") || - typeof d.version !== "string" || !/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(d.version) || - typeof d.releaseTag !== "string" || !d.releaseTag.trim() || - typeof d.commitSha !== "string" || !/^[0-9a-f]{40,64}$/i.test(d.commitSha) + typeof d.platformEname !== "string" || + !d.platformEname.startsWith("@") || + typeof d.versionEname !== "string" || + !d.versionEname.startsWith("@") || + typeof d.version !== "string" || + !/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(d.version) || + typeof d.releaseTag !== "string" || + !d.releaseTag.trim() || + typeof d.commitSha !== "string" || + !/^[0-9a-f]{40,64}$/i.test(d.commitSha) ) { throw new ValidationError("software_version data is invalid"); } @@ -193,7 +222,9 @@ function validateBindingDocumentData( } default: { const _exhaustive: never = type; - throw new ValidationError(`Unknown binding document type: ${_exhaustive}`); + throw new ValidationError( + `Unknown binding document type: ${_exhaustive}`, + ); } } } @@ -213,7 +244,10 @@ export interface AddCounterpartySignatureInput { export class BindingDocumentService { private registryUrl: string; - constructor(private db: DbService, registryUrl?: string) { + constructor( + private db: DbService, + registryUrl?: string, + ) { this.registryUrl = registryUrl || process.env.PUBLIC_REGISTRY_URL || ""; } @@ -239,7 +273,11 @@ export class BindingDocumentService { private async verifyUserSignature( signer: string, signature: string, - doc: { subject: string; type: BindingDocumentType; data: BindingDocumentData }, + doc: { + subject: string; + type: BindingDocumentType; + data: BindingDocumentData; + }, ): Promise { return this.verifyUserPayload( signer, @@ -250,9 +288,14 @@ export class BindingDocumentService { private async verifyBundleSignature( signature: BindingDocumentSignature, - doc: { subject: string; type: BindingDocumentType; data: BindingDocumentData }, + doc: { + subject: string; + type: BindingDocumentType; + data: BindingDocumentData; + }, ): Promise { - if (signature.scope !== "bundle" || !signature.signedPayload) return false; + if (signature.scope !== "bundle" || !signature.signedPayload) + return false; try { const bundle = JSON.parse(signature.signedPayload) as { type?: unknown; @@ -264,13 +307,17 @@ export class BindingDocumentService { bundle.version !== 1 || !Array.isArray(bundle.documents) || bundle.documents.length !== 2 - ) return false; + ) + return false; const expectedHash = computeBindingDocumentHash(doc); const member = bundle.documents.some((item) => { if (!item || typeof item !== "object") return false; const entry = item as Record; - return entry.hash === expectedHash && - entry.subject === doc.subject && entry.type === doc.type; + return ( + entry.hash === expectedHash && + entry.subject === doc.subject && + entry.type === doc.type + ); }); if (!member) return false; const digest = createHash("sha256") @@ -298,7 +345,9 @@ export class BindingDocumentService { return Uint8Array.from(Buffer.from(padded, "base64")); } - private parseProvisionerSigner(signer: string): { jwksUrl: string; kid: string } | null { + private parseProvisionerSigner( + signer: string, + ): { jwksUrl: string; kid: string } | null { try { const url = new URL(signer); if (!url.pathname.endsWith("/.well-known/jwks.json")) return null; @@ -351,10 +400,14 @@ export class BindingDocumentService { async createBindingDocument( input: CreateBindingDocumentInput, eName: string, + awareness?: AwarenessWriteContext, ): Promise<{ id: string; bindingDocument: BindingDocument }> { const normalizedSubject = this.normalizeSubject(input.subject); - const validatedData = validateBindingDocumentData(input.type, input.data); + const validatedData = validateBindingDocumentData( + input.type, + input.data, + ); const docToVerify = { subject: normalizedSubject, @@ -362,9 +415,14 @@ export class BindingDocumentService { data: validatedData, }; const expectedHash = computeBindingDocumentHash(docToVerify); - const hasLegacyHashSignature = input.ownerSignature.signature === expectedHash; - const isProvisionerSigner = /^https?:\/\//.test(input.ownerSignature.signer); - const isDeploymentDocument = input.type === "deployment_key" || input.type === "software_version"; + const hasLegacyHashSignature = + input.ownerSignature.signature === expectedHash; + const isProvisionerSigner = /^https?:\/\//.test( + input.ownerSignature.signer, + ); + const isDeploymentDocument = + input.type === "deployment_key" || + input.type === "software_version"; const hasValidUserSignature = !hasLegacyHashSignature && @@ -374,20 +432,31 @@ export class BindingDocumentService { input.ownerSignature.signature, docToVerify, )); - const hasValidBundleSignature = isDeploymentDocument && - (await this.verifyBundleSignature(input.ownerSignature, docToVerify)); + const hasValidBundleSignature = + isDeploymentDocument && + (await this.verifyBundleSignature( + input.ownerSignature, + docToVerify, + )); if ( (isDeploymentDocument && !hasValidBundleSignature) || - (!isDeploymentDocument && !hasLegacyHashSignature && !isProvisionerSigner && !hasValidUserSignature) + (!isDeploymentDocument && + !hasLegacyHashSignature && + !isProvisionerSigner && + !hasValidUserSignature) ) { throw new ValidationError("Invalid owner signature"); } if ( input.type === "deployment_key" && - input.ownerSignature.signer !== (validatedData as BindingDocumentDeploymentKeyData).deployerEname + input.ownerSignature.signer !== + (validatedData as BindingDocumentDeploymentKeyData) + .deployerEname ) { - throw new ValidationError("deployment_key must be signed by its deployer"); + throw new ValidationError( + "deployment_key must be signed by its deployer", + ); } const bindingDocument: BindingDocument = { @@ -397,7 +466,9 @@ export class BindingDocumentService { signatures: [input.ownerSignature], }; - const isPublicDeploymentDocument = input.type === "deployment_key" || input.type === "software_version"; + const isPublicDeploymentDocument = + input.type === "deployment_key" || + input.type === "software_version"; const acl = isPublicDeploymentDocument ? ["*"] : [normalizedSubject]; const result = await this.db.storeMetaEnvelope( { @@ -407,6 +478,7 @@ export class BindingDocumentService { }, [normalizedSubject], eName, + awareness, ); return { @@ -418,6 +490,7 @@ export class BindingDocumentService { async addCounterpartySignature( input: AddCounterpartySignatureInput, eName: string, + awareness?: AwarenessWriteContext, ): Promise { const metaEnvelope = await this.db.findMetaEnvelopeById( input.metaEnvelopeId, @@ -487,6 +560,7 @@ export class BindingDocumentService { }, [bindingDocument.subject], eName, + awareness, ); return updatedBindingDocument; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7e15de67..47d16410c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4009,6 +4009,9 @@ importers: specifier: ^9.0.1 version: 9.0.1 devDependencies: + '@testcontainers/postgresql': + specifier: ^10.0.1 + version: 10.28.0 '@types/cors': specifier: ^2.8.17 version: 2.8.19 @@ -4036,6 +4039,9 @@ importers: typescript: specifier: ^5.3.3 version: 5.9.3 + vitest: + specifier: ^1.6.1 + version: 1.6.1(@types/node@20.19.26)(jsdom@19.0.0(bufferutil@4.1.0))(lightningcss@1.31.1)(sass@1.98.0)(terser@5.46.0) services/awareness-service/portal: dependencies: diff --git a/services/awareness-service/api/package.json b/services/awareness-service/api/package.json index 3a7c8fb0d..be3600958 100644 --- a/services/awareness-service/api/package.json +++ b/services/awareness-service/api/package.json @@ -7,6 +7,7 @@ "start": "node dist/index.js", "dev": "nodemon --exec ts-node src/index.ts", "build": "tsc", + "test": "vitest run", "typeorm": "typeorm-ts-node-commonjs", "migration:generate": "npm run typeorm migration:generate -- -d src/database/data-source.ts", "migration:run": "npm run typeorm migration:run -- -d src/database/data-source.ts", @@ -35,8 +36,10 @@ "@types/node": "^20.11.24", "@types/pg": "^8.11.2", "@types/uuid": "^9.0.8", + "@testcontainers/postgresql": "^10.0.1", "nodemon": "^3.0.3", "ts-node": "^10.9.2", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "vitest": "^1.6.1" } } diff --git a/services/awareness-service/api/src/config.ts b/services/awareness-service/api/src/config.ts index e67c45047..e7e0e41ac 100644 --- a/services/awareness-service/api/src/config.ts +++ b/services/awareness-service/api/src/config.ts @@ -1,5 +1,5 @@ +import path from "node:path"; import { config as loadEnv } from "dotenv"; -import path from "path"; loadEnv({ path: path.resolve(__dirname, "../../../../.env") }); @@ -20,10 +20,15 @@ function timerInterval(name: string, fallback: number): number { : fallback; } +function positiveInteger(name: string, fallback: number): number { + const value = Number(process.env[name]); + return Number.isSafeInteger(value) && value > 0 ? value : fallback; +} + export const config = { /** Postgres connection string for the AaaS database. */ databaseUrl: process.env.AWARENESS_DATABASE_URL ?? "", - apiPort: parseInt(process.env.AWARENESS_API_PORT ?? "4100", 10), + apiPort: Number.parseInt(process.env.AWARENESS_API_PORT ?? "4100", 10), /** Shared secret evault-core must present on POST /ingest. */ ingestSecret: process.env.AWARENESS_INGEST_SECRET ?? "", /** Registry used both for catch-all seeding and W3DS signature checks. */ @@ -36,8 +41,7 @@ export const config = { .filter(Boolean), /** Secret used to sign portal session JWTs. */ jwtSecret: process.env.AAAS_JWT_SECRET ?? "awareness-dev-secret", - maxAttempts: parseInt(process.env.AWARENESS_MAX_ATTEMPTS ?? "3", 10), - deliveryPollMs: parseInt( + deliveryPollMs: Number.parseInt( process.env.AWARENESS_DELIVERY_POLL_MS ?? "2000", 10, ), @@ -46,6 +50,26 @@ export const config = { /** Public base URL of the AaaS API, used to build W3DS auth callbacks. */ publicUrl: process.env.AWARENESS_PUBLIC_URL ?? "http://localhost:4100", dbCaCert: process.env.DB_CA_CERT, + workerId: + process.env.AAAS_WORKER_ID ?? + `${process.env.HOSTNAME ?? "local"}-${process.pid}`, + deliveryLeaseMs: positiveInteger("AWARENESS_DELIVERY_LEASE_MS", 30_000), + deliveryBatchTimeoutMs: positiveInteger( + "AWARENESS_DELIVERY_BATCH_TIMEOUT_MS", + 25_000, + ), + deliveryRetryWindowMs: positiveInteger( + "AWARENESS_DELIVERY_RETRY_WINDOW_MS", + 24 * 60 * 60 * 1000, + ), + workerHeartbeatMs: positiveInteger("AWARENESS_WORKER_HEARTBEAT_MS", 10_000), + workerStaleMs: positiveInteger("AWARENESS_WORKER_STALE_MS", 30_000), + dbStatementTimeoutMs: positiveInteger( + "AWARENESS_DB_STATEMENT_TIMEOUT_MS", + 10_000, + ), + dbQueryTimeoutMs: positiveInteger("AWARENESS_DB_QUERY_TIMEOUT_MS", 12_000), + dbLockTimeoutMs: positiveInteger("AWARENESS_DB_LOCK_TIMEOUT_MS", 5_000), }; export { required }; diff --git a/services/awareness-service/api/src/controllers/AdminController.ts b/services/awareness-service/api/src/controllers/AdminController.ts index 564feb4d7..426dfef9e 100644 --- a/services/awareness-service/api/src/controllers/AdminController.ts +++ b/services/awareness-service/api/src/controllers/AdminController.ts @@ -113,9 +113,7 @@ export function adminRouter(): Router { const includeResolved = req.query.resolved === "true"; // Metadata only - the `payload` column holds the full webhook body and // would bloat the list. Fetch it on replay if ever needed. - const deadLetters = await AppDataSource.getRepository( - DeadLetter, - ).find({ + const deadLetters = await AppDataSource.getRepository(DeadLetter).find({ select: [ "id", "deliveryId", @@ -146,6 +144,7 @@ export function adminRouter(): Router { .innerJoin(Consumer, "c", "c.id = s.consumerId") .select([ 'd.id AS "deliveryId"', + 'd.eventId AS "eventId"', 'd.packetId AS "packetId"', 'd.status AS "status"', 'd.attempts AS "attempts"', @@ -184,7 +183,8 @@ export function adminRouter(): Router { } }); - // Replay re-queues the original delivery and resolves the dead letter. + // Replay re-queues the original delivery. The worker resolves the dead + // letter only after the subscriber acknowledges a later attempt. router.post("/api/admin/dead-letters/:id/replay", async (req, res) => { const dlRepo = AppDataSource.getRepository(DeadLetter); const deadLetter = await dlRepo.findOne({ @@ -198,12 +198,16 @@ export function adminRouter(): Router { status: "pending", attempts: 0, nextAttemptAt: new Date(), + retryStartedAt: new Date(), lastError: null, lastResponseStatus: null, + deliveredAt: null, + leaseOwner: null, + leaseToken: null, + leaseExpiresAt: null, }, ); - deadLetter.resolved = true; - await dlRepo.save(deadLetter); + // Resolution is recorded by the worker only after a successful POST. res.json({ ok: true }); }); diff --git a/services/awareness-service/api/src/controllers/ConsumerController.ts b/services/awareness-service/api/src/controllers/ConsumerController.ts index 501e0c7e2..5d19a6d5c 100644 --- a/services/awareness-service/api/src/controllers/ConsumerController.ts +++ b/services/awareness-service/api/src/controllers/ConsumerController.ts @@ -55,10 +55,7 @@ export function consumerRouter(): Router { }); router.delete("/api/me/api-keys/:id", async (req, res) => { - const ok = await apiKeyService.revoke( - req.params.id, - req.consumer!.id, - ); + const ok = await apiKeyService.revoke(req.params.id, req.consumer!.id); if (!ok) return res.status(404).json({ error: "not found" }); res.json({ ok: true }); }); @@ -78,6 +75,7 @@ export function consumerRouter(): Router { "d.id", "d.subscriptionId", "d.packetId", + "d.eventId", "d.status", "d.attempts", "d.nextAttemptAt", diff --git a/services/awareness-service/api/src/controllers/IngestController.ts b/services/awareness-service/api/src/controllers/IngestController.ts index 87ae8b2cd..613c30d60 100644 --- a/services/awareness-service/api/src/controllers/IngestController.ts +++ b/services/awareness-service/api/src/controllers/IngestController.ts @@ -1,6 +1,6 @@ import { Router } from "express"; import { config } from "../config"; -import { IngestService } from "../services/IngestService"; +import { EventIdConflictError, IngestService } from "../services/IngestService"; import type { AwarenessPayload } from "../types"; /** @@ -29,10 +29,13 @@ export function ingestRouter(): Router { try { const result = await service.ingest(body); console.log( - `[ingest] id=${body.id} schemaId=${body.schemaId} w3id=${body.w3id ?? ""} queued=${result.deliveriesQueued}`, + `[ingest] eventId=${result.eventId} id=${body.id} schemaId=${body.schemaId} w3id=${body.w3id ?? ""} duplicate=${result.duplicate} queued=${result.deliveriesQueued}`, ); return res.json({ ok: true, ...result }); } catch (err) { + if (err instanceof EventIdConflictError) { + return res.status(409).json({ error: err.message }); + } console.error("[ingest] failed:", err); return res.status(500).json({ error: "ingest failed" }); } diff --git a/services/awareness-service/api/src/controllers/QueryController.ts b/services/awareness-service/api/src/controllers/QueryController.ts index 104bb643e..7f8089b6b 100644 --- a/services/awareness-service/api/src/controllers/QueryController.ts +++ b/services/awareness-service/api/src/controllers/QueryController.ts @@ -2,6 +2,7 @@ import { Router } from "express"; import { Brackets, type SelectQueryBuilder } from "typeorm"; import { AppDataSource } from "../database/data-source"; import { Packet } from "../database/entities/Packet"; +import { AwarenessEvent } from "../database/entities/AwarenessEvent"; import { consumerAuth } from "../middleware/consumerAuth"; import { decodeCursor, encodeCursor } from "../utils/cursor"; @@ -24,9 +25,7 @@ export function queryRouter(): Router { .filter(Boolean); const evault = typeof req.query.evault === "string" ? req.query.evault : null; - const from = req.query.from - ? new Date(String(req.query.from)) - : null; + const from = req.query.from ? new Date(String(req.query.from)) : null; const to = req.query.to ? new Date(String(req.query.to)) : null; let limit = parseInt(String(req.query.limit ?? DEFAULT_LIMIT), 10); @@ -44,10 +43,11 @@ export function queryRouter(): Router { // Applies the ontology / eVault / time-range filters (everything // except the pagination cursor) to a fresh query builder. - const withFilters = (): SelectQueryBuilder => { - const qb = AppDataSource.getRepository(Packet).createQueryBuilder( - "p", - ); + const withFilters = (): SelectQueryBuilder => { + const qb = + AppDataSource.getRepository(AwarenessEvent).createQueryBuilder( + "p", + ); if (ontologies.length > 0) { qb.andWhere("p.ontology IN (:...ontologies)", { ontologies }); } @@ -69,7 +69,7 @@ export function queryRouter(): Router { // whether more pages follow. const qb = withFilters() .orderBy("p.receivedAt", "ASC") - .addOrderBy("p.id", "ASC") + .addOrderBy("p.eventId", "ASC") .take(limit + 1); if (typeof req.query.cursor === "string" && req.query.cursor) { @@ -82,7 +82,7 @@ export function queryRouter(): Router { w.where("p.receivedAt > :cReceived", { cReceived: cursor.receivedAt, }).orWhere( - "(p.receivedAt = :cReceived AND p.id > :cId)", + "(p.receivedAt = :cReceived AND p.eventId > :cId)", { cReceived: cursor.receivedAt, cId: cursor.id }, ); }), @@ -91,8 +91,20 @@ export function queryRouter(): Router { const rows = await qb.getMany(); const hasMore = rows.length > limit; - const packets = hasMore ? rows.slice(0, limit) : rows; - const last = packets[packets.length - 1]; + const events = hasMore ? rows.slice(0, limit) : rows; + const packets = events.map((event) => ({ + eventId: event.eventId, + id: event.packetId, + ontology: event.ontology, + evaultPublicKey: event.evaultPublicKey, + w3id: event.w3id, + data: event.data, + operation: event.operation, + streamVersion: event.streamVersion, + occurredAt: event.occurredAt, + receivedAt: event.receivedAt, + })); + const last = events[events.length - 1]; return res.json({ packets, @@ -105,7 +117,7 @@ export function queryRouter(): Router { hasMore && last ? encodeCursor({ receivedAt: last.receivedAt.toISOString(), - id: last.id, + id: last.eventId, }) : null, }); @@ -118,7 +130,20 @@ export function queryRouter(): Router { where: { id: req.params.id }, }); if (!packet) return res.status(404).json({ error: "not found" }); - return res.json({ packet }); + const latestEvent = await AppDataSource.getRepository(AwarenessEvent) + .createQueryBuilder("e") + .where("e.packetId = :packetId", { packetId: req.params.id }) + .orderBy("e.receivedAt", "DESC") + .addOrderBy("e.eventId", "DESC") + .getOne(); + return res.json({ + packet: { + ...packet, + eventId: latestEvent?.eventId ?? null, + streamVersion: latestEvent?.streamVersion ?? null, + occurredAt: latestEvent?.occurredAt ?? packet.receivedAt, + }, + }); }); return router; diff --git a/services/awareness-service/api/src/controllers/SystemController.ts b/services/awareness-service/api/src/controllers/SystemController.ts new file mode 100644 index 000000000..68a2c9698 --- /dev/null +++ b/services/awareness-service/api/src/controllers/SystemController.ts @@ -0,0 +1,111 @@ +import { Router } from "express"; +import { AppDataSource } from "../database/data-source"; +import { WorkerHeartbeat } from "../database/entities/WorkerHeartbeat"; +import { config } from "../config"; + +interface QueueStats { + pending: string; + failed: string; + delivering: string; + dead: string; + expired_leases: string; + oldest_pending_seconds: string | null; +} + +export async function queueStats(): Promise { + const rows = await AppDataSource.query(` + SELECT + count(*) FILTER (WHERE status = 'pending')::text AS pending, + count(*) FILTER (WHERE status = 'failed')::text AS failed, + count(*) FILTER (WHERE status = 'delivering')::text AS delivering, + count(*) FILTER (WHERE status = 'dead')::text AS dead, + count(*) FILTER ( + WHERE status = 'delivering' AND "leaseExpiresAt" <= now() + )::text AS expired_leases, + extract(epoch FROM now() - ( + min("createdAt") FILTER ( + WHERE status IN ('pending', 'failed', 'delivering') + ) + ))::text AS oldest_pending_seconds + FROM deliveries + `); + return rows[0]; +} + +export function systemRouter(): Router { + const router = Router(); + + router.get("/health", (_req, res) => { + res.json({ status: "ok", service: "awareness-service" }); + }); + + router.get("/ready", async (_req, res) => { + try { + await AppDataSource.query("SELECT 1"); + const migrationsPending = await AppDataSource.showMigrations(); + const heartbeat = await AppDataSource.getRepository(WorkerHeartbeat) + .createQueryBuilder("h") + .orderBy("h.heartbeatAt", "DESC") + .getOne(); + const workerAgeMs = heartbeat + ? Date.now() - heartbeat.heartbeatAt.getTime() + : null; + const workerReady = + workerAgeMs !== null && workerAgeMs <= config.workerStaleMs; + const stats = await queueStats(); + const ready = workerReady && !migrationsPending; + return res.status(ready ? 200 : 503).json({ + status: ready ? "ready" : "not-ready", + database: "ok", + migrations: migrationsPending ? "pending" : "current", + worker: workerReady ? "ok" : "stale", + workerAgeMs, + queue: stats, + }); + } catch (error) { + return res.status(503).json({ + status: "not-ready", + database: "unavailable", + error: error instanceof Error ? error.message : String(error), + }); + } + }); + + router.get("/metrics", async (_req, res) => { + try { + const stats = await queueStats(); + const heartbeat = await AppDataSource.getRepository(WorkerHeartbeat) + .createQueryBuilder("h") + .orderBy("h.heartbeatAt", "DESC") + .getOne(); + const heartbeatAge = heartbeat + ? Math.max(0, (Date.now() - heartbeat.heartbeatAt.getTime()) / 1000) + : -1; + const lines = [ + "# HELP aaas_deliveries Delivery rows by queue state.", + "# TYPE aaas_deliveries gauge", + `aaas_deliveries{status="pending"} ${stats.pending}`, + `aaas_deliveries{status="failed"} ${stats.failed}`, + `aaas_deliveries{status="delivering"} ${stats.delivering}`, + `aaas_deliveries{status="dead"} ${stats.dead}`, + "# HELP aaas_expired_leases Deliveries whose worker lease expired.", + "# TYPE aaas_expired_leases gauge", + `aaas_expired_leases ${stats.expired_leases}`, + "# HELP aaas_oldest_pending_seconds Age of the oldest active delivery.", + "# TYPE aaas_oldest_pending_seconds gauge", + `aaas_oldest_pending_seconds ${stats.oldest_pending_seconds ?? 0}`, + "# HELP aaas_worker_heartbeat_age_seconds Age of the newest worker heartbeat; -1 means absent.", + "# TYPE aaas_worker_heartbeat_age_seconds gauge", + `aaas_worker_heartbeat_age_seconds ${heartbeatAge}`, + "", + ]; + res.type("text/plain; version=0.0.4").send(lines.join("\n")); + } catch (error) { + res.status(503).type("text/plain").send( + `# metrics unavailable: ${error instanceof Error ? error.message : String(error)}\n`, + ); + } + }); + + return router; +} diff --git a/services/awareness-service/api/src/database/data-source.ts b/services/awareness-service/api/src/database/data-source.ts index 2e184c1b0..3489449f9 100644 --- a/services/awareness-service/api/src/database/data-source.ts +++ b/services/awareness-service/api/src/database/data-source.ts @@ -9,6 +9,8 @@ import { DeadLetter } from "./entities/DeadLetter"; import { Delivery } from "./entities/Delivery"; import { Packet } from "./entities/Packet"; import { Subscription } from "./entities/Subscription"; +import { AwarenessEvent } from "./entities/AwarenessEvent"; +import { WorkerHeartbeat } from "./entities/WorkerHeartbeat"; export const AppDataSource = new DataSource({ type: "postgres", @@ -23,6 +25,8 @@ export const AppDataSource = new DataSource({ Subscription, Delivery, DeadLetter, + AwarenessEvent, + WorkerHeartbeat, ], migrations: [path.join(__dirname, "migrations", "*.{ts,js}")], ssl: config.dbCaCert @@ -33,5 +37,8 @@ export const AppDataSource = new DataSource({ min: 2, idleTimeoutMillis: 30000, connectionTimeoutMillis: 5000, + statement_timeout: config.dbStatementTimeoutMs, + query_timeout: config.dbQueryTimeoutMs, + lock_timeout: config.dbLockTimeoutMs, }, }); diff --git a/services/awareness-service/api/src/database/entities/AwarenessEvent.ts b/services/awareness-service/api/src/database/entities/AwarenessEvent.ts new file mode 100644 index 000000000..ee5db4843 --- /dev/null +++ b/services/awareness-service/api/src/database/entities/AwarenessEvent.ts @@ -0,0 +1,47 @@ +import { Column, Entity, Index, PrimaryColumn } from "typeorm"; +import type { PacketOperation } from "./Packet"; + +/** + * Immutable awareness history. Packet remains the latest-state projection for + * backwards-compatible point lookups; this table is the durable event log. + */ +@Entity("awareness_events") +@Index("idx_awareness_events_received_event", ["receivedAt", "eventId"]) +@Index("idx_awareness_events_envelope_version", ["packetId", "streamVersion"]) +@Index("idx_awareness_events_ontology_received", ["ontology", "receivedAt"]) +export class AwarenessEvent { + @PrimaryColumn({ type: "varchar" }) + eventId!: string; + + /** MetaEnvelope id from the wire payload. */ + @Index("idx_awareness_events_packet") + @Column({ type: "varchar" }) + packetId!: string; + + @Column({ type: "varchar" }) + ontology!: string; + + @Column({ type: "varchar", nullable: true }) + evaultPublicKey!: string | null; + + @Column({ type: "varchar", nullable: true }) + w3id!: string | null; + + @Column({ type: "jsonb", nullable: true }) + data!: any; + + @Column({ type: "varchar", default: "create" }) + operation!: PacketOperation; + + @Column({ type: "bigint", nullable: true }) + streamVersion!: string | null; + + @Column({ type: "varchar", nullable: true }) + requestingPlatform!: string | null; + + @Column({ type: "timestamptz" }) + occurredAt!: Date; + + @Column({ type: "timestamptz", default: () => "now()" }) + receivedAt!: Date; +} diff --git a/services/awareness-service/api/src/database/entities/DeadLetter.ts b/services/awareness-service/api/src/database/entities/DeadLetter.ts index db8d3d77d..3caa332c2 100644 --- a/services/awareness-service/api/src/database/entities/DeadLetter.ts +++ b/services/awareness-service/api/src/database/entities/DeadLetter.ts @@ -15,6 +15,7 @@ export class DeadLetter { @PrimaryGeneratedColumn("uuid") id!: string; + @Index("uq_dead_letters_delivery", { unique: true }) @Column({ type: "uuid" }) deliveryId!: string; diff --git a/services/awareness-service/api/src/database/entities/Delivery.ts b/services/awareness-service/api/src/database/entities/Delivery.ts index a78edb9e9..7a96e0212 100644 --- a/services/awareness-service/api/src/database/entities/Delivery.ts +++ b/services/awareness-service/api/src/database/entities/Delivery.ts @@ -4,7 +4,6 @@ import { Entity, Index, PrimaryGeneratedColumn, - Unique, } from "typeorm"; export type DeliveryStatus = @@ -15,19 +14,22 @@ export type DeliveryStatus = | "dead"; /** - * A queued webhook delivery of one packet to one subscription. The unique - * (subscriptionId, packetId, contentHash) constraint dedupes by content: a - * retried POST of the same payload is idempotent, while re-ingesting an updated - * envelope (new contentHash) queues a fresh delivery. + * A queued webhook delivery of one immutable event to one subscription. */ @Entity("deliveries") -@Unique("uq_delivery_subscription_packet_content", [ - "subscriptionId", - "packetId", - "contentHash", -]) +@Index("uq_delivery_subscription_event", ["subscriptionId", "eventId"], { + unique: true, +}) // Serves the consumer dashboard's newest-first delivery list. @Index("idx_deliveries_subscription_created", ["subscriptionId", "createdAt"]) +@Index( + "idx_deliveries_active_stream_order", + ["subscriptionId", "packetId", "createdAt", "id"], + { where: `"status" IN ('pending', 'failed', 'delivering')` }, +) +@Index("idx_deliveries_claim_due", ["nextAttemptAt", "createdAt", "id"], { + where: `"status" IN ('pending', 'failed')`, +}) export class Delivery { @PrimaryGeneratedColumn("uuid") id!: string; @@ -39,7 +41,12 @@ export class Delivery { @Column({ type: "varchar" }) packetId!: string; - /** SHA-256 of the packet payload at ingest; dedupes deliveries by content. */ + /** Immutable source event id. Legacy rows are backfilled during migration. */ + @Index("idx_deliveries_event") + @Column({ type: "varchar" }) + eventId!: string; + + /** SHA-256 audit fingerprint of the exact payload at ingest. */ @Column({ type: "varchar" }) contentHash!: string; @@ -69,4 +76,19 @@ export class Delivery { @Column({ type: "timestamptz", nullable: true }) deliveredAt!: Date | null; + + /** Start of the current automatic retry window (reset by admin replay). */ + @Column({ type: "timestamptz", default: () => "now()" }) + retryStartedAt!: Date; + + /** Token-fenced lease; stale workers cannot complete a reclaimed row. */ + @Column({ type: "varchar", nullable: true }) + leaseOwner!: string | null; + + @Column({ type: "uuid", nullable: true }) + leaseToken!: string | null; + + @Index("idx_deliveries_lease_expires") + @Column({ type: "timestamptz", nullable: true }) + leaseExpiresAt!: Date | null; } diff --git a/services/awareness-service/api/src/database/entities/Packet.ts b/services/awareness-service/api/src/database/entities/Packet.ts index 04e9561d0..d35bcf996 100644 --- a/services/awareness-service/api/src/database/entities/Packet.ts +++ b/services/awareness-service/api/src/database/entities/Packet.ts @@ -9,8 +9,8 @@ import { export type PacketOperation = "create" | "update" | "delete"; /** - * A single awareness packet ingested from an eVault. `id` is the MetaEnvelope - * id supplied by evault-core, so re-ingestion of the same envelope upserts. + * Latest-state projection for one MetaEnvelope. Immutable event history lives + * in AwarenessEvent; `id` is the source MetaEnvelope id and therefore upserts. */ @Entity("packets") @Index("idx_packets_ontology_received", ["ontology", "receivedAt"]) diff --git a/services/awareness-service/api/src/database/entities/WorkerHeartbeat.ts b/services/awareness-service/api/src/database/entities/WorkerHeartbeat.ts new file mode 100644 index 000000000..b9b1403a3 --- /dev/null +++ b/services/awareness-service/api/src/database/entities/WorkerHeartbeat.ts @@ -0,0 +1,20 @@ +import { Column, Entity, PrimaryColumn } from "typeorm"; + +/** Cross-process worker health, read by the API readiness endpoint. */ +@Entity("worker_heartbeats") +export class WorkerHeartbeat { + @PrimaryColumn({ type: "varchar" }) + workerId!: string; + + @Column({ type: "timestamptz" }) + heartbeatAt!: Date; + + @Column({ type: "timestamptz", nullable: true }) + tickStartedAt!: Date | null; + + @Column({ type: "timestamptz", nullable: true }) + lastCompletedAt!: Date | null; + + @Column({ type: "text", nullable: true }) + lastError!: string | null; +} diff --git a/services/awareness-service/api/src/database/migrations/1715200000000-Init.ts b/services/awareness-service/api/src/database/migrations/1715200000000-Init.ts index 098471c09..3141e8760 100644 --- a/services/awareness-service/api/src/database/migrations/1715200000000-Init.ts +++ b/services/awareness-service/api/src/database/migrations/1715200000000-Init.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import type { MigrationInterface, QueryRunner } from "typeorm"; /** * Initial schema for Awareness as a Service: awareness packets, consumers and diff --git a/services/awareness-service/api/src/database/migrations/1780000000000-AddDeliveryContentHash.ts b/services/awareness-service/api/src/database/migrations/1780000000000-AddDeliveryContentHash.ts deleted file mode 100644 index 6fea868a1..000000000 --- a/services/awareness-service/api/src/database/migrations/1780000000000-AddDeliveryContentHash.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; - -/** - * Dedupe deliveries by payload content, not packet id alone. Adds a - * `contentHash` column to `deliveries` and swaps the unique key from - * (subscriptionId, packetId) to (subscriptionId, packetId, contentHash) so that - * re-ingesting an updated MetaEnvelope queues a fresh delivery instead of being - * silently dropped by the old constraint. - */ -export class AddDeliveryContentHash1780000000000 - implements MigrationInterface -{ - name = "AddDeliveryContentHash1780000000000"; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "deliveries" ADD COLUMN "contentHash" varchar`, - ); - // Backfill legacy rows with a single constant: this preserves the old - // one-delivery-per-(subscription, packet) semantics for rows that - // predate content-based dedup. - await queryRunner.query( - `UPDATE "deliveries" SET "contentHash" = '' WHERE "contentHash" IS NULL`, - ); - await queryRunner.query( - `ALTER TABLE "deliveries" ALTER COLUMN "contentHash" SET NOT NULL`, - ); - await queryRunner.query( - `ALTER TABLE "deliveries" DROP CONSTRAINT "uq_delivery_subscription_packet"`, - ); - await queryRunner.query( - `ALTER TABLE "deliveries" ADD CONSTRAINT "uq_delivery_subscription_packet_content" UNIQUE ("subscriptionId", "packetId", "contentHash")`, - ); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "deliveries" DROP CONSTRAINT "uq_delivery_subscription_packet_content"`, - ); - await queryRunner.query( - `ALTER TABLE "deliveries" ADD CONSTRAINT "uq_delivery_subscription_packet" UNIQUE ("subscriptionId", "packetId")`, - ); - await queryRunner.query( - `ALTER TABLE "deliveries" DROP COLUMN "contentHash"`, - ); - } -} diff --git a/services/awareness-service/api/src/database/migrations/1780404367748-AddDeliveryContentHash.ts b/services/awareness-service/api/src/database/migrations/1780404367748-AddDeliveryContentHash.ts index aa198a9e2..692a5aa00 100644 --- a/services/awareness-service/api/src/database/migrations/1780404367748-AddDeliveryContentHash.ts +++ b/services/awareness-service/api/src/database/migrations/1780404367748-AddDeliveryContentHash.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import type { MigrationInterface, QueryRunner } from "typeorm"; /** * Dedupe deliveries by payload content, not packet id alone. Adds a @@ -14,7 +14,7 @@ export class AddDeliveryContentHash1780404367748 implements MigrationInterface { // Add nullable first, then backfill: a straight `ADD ... NOT NULL` // fails on any table that already has delivery rows. await queryRunner.query( - `ALTER TABLE "deliveries" ADD "contentHash" character varying`, + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "contentHash" character varying`, ); // Backfill legacy rows with a single constant. This preserves the old // one-delivery-per-(subscription, packet) semantics for rows that @@ -26,10 +26,13 @@ export class AddDeliveryContentHash1780404367748 implements MigrationInterface { `ALTER TABLE "deliveries" ALTER COLUMN "contentHash" SET NOT NULL`, ); await queryRunner.query( - `ALTER TABLE "deliveries" DROP CONSTRAINT "uq_delivery_subscription_packet"`, + `ALTER TABLE "deliveries" DROP CONSTRAINT IF EXISTS "uq_delivery_subscription_packet"`, ); await queryRunner.query( - `ALTER TABLE "deliveries" ADD CONSTRAINT "uq_delivery_subscription_packet_content" UNIQUE ("subscriptionId", "packetId", "contentHash")`, + `DO $$ BEGIN + ALTER TABLE "deliveries" ADD CONSTRAINT "uq_delivery_subscription_packet_content" UNIQUE ("subscriptionId", "packetId", "contentHash"); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$`, ); } diff --git a/services/awareness-service/api/src/database/migrations/1780404367749-AddDeliveryPayload.ts b/services/awareness-service/api/src/database/migrations/1780404367749-AddDeliveryPayload.ts index 171eb0f26..68bcdf25d 100644 --- a/services/awareness-service/api/src/database/migrations/1780404367749-AddDeliveryPayload.ts +++ b/services/awareness-service/api/src/database/migrations/1780404367749-AddDeliveryPayload.ts @@ -1,13 +1,11 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import type { MigrationInterface, QueryRunner } from "typeorm"; /** Preserve the exact event body on each queued delivery. */ export class AddDeliveryPayload1780404367749 implements MigrationInterface { name = "AddDeliveryPayload1780404367749"; public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "deliveries" ADD "payload" jsonb`, - ); + await queryRunner.query(`ALTER TABLE "deliveries" ADD "payload" jsonb`); } public async down(queryRunner: QueryRunner): Promise { diff --git a/services/awareness-service/api/src/database/migrations/1786492800000-AddDeliveriesSubscriptionCreatedIndex.ts b/services/awareness-service/api/src/database/migrations/1786492800000-AddDeliveriesSubscriptionCreatedIndex.ts index e458a8ca6..a2dd0a3a4 100644 --- a/services/awareness-service/api/src/database/migrations/1786492800000-AddDeliveriesSubscriptionCreatedIndex.ts +++ b/services/awareness-service/api/src/database/migrations/1786492800000-AddDeliveriesSubscriptionCreatedIndex.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import type { MigrationInterface, QueryRunner } from "typeorm"; /** * Support `/api/me/deliveries`, which lists a consumer's most recent deliveries diff --git a/services/awareness-service/api/src/database/migrations/1789430400000-DurableAwarenessEvents.ts b/services/awareness-service/api/src/database/migrations/1789430400000-DurableAwarenessEvents.ts new file mode 100644 index 000000000..bd0b52706 --- /dev/null +++ b/services/awareness-service/api/src/database/migrations/1789430400000-DurableAwarenessEvents.ts @@ -0,0 +1,178 @@ +import type { MigrationInterface, QueryRunner } from "typeorm"; + +/** Immutable events, token-fenced delivery leases and cross-process health. */ +export class DurableAwarenessEvents1789430400000 implements MigrationInterface { + name = "DurableAwarenessEvents1789430400000"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "awareness_events" ( + "eventId" varchar NOT NULL, + "packetId" varchar NOT NULL, + "ontology" varchar NOT NULL, + "evaultPublicKey" varchar, + "w3id" varchar, + "data" jsonb, + "operation" varchar NOT NULL DEFAULT 'create', + "streamVersion" bigint, + "requestingPlatform" varchar, + "occurredAt" timestamptz NOT NULL, + "receivedAt" timestamptz NOT NULL DEFAULT now(), + CONSTRAINT "PK_awareness_events" PRIMARY KEY ("eventId") + ) + `); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_awareness_events_received_event" ON "awareness_events" ("receivedAt", "eventId")`, + ); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_awareness_events_envelope_version" ON "awareness_events" ("packetId", "streamVersion")`, + ); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_awareness_events_ontology_received" ON "awareness_events" ("ontology", "receivedAt")`, + ); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_awareness_events_packet" ON "awareness_events" ("packetId")`, + ); + await queryRunner.query(` + INSERT INTO "awareness_events" ( + "eventId", "packetId", "ontology", "evaultPublicKey", "w3id", + "data", "operation", "occurredAt", "receivedAt" + ) + SELECT + 'legacy-packet:' || p.id, + p.id, p.ontology, p."evaultPublicKey", p.w3id, + p.data, p.operation, p."receivedAt", p."receivedAt" + FROM packets p + ON CONFLICT ("eventId") DO NOTHING + `); + + await queryRunner.query( + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "eventId" varchar`, + ); + // A legacy delivery snapshot is the best recoverable representation of + // the pre-event-id history. The delivery id makes every row distinct; + // new producers supply stable ids and are deduplicated correctly. + await queryRunner.query( + `UPDATE "deliveries" SET "eventId" = 'legacy:' || "id"::text WHERE "eventId" IS NULL`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ALTER COLUMN "eventId" SET NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "leaseOwner" varchar`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "leaseToken" uuid`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "leaseExpiresAt" timestamptz`, + ); + // Rows claimed by the old in-process latch have no lease and would be + // unrecoverable under lease semantics unless explicitly released. + await queryRunner.query(` + UPDATE "deliveries" + SET status = 'failed', + "nextAttemptAt" = now(), + "lastError" = coalesce("lastError", 'released by lease migration') + WHERE status = 'delivering' AND "leaseExpiresAt" IS NULL + `); + await queryRunner.query( + `ALTER TABLE "deliveries" ADD COLUMN IF NOT EXISTS "retryStartedAt" timestamptz`, + ); + await queryRunner.query( + `UPDATE "deliveries" SET "retryStartedAt" = "createdAt" WHERE "retryStartedAt" IS NULL`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ALTER COLUMN "retryStartedAt" SET DEFAULT now()`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ALTER COLUMN "retryStartedAt" SET NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP CONSTRAINT IF EXISTS "uq_delivery_subscription_packet_content"`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX IF NOT EXISTS "uq_delivery_subscription_event" ON "deliveries" ("subscriptionId", "eventId")`, + ); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_deliveries_event" ON "deliveries" ("eventId")`, + ); + await queryRunner.query( + `CREATE INDEX IF NOT EXISTS "idx_deliveries_lease_expires" ON "deliveries" ("leaseExpiresAt")`, + ); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "idx_deliveries_active_stream_order" + ON "deliveries" ("subscriptionId", "packetId", "createdAt", "id") + WHERE status IN ('pending', 'failed', 'delivering') + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "idx_deliveries_claim_due" + ON "deliveries" ("nextAttemptAt", "createdAt", "id") + WHERE status IN ('pending', 'failed') + `); + + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "worker_heartbeats" ( + "workerId" varchar NOT NULL, + "heartbeatAt" timestamptz NOT NULL, + "tickStartedAt" timestamptz, + "lastCompletedAt" timestamptz, + "lastError" text, + CONSTRAINT "PK_worker_heartbeats" PRIMARY KEY ("workerId") + ) + `); + + // Guarantee that repeated failure handling creates only one dead letter. + await queryRunner.query(` + DELETE FROM "dead_letters" a + USING "dead_letters" b + WHERE a."deliveryId" = b."deliveryId" + AND ( + a."createdAt" > b."createdAt" + OR (a."createdAt" = b."createdAt" AND a.id > b.id) + ) + `); + await queryRunner.query( + `CREATE UNIQUE INDEX IF NOT EXISTS "uq_dead_letters_delivery" ON "dead_letters" ("deliveryId")`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX IF EXISTS "uq_dead_letters_delivery"`, + ); + await queryRunner.query(`DROP TABLE IF EXISTS "worker_heartbeats"`); + await queryRunner.query( + `DROP INDEX IF EXISTS "idx_deliveries_lease_expires"`, + ); + await queryRunner.query( + `DROP INDEX IF EXISTS "idx_deliveries_active_stream_order"`, + ); + await queryRunner.query( + `DROP INDEX IF EXISTS "idx_deliveries_claim_due"`, + ); + await queryRunner.query(`DROP INDEX IF EXISTS "idx_deliveries_event"`); + await queryRunner.query( + `DROP INDEX IF EXISTS "uq_delivery_subscription_event"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP COLUMN IF EXISTS "leaseExpiresAt"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP COLUMN IF EXISTS "leaseToken"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP COLUMN IF EXISTS "leaseOwner"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP COLUMN IF EXISTS "retryStartedAt"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" DROP COLUMN IF EXISTS "eventId"`, + ); + await queryRunner.query( + `ALTER TABLE "deliveries" ADD CONSTRAINT "uq_delivery_subscription_packet_content" UNIQUE ("subscriptionId", "packetId", "contentHash")`, + ); + await queryRunner.query(`DROP TABLE IF EXISTS "awareness_events"`); + } +} diff --git a/services/awareness-service/api/src/index.ts b/services/awareness-service/api/src/index.ts index 53cdde070..ee293440d 100644 --- a/services/awareness-service/api/src/index.ts +++ b/services/awareness-service/api/src/index.ts @@ -1,9 +1,9 @@ import "reflect-metadata"; +import type { Server } from "node:http"; import { apiReference } from "@scalar/express-api-reference"; import cors from "cors"; import express from "express"; import { config } from "./config"; -import { openApiDocument } from "./openapi"; import { adminRouter } from "./controllers/AdminController"; import { applicationRouter } from "./controllers/ApplicationController"; import { authRouter } from "./controllers/AuthController"; @@ -11,21 +11,25 @@ import { consumerRouter } from "./controllers/ConsumerController"; import { ingestRouter } from "./controllers/IngestController"; import { queryRouter } from "./controllers/QueryController"; import { subscriptionRouter } from "./controllers/SubscriptionController"; +import { systemRouter } from "./controllers/SystemController"; import { AppDataSource } from "./database/data-source"; +import { openApiDocument } from "./openapi"; import { DeliveryEngine } from "./services/DeliveryEngine"; import { SeedService } from "./services/SeedService"; async function start(): Promise { await AppDataSource.initialize(); console.log("[aaas] database connected"); + if (await AppDataSource.showMigrations()) { + throw new Error( + "Pending AaaS database migrations; run `pnpm --filter awareness-service-api migration:run` before starting", + ); + } const app = express(); app.use(cors()); app.use(express.json({ limit: "5mb" })); - - app.get("/health", (_req, res) => { - res.json({ status: "ok", service: "awareness-service" }); - }); + app.use(systemRouter()); // Raw OpenAPI document + interactive Scalar API reference at /docs. app.get("/openapi.json", (_req, res) => { @@ -54,12 +58,23 @@ async function start(): Promise { await seedService.syncCatchAll(); seedService.start(); + const server: Server = app.listen(config.apiPort, () => { + console.log(`[aaas] API listening on :${config.apiPort}`); + }); + + // AaaS intentionally runs API and delivery in one deployable process. const deliveryEngine = new DeliveryEngine(); deliveryEngine.start(); - app.listen(config.apiPort, () => { - console.log(`[aaas] API listening on :${config.apiPort}`); - }); + const shutdown = async (signal: string) => { + console.log(`[aaas] ${signal} received, shutting down`); + seedService.stop(); + await deliveryEngine.stop(); + await new Promise((resolve) => server.close(() => resolve())); + await AppDataSource.destroy(); + }; + process.once("SIGTERM", () => void shutdown("SIGTERM")); + process.once("SIGINT", () => void shutdown("SIGINT")); } start().catch((err) => { diff --git a/services/awareness-service/api/src/openapi.ts b/services/awareness-service/api/src/openapi.ts index ada41a1d1..3e10b0a97 100644 --- a/services/awareness-service/api/src/openapi.ts +++ b/services/awareness-service/api/src/openapi.ts @@ -12,7 +12,7 @@ export const openApiDocument = { openapi: "3.1.0", info: { title: "Awareness as a Service API", - version: "1.0.0", + version: "1.1.0", description: "Consume MetaEnvelope awareness packets: poll the packet history " + "by ontology, eVault and time range, and register webhook " + @@ -47,6 +47,10 @@ export const openApiDocument = { type: "object", description: "A stored awareness packet.", properties: { + eventId: { + type: "string", + description: "Stable idempotency key for this event", + }, id: { type: "string", description: "MetaEnvelope id" }, ontology: { type: "string" }, evaultPublicKey: { type: "string", nullable: true }, @@ -55,8 +59,17 @@ export const openApiDocument = { nullable: true, description: "Owner's W3ID (eName)", }, - data: { type: "object", nullable: true, additionalProperties: true }, - operation: { type: "string", enum: ["create", "update", "delete"] }, + data: { + type: "object", + nullable: true, + additionalProperties: true, + }, + operation: { + type: "string", + enum: ["create", "update", "delete"], + }, + streamVersion: { type: "integer", nullable: true }, + occurredAt: { type: "string", format: "date-time" }, receivedAt: { type: "string", format: "date-time" }, }, }, @@ -90,7 +103,10 @@ export const openApiDocument = { description: "Defaults to `/api/webhook`", }, - ontologyFilter: { type: "array", items: { type: "string" } }, + ontologyFilter: { + type: "array", + items: { type: "string" }, + }, evaultFilter: { type: "array", items: { type: "string" } }, secret: { type: "string", @@ -105,6 +121,10 @@ export const openApiDocument = { id: { type: "string", format: "uuid" }, subscriptionId: { type: "string", format: "uuid" }, packetId: { type: "string" }, + eventId: { + type: "string", + description: "Stable source event id", + }, status: { type: "string", enum: [ @@ -119,7 +139,11 @@ export const openApiDocument = { nextAttemptAt: { type: "string", format: "date-time" }, lastError: { type: "string", nullable: true }, lastResponseStatus: { type: "integer", nullable: true }, - deliveredAt: { type: "string", format: "date-time", nullable: true }, + deliveredAt: { + type: "string", + format: "date-time", + nullable: true, + }, }, }, Consumer: { @@ -160,6 +184,25 @@ export const openApiDocument = { }, }, }, + "/ready": { + get: { + tags: ["System"], + summary: "Database, migration and delivery-worker readiness", + responses: { + "200": { description: "Service is ready" }, + "503": { + description: "A dependency or worker is unhealthy", + }, + }, + }, + }, + "/metrics": { + get: { + tags: ["System"], + summary: "Prometheus queue and worker metrics", + responses: { "200": { description: "Prometheus text format" } }, + }, + }, "/api/packets": { get: { tags: ["Query"], @@ -213,7 +256,9 @@ export const openApiDocument = { properties: { packets: { type: "array", - items: { $ref: "#/components/schemas/Packet" }, + items: { + $ref: "#/components/schemas/Packet", + }, }, count: { type: "integer", @@ -234,7 +279,10 @@ export const openApiDocument = { "ceil(total / pageSize)", }, hasMore: { type: "boolean" }, - nextCursor: { type: "string", nullable: true }, + nextCursor: { + type: "string", + nullable: true, + }, }, }, }, @@ -250,7 +298,8 @@ export const openApiDocument = { get: { tags: ["Query"], summary: "Get a single awareness packet (MetaEnvelope) by ID", - description: "Fetch one awareness packet by its MetaEnvelope id.", + description: + "Fetch one awareness packet by its MetaEnvelope id.", security: [{ consumerAuth: [] }], parameters: [ { @@ -318,7 +367,9 @@ export const openApiDocument = { required: true, content: { "application/json": { - schema: { $ref: "#/components/schemas/SubscriptionInput" }, + schema: { + $ref: "#/components/schemas/SubscriptionInput", + }, }, }, }, @@ -361,10 +412,14 @@ export const openApiDocument = { "application/json": { schema: { allOf: [ - { $ref: "#/components/schemas/SubscriptionInput" }, + { + $ref: "#/components/schemas/SubscriptionInput", + }, { type: "object", - properties: { active: { type: "boolean" } }, + properties: { + active: { type: "boolean" }, + }, }, ], }, @@ -419,7 +474,9 @@ export const openApiDocument = { description: "Consumer profile", content: { "application/json": { - schema: { $ref: "#/components/schemas/Consumer" }, + schema: { + $ref: "#/components/schemas/Consumer", + }, }, }, }, @@ -446,8 +503,12 @@ export const openApiDocument = { type: "object", properties: { id: { type: "string" }, - keyPrefix: { type: "string" }, - revoked: { type: "boolean" }, + keyPrefix: { + type: "string", + }, + revoked: { + type: "boolean", + }, }, }, }, @@ -477,7 +538,8 @@ export const openApiDocument = { keyPrefix: { type: "string" }, apiKey: { type: "string", - description: "Plaintext, shown once", + description: + "Plaintext, shown once", }, }, }, @@ -520,7 +582,8 @@ export const openApiDocument = { ], responses: { "200": { - description: "Recent deliveries across your subscriptions", + description: + "Recent deliveries across your subscriptions", content: { "application/json": { schema: { @@ -528,7 +591,9 @@ export const openApiDocument = { properties: { deliveries: { type: "array", - items: { $ref: "#/components/schemas/Delivery" }, + items: { + $ref: "#/components/schemas/Delivery", + }, }, }, }, diff --git a/services/awareness-service/api/src/scripts/backfill-neo4j.ts b/services/awareness-service/api/src/scripts/backfill-neo4j.ts index 43e81b0cd..b955a8533 100644 --- a/services/awareness-service/api/src/scripts/backfill-neo4j.ts +++ b/services/awareness-service/api/src/scripts/backfill-neo4j.ts @@ -1,14 +1,16 @@ import "reflect-metadata"; import neo4j from "neo4j-driver"; import { AppDataSource } from "../database/data-source"; +import { AwarenessEvent } from "../database/entities/AwarenessEvent"; import { Packet } from "../database/entities/Packet"; /** * One-time backfill. AaaS runs on the same physical node as evault-core's Neo4j, * so this script reads MetaEnvelopes straight from the graph and seeds the - * `packets` table. It is idempotent (upsert keyed on packet id) and re-runnable. + * latest-state `packets` projection and immutable event history. It is + * idempotent (stable event id and packet upsert) and re-runnable. * - * It seeds the packet store ONLY - it deliberately does not create deliveries, + * It deliberately does not create deliveries, * which would spam subscribers with the entire history on go-live. */ @@ -43,7 +45,6 @@ async function main(): Promise { const driver = neo4j.driver(uri, neo4j.auth.basic(user, password)); await AppDataSource.initialize(); - const packetRepo = AppDataSource.getRepository(Packet); const backfillTs = new Date(); let skip = 0; @@ -55,9 +56,11 @@ async function main(): Promise { let rows: any[]; try { const result = await session.run( - `MATCH (m:MetaEnvelope)-[:LINKS_TO]->(e:Envelope) + `MATCH (m:MetaEnvelope) + OPTIONAL MATCH (m)-[:LINKS_TO]->(e:Envelope) RETURN m.id AS id, m.ontology AS ontology, m.eName AS eName, - collect({ontology: e.ontology, value: e.value, valueType: e.valueType}) AS envelopes + collect(CASE WHEN e IS NULL THEN null ELSE {ontology: e.ontology, value: e.value, valueType: e.valueType} END) AS envelopes + ORDER BY id, eName SKIP $skip LIMIT $batch`, { skip: neo4j.int(skip), batch: neo4j.int(BATCH) }, ); @@ -73,7 +76,7 @@ async function main(): Promise { if (rows.length === 0) break; - const packets = rows + const snapshots = rows .filter((row) => row.id && row.ontology) .map((row) => { const data: Record = {}; @@ -85,15 +88,7 @@ async function main(): Promise { ); } } - return packetRepo.create({ - id: row.id, - ontology: row.ontology, - w3id: row.eName ?? null, - evaultPublicKey, - data, - operation: "create" as const, - receivedAt: backfillTs, - }); + return { row, data }; }); // The graph can hold several MetaEnvelope nodes with the same id @@ -101,11 +96,50 @@ async function main(): Promise { // touches the same conflict target twice in one statement, so // collapse duplicates within the batch first (last write wins). const deduped = Array.from( - new Map(packets.map((p) => [p.id, p])).values(), + new Map( + snapshots.map((snapshot) => [snapshot.row.id, snapshot]), + ).values(), ); if (deduped.length > 0) { - await packetRepo.upsert(deduped, ["id"]); + await AppDataSource.transaction(async (manager) => { + const packetRepo = manager.getRepository(Packet); + await packetRepo.upsert( + deduped.map(({ row, data }) => + packetRepo.create({ + id: row.id, + ontology: row.ontology, + w3id: row.eName ?? null, + evaultPublicKey, + data: data as any, + operation: "create" as const, + receivedAt: backfillTs, + }), + ), + ["id"], + ); + await manager + .getRepository(AwarenessEvent) + .createQueryBuilder() + .insert() + .values( + deduped.map(({ row, data }) => ({ + eventId: `legacy-packet:${row.id}`, + packetId: row.id, + ontology: row.ontology, + w3id: row.eName ?? null, + evaultPublicKey, + data: data as any, + operation: "create" as const, + streamVersion: null, + requestingPlatform: null, + occurredAt: backfillTs, + receivedAt: backfillTs, + })), + ) + .orIgnore() + .execute(); + }); total += deduped.length; } console.log(`[backfill] processed ${total} packets...`); diff --git a/services/awareness-service/api/src/services/DeliveryEngine.ts b/services/awareness-service/api/src/services/DeliveryEngine.ts index 0d79cd7ed..38fe1fa62 100644 --- a/services/awareness-service/api/src/services/DeliveryEngine.ts +++ b/services/awareness-service/api/src/services/DeliveryEngine.ts @@ -5,155 +5,200 @@ import { DeadLetter } from "../database/entities/DeadLetter"; import { Delivery } from "../database/entities/Delivery"; import { Packet } from "../database/entities/Packet"; import { Subscription } from "../database/entities/Subscription"; +import { WorkerHeartbeat } from "../database/entities/WorkerHeartbeat"; import { config } from "../config"; import { nextAttemptAt } from "../utils/backoff"; import type { AwarenessPayload } from "../types"; const BATCH_SIZE = 50; -/** - * True for transient "Postgres is not ready" errors - server restarting, in - * recovery, or unreachable. These resolve on their own once the DB is back. - */ -function isDbUnavailable(err: any): boolean { - const code = err?.code ?? err?.driverError?.code; - return ( - code === "57P03" || // cannot connect now / in recovery - code === "57P01" || // admin shutdown - code === "08006" || // connection failure - code === "08001" || // unable to establish connection - code === "08003" || // connection does not exist - code === "ECONNREFUSED" || - code === "ETIMEDOUT" || - code === "ENOTFOUND" - ); +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function withDeadline(promise: Promise, ms: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`delivery batch exceeded ${ms}ms deadline`)), + ms, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); } /** - * Background worker that drains the deliveries queue. Each tick atomically - * claims a batch of due deliveries (FOR UPDATE SKIP LOCKED so concurrent ticks - * never double-send), POSTs each to its subscription target, and either marks - * it delivered or reschedules it with exponential backoff. Once a delivery - * exhausts AWARENESS_MAX_ATTEMPTS it is written to the dead-letter table. + * Self-scheduling, lease-based delivery worker. No unresolved tick can disable + * future polling: database calls have driver-level deadlines, each batch has a + * hard deadline, and expired token-fenced leases are safe to reclaim. */ export class DeliveryEngine { - private timer?: NodeJS.Timeout; - private running = false; - private dbDown = false; + private stopping = false; + private loop?: Promise; + private watchdog?: NodeJS.Timeout; + private activeTickStartedAt: Date | null = null; + private readonly unsettledTicks = new Map, Date>(); start(): void { - this.timer = setInterval(() => { - void this.tick(); - }, config.deliveryPollMs); + if (this.loop) return; + this.stopping = false; + this.loop = this.runLoop(); + this.watchdog = setInterval(() => { + const starts = [ + ...(this.activeTickStartedAt ? [this.activeTickStartedAt] : []), + ...this.unsettledTicks.values(), + ]; + const oldest = starts.sort( + (left, right) => left.getTime() - right.getTime(), + )[0]; + if ( + oldest && + Date.now() - oldest.getTime() > + config.deliveryBatchTimeoutMs + config.dbQueryTimeoutMs + ) { + console.error( + `[aaas] worker watchdog: work has not returned since ${oldest.toISOString()}`, + ); + process.exit(1); + } + }, config.workerHeartbeatMs); console.log( - `[aaas] delivery engine started (poll ${config.deliveryPollMs}ms)`, + `[aaas] delivery worker ${config.workerId} started (poll ${config.deliveryPollMs}ms, lease ${config.deliveryLeaseMs}ms)`, ); } - stop(): void { - if (this.timer) clearInterval(this.timer); + async stop(): Promise { + this.stopping = true; + if (this.watchdog) clearInterval(this.watchdog); + await this.loop; + this.loop = undefined; } - private async tick(): Promise { - if (this.running) return; // skip overlapping ticks - this.running = true; - try { - const claimed = await this.claimBatch(); - // A slow or malicious subscription must not block unrelated - // platforms in the same batch. Each request has its own timeout; - // drain the claimed batch concurrently so failures are isolated. - const results = await Promise.allSettled( - claimed.map((delivery) => this.attemptDelivery(delivery)), + private async runLoop(): Promise { + while (!this.stopping) { + const tickStartedAt = new Date(); + this.activeTickStartedAt = tickStartedAt; + let lastError: string | null = null; + await this.writeHeartbeat(tickStartedAt, null, null); + const tick = this.tick(); + this.unsettledTicks.set(tick, tickStartedAt); + tick.then( + () => this.unsettledTicks.delete(tick), + () => this.unsettledTicks.delete(tick), ); - const rejected = results - .map((result, index) => ({ result, delivery: claimed[index] })) - .filter( - (item): item is { - result: PromiseRejectedResult; - delivery: Delivery; - } => item.result.status === "rejected", - ); - for (const { result, delivery } of rejected) { - const message = - result.reason instanceof Error - ? result.reason.message - : String(result.reason); - await AppDataSource.getRepository(Delivery).update(delivery.id, { - status: "failed", - lastError: message, - nextAttemptAt: new Date(), - }); - console.error( - `[aaas] delivery ${delivery.id} task failed before completion: ${message}`, - ); + try { + await withDeadline(tick, config.deliveryBatchTimeoutMs); + } catch (error) { + lastError = errorMessage(error); + console.error(`[aaas] delivery tick failed: ${lastError}`); + } finally { + await this.writeHeartbeat(null, new Date(), lastError); + this.activeTickStartedAt = null; } - this.dbDown = false; - } catch (err) { - if (isDbUnavailable(err)) { - // Postgres is restarting / in recovery - transient. Log once - // per outage instead of dumping a stack trace every tick. - if (!this.dbDown) { - this.dbDown = true; - console.warn( - "[aaas] database unavailable, pausing delivery until it recovers", - ); - } - } else { - this.dbDown = false; - console.error("[aaas] delivery tick failed:", err); - } - } finally { - this.running = false; + if (!this.stopping) await delay(config.deliveryPollMs); + } + } + + protected async writeHeartbeat( + tickStartedAt: Date | null, + lastCompletedAt: Date | null, + lastError: string | null, + ): Promise { + try { + await AppDataSource.getRepository(WorkerHeartbeat).upsert( + { + workerId: config.workerId, + heartbeatAt: new Date(), + tickStartedAt, + lastCompletedAt, + lastError, + }, + ["workerId"], + ); + } catch (error) { + console.error( + `[aaas] failed to persist worker heartbeat: ${errorMessage(error)}`, + ); } } - /** Atomically move a batch of due deliveries to `delivering`. */ + protected async tick(): Promise { + const claimed = await this.claimBatch(); + const results = await Promise.allSettled( + claimed.map((delivery) => this.attemptDelivery(delivery)), + ); + for (let index = 0; index < results.length; index += 1) { + const result = results[index]; + if (result.status === "fulfilled") continue; + const delivery = claimed[index]; + const message = `delivery task failed unexpectedly: ${errorMessage(result.reason)}`; + await this.rescheduleUnexpected(delivery, message); + console.error(`[aaas] delivery ${delivery.id}: ${message}`); + } + } + + /** Atomically claim due rows with one token-fenced, expiring batch lease. */ private async claimBatch(): Promise { - // UPDATE ... RETURNING via the query builder so the returned rows are - // exposed as a well-defined `.raw` array. The inner SELECT ... FOR - // UPDATE SKIP LOCKED keeps the claim safe across concurrent ticks. + const leaseToken = crypto.randomUUID(); + const leaseExpiresAt = new Date(Date.now() + config.deliveryLeaseMs); const result = await AppDataSource.getRepository(Delivery) .createQueryBuilder() .update(Delivery) - .set({ status: "delivering" }) + .set({ + status: "delivering", + leaseOwner: config.workerId, + leaseToken, + leaseExpiresAt, + }) .where( - // Only pending/failed deliveries still under the attempt - // limit are claimable. `dead` deliveries (and any that - // already hit the cap) are terminal and never re-claimed. `id IN ( - SELECT id FROM deliveries + SELECT d.id FROM deliveries d WHERE ( - status IN ('pending', 'failed') - OR ( - status = 'delivering' - AND "nextAttemptAt" <= now() - interval '1 minute' - ) + (d.status IN ('pending', 'failed') AND d."nextAttemptAt" <= now()) + OR (d.status = 'delivering' AND d."leaseExpiresAt" <= now()) + ) + AND NOT EXISTS ( + SELECT 1 FROM deliveries earlier + WHERE earlier."subscriptionId" = d."subscriptionId" + AND earlier."packetId" = d."packetId" + AND earlier.status IN ('pending', 'failed', 'delivering') + AND ( + earlier."createdAt" < d."createdAt" + OR (earlier."createdAt" = d."createdAt" AND earlier.id < d.id) ) - AND attempts < :maxAttempts - AND "nextAttemptAt" <= now() - ORDER BY "nextAttemptAt" + ) + ORDER BY d."nextAttemptAt", d."createdAt", d.id LIMIT :limit FOR UPDATE SKIP LOCKED )`, - { limit: BATCH_SIZE, maxAttempts: config.maxAttempts }, + { limit: BATCH_SIZE }, ) .returning("*") .execute(); - return (result.raw ?? []) as Delivery[]; } private async attemptDelivery(delivery: Delivery): Promise { - if (!delivery?.id) { - console.warn("[aaas] skipping delivery row with no id"); - return; - } const subscription = await AppDataSource.getRepository( Subscription, ).findOne({ where: { id: delivery.subscriptionId } }); - const packet = await AppDataSource.getRepository(Packet).findOne({ - where: { id: delivery.packetId }, - }); + const packet = delivery.payload + ? null + : await AppDataSource.getRepository(Packet).findOne({ + where: { id: delivery.packetId }, + }); if (!subscription || (!packet && !delivery.payload)) { await this.fail( @@ -161,14 +206,13 @@ export class DeliveryEngine { subscription, "subscription or packet no longer exists", null, + (delivery.payload ?? {}) as AwarenessPayload, ); return; } - // New rows carry an immutable snapshot so rapid updates to the same - // envelope cannot overwrite an older event before it is delivered. - // Fall back to Packet for deliveries created before this column existed. const payload: AwarenessPayload = delivery.payload ?? { + eventId: delivery.eventId, id: packet!.id, w3id: packet!.w3id, evaultPublicKey: packet!.evaultPublicKey, @@ -176,14 +220,17 @@ export class DeliveryEngine { schemaId: packet!.ontology, operation: packet!.operation, }; + payload.eventId ??= delivery.eventId; + const serializedPayload = JSON.stringify(payload); const headers: Record = { "Content-Type": "application/json", + "x-aaas-event-id": delivery.eventId, }; if (subscription.secret) { headers["x-aaas-signature"] = crypto .createHmac("sha256", subscription.secret) - .update(JSON.stringify(payload)) + .update(serializedPayload) .digest("hex"); } @@ -192,67 +239,139 @@ export class DeliveryEngine { headers, timeout: 5000, }); - await AppDataSource.getRepository(Delivery).update(delivery.id, { - status: "delivered", - attempts: delivery.attempts + 1, - deliveredAt: new Date(), - lastResponseStatus: res.status, - lastError: null, - }); - } catch (err: any) { - const responseStatus = err?.response?.status ?? null; - const message = - err?.message ?? "unknown webhook delivery failure"; - await this.fail(delivery, subscription, message, responseStatus, { + const update = await AppDataSource.getRepository(Delivery).update( + { id: delivery.id, leaseToken: delivery.leaseToken! }, + { + status: "delivered", + attempts: Number(delivery.attempts) + 1, + deliveredAt: new Date(), + lastResponseStatus: res.status, + lastError: null, + leaseOwner: null, + leaseToken: null, + leaseExpiresAt: null, + }, + ); + if (update.affected) { + await AppDataSource.getRepository(DeadLetter).update( + { deliveryId: delivery.id }, + { resolved: true }, + ); + } + } catch (error: any) { + await this.fail( + delivery, + subscription, + error?.message ?? "unknown webhook delivery failure", + error?.response?.status ?? null, payload, - }); + ); } } + private async rescheduleUnexpected( + delivery: Delivery, + message: string, + ): Promise { + await AppDataSource.getRepository(Delivery).update( + { id: delivery.id, leaseToken: delivery.leaseToken! }, + { + status: "failed", + lastError: message, + nextAttemptAt: new Date(), + leaseOwner: null, + leaseToken: null, + leaseExpiresAt: null, + }, + ); + } + private async fail( delivery: Delivery, subscription: Subscription | null, message: string, responseStatus: number | null, - ctx?: { payload: AwarenessPayload }, + payload: AwarenessPayload, ): Promise { - const attempts = delivery.attempts + 1; - const deliveryRepo = AppDataSource.getRepository(Delivery); + const attempts = Number(delivery.attempts) + 1; + const retryStartedAt = new Date( + delivery.retryStartedAt ?? delivery.createdAt, + ).getTime(); + const deadline = new Date( + retryStartedAt + config.deliveryRetryWindowMs, + ); + const now = new Date(); - if (attempts >= config.maxAttempts) { - // Terminal: mark `dead` so the engine never re-claims it. (Using - // `failed` here let exhausted deliveries be picked up again every - // tick, inflating attempts and spawning duplicate dead letters.) - await deliveryRepo.update(delivery.id, { - status: "dead", - attempts, - lastError: message, - lastResponseStatus: responseStatus, - }); - await AppDataSource.getRepository(DeadLetter).insert({ - deliveryId: delivery.id, - subscriptionId: delivery.subscriptionId, - packetId: delivery.packetId, - consumerId: subscription?.consumerId ?? delivery.subscriptionId, - payload: (ctx?.payload ?? {}) as any, - targetUrl: subscription?.targetUrl ?? "", - totalAttempts: attempts, - lastError: message, - lastResponseStatus: responseStatus, - resolved: false, + if (now >= deadline) { + await AppDataSource.transaction(async (manager) => { + const update = await manager.getRepository(Delivery).update( + { id: delivery.id, leaseToken: delivery.leaseToken! }, + { + status: "dead", + attempts, + lastError: message, + lastResponseStatus: responseStatus, + leaseOwner: null, + leaseToken: null, + leaseExpiresAt: null, + }, + ); + if (!update.affected) return; + await manager + .getRepository(DeadLetter) + .createQueryBuilder() + .insert() + .values({ + deliveryId: delivery.id, + subscriptionId: delivery.subscriptionId, + packetId: delivery.packetId, + consumerId: + subscription?.consumerId ?? delivery.subscriptionId, + payload: payload as any, + targetUrl: subscription?.targetUrl ?? "", + totalAttempts: attempts, + lastError: message, + lastResponseStatus: responseStatus, + resolved: false, + }) + .orIgnore() + .execute(); + await manager.getRepository(DeadLetter).update( + { deliveryId: delivery.id }, + { + subscriptionId: delivery.subscriptionId, + packetId: delivery.packetId, + consumerId: + subscription?.consumerId ?? delivery.subscriptionId, + payload: payload as any, + targetUrl: subscription?.targetUrl ?? "", + totalAttempts: attempts, + lastError: message, + lastResponseStatus: responseStatus, + resolved: false, + }, + ); }); console.warn( - `[aaas] delivery ${delivery.id} dead-lettered after ${attempts} attempts`, + `[aaas] delivery ${delivery.id} dead-lettered after ${config.deliveryRetryWindowMs}ms retry window`, ); return; } - await deliveryRepo.update(delivery.id, { - status: "failed", - attempts, - lastError: message, - lastResponseStatus: responseStatus, - nextAttemptAt: nextAttemptAt(attempts), - }); + const scheduled = nextAttemptAt(attempts); + const next = scheduled > deadline ? deadline : scheduled; + await AppDataSource.getRepository(Delivery).update( + { id: delivery.id, leaseToken: delivery.leaseToken! }, + { + status: "failed", + attempts, + lastError: message, + lastResponseStatus: responseStatus, + nextAttemptAt: next, + leaseOwner: null, + leaseToken: null, + leaseExpiresAt: null, + }, + ); } } diff --git a/services/awareness-service/api/src/services/IngestService.ts b/services/awareness-service/api/src/services/IngestService.ts index aab5a942c..94247fe2c 100644 --- a/services/awareness-service/api/src/services/IngestService.ts +++ b/services/awareness-service/api/src/services/IngestService.ts @@ -1,8 +1,9 @@ import { AppDataSource } from "../database/data-source"; import { Delivery } from "../database/entities/Delivery"; +import { AwarenessEvent } from "../database/entities/AwarenessEvent"; import { Packet } from "../database/entities/Packet"; import type { AwarenessPayload } from "../types"; -import { contentHash } from "../utils/contentHash"; +import { contentHash, stableStringify } from "../utils/contentHash"; import { SubscriptionMatcher } from "./SubscriptionMatcher"; /** Returns the normalised origin of a URL, or null if it cannot be parsed. */ @@ -14,92 +15,196 @@ function safeOrigin(url: string): string | null { } } +function eventFingerprint(input: { + packetId: string; + ontology: string; + evaultPublicKey?: string | null; + w3id?: string | null; + data?: unknown; + operation?: string; + streamVersion?: number | string | null; + requestingPlatform?: string | null; +}): string { + return stableStringify({ + packetId: input.packetId, + ontology: input.ontology, + evaultPublicKey: input.evaultPublicKey ?? null, + w3id: input.w3id ?? null, + data: input.data ?? null, + operation: input.operation ?? "create", + streamVersion: + input.streamVersion === null || input.streamVersion === undefined + ? null + : String(input.streamVersion), + requestingPlatform: input.requestingPlatform ?? null, + }); +} + +export class EventIdConflictError extends Error { + constructor(eventId: string) { + super(`eventId ${eventId} was already used for a different event`); + this.name = "EventIdConflictError"; + } +} + /** * Persists an incoming awareness packet and queues a webhook delivery for every - * subscription that matches it. Re-ingesting the same packet is idempotent: the - * packet is upserted and duplicate deliveries are skipped by a unique - * (subscriptionId, packetId) constraint. + * subscription that matches it. Re-ingesting the same source event is + * idempotent; reusing an event id for different content is rejected. */ export class IngestService { private matcher = new SubscriptionMatcher(); - async ingest( - payload: AwarenessPayload, - ): Promise<{ packetId: string; deliveriesQueued: number }> { - const packetRepo = AppDataSource.getRepository(Packet); - - const packet = packetRepo.create({ + async ingest(payload: AwarenessPayload): Promise<{ + packetId: string; + eventId: string; + duplicate: boolean; + deliveriesQueued: number; + }> { + const deliveryPayload: AwarenessPayload = { + eventId: payload.eventId, id: payload.id, - ontology: payload.schemaId, - evaultPublicKey: payload.evaultPublicKey ?? null, w3id: payload.w3id ?? null, + evaultPublicKey: payload.evaultPublicKey ?? null, data: payload.data ?? null, + schemaId: payload.schemaId, operation: payload.operation ?? "create", - receivedAt: new Date(), - }); + streamVersion: payload.streamVersion ?? null, + occurredAt: payload.occurredAt, + }; + // Legacy callers have no source event id. Preserve their old retry + // idempotency until every eVault has rolled onto the durable outbox. + const eventId = + payload.eventId ?? `legacy:${contentHash(deliveryPayload)}`; + deliveryPayload.eventId = eventId; + const occurredAt = payload.occurredAt + ? new Date(payload.occurredAt) + : new Date(); + if (Number.isNaN(occurredAt.getTime())) { + throw new Error("occurredAt must be an ISO timestamp"); + } - await packetRepo.upsert(packet, ["id"]); + return AppDataSource.transaction(async (manager) => { + const packetRepo = manager.getRepository(Packet); + const packet = packetRepo.create({ + id: payload.id, + ontology: payload.schemaId, + evaultPublicKey: payload.evaultPublicKey ?? null, + w3id: payload.w3id ?? null, + data: payload.data ?? null, + operation: payload.operation ?? "create", + receivedAt: new Date(), + }); - let subscriptions = await this.matcher.match(packet); + const insertEvent = await manager + .getRepository(AwarenessEvent) + .createQueryBuilder() + .insert() + .values({ + eventId, + packetId: payload.id, + ontology: payload.schemaId, + evaultPublicKey: payload.evaultPublicKey ?? null, + w3id: payload.w3id ?? null, + data: (payload.data ?? null) as any, + operation: payload.operation ?? "create", + streamVersion: + payload.streamVersion === null || + payload.streamVersion === undefined + ? null + : String(payload.streamVersion), + requestingPlatform: payload.requestingPlatform ?? null, + occurredAt, + receivedAt: new Date(), + }) + .orIgnore() + .returning('"eventId"') + .execute(); - // Skip delivering the packet back to the platform that triggered it - - // the same ping-pong guard evault-core's old fanout enforced. - if (payload.requestingPlatform) { - const origin = safeOrigin(payload.requestingPlatform); - if (origin) { - subscriptions = subscriptions.filter( - (sub) => safeOrigin(sub.targetUrl) !== origin, - ); + if ((insertEvent.raw ?? []).length === 0) { + const existing = await manager + .getRepository(AwarenessEvent) + .findOneByOrFail({ eventId }); + const incomingFingerprint = eventFingerprint({ + packetId: payload.id, + ontology: payload.schemaId, + evaultPublicKey: payload.evaultPublicKey, + w3id: payload.w3id, + data: payload.data, + operation: payload.operation, + streamVersion: payload.streamVersion, + requestingPlatform: payload.requestingPlatform, + }); + const existingFingerprint = eventFingerprint({ + packetId: existing.packetId, + ontology: existing.ontology, + evaultPublicKey: existing.evaultPublicKey, + w3id: existing.w3id, + data: existing.data, + operation: existing.operation, + streamVersion: existing.streamVersion, + requestingPlatform: existing.requestingPlatform, + }); + if (incomingFingerprint !== existingFingerprint) { + throw new EventIdConflictError(eventId); + } + return { + packetId: payload.id, + eventId, + duplicate: true, + deliveriesQueued: 0, + }; } - } - if (subscriptions.length === 0) { - return { packetId: packet.id, deliveriesQueued: 0 }; - } + await packetRepo.upsert(packet, ["id"]); + let subscriptions = await this.matcher.match(packet, manager); - // Dedupe deliveries by payload content rather than packet id alone: - // re-ingesting an updated envelope (new hash) must queue a new delivery, - // while a retried POST of the same payload (same hash) must not. - // The operation is part of the event identity. A delete can carry the - // same (often null) data as another event and must still be delivered. - const deliveryPayload: AwarenessPayload = { - id: payload.id, - w3id: payload.w3id ?? null, - evaultPublicKey: payload.evaultPublicKey ?? null, - data: payload.data ?? null, - schemaId: payload.schemaId, - operation: payload.operation ?? "create", - }; - const hash = contentHash(deliveryPayload); + if (payload.requestingPlatform) { + const origin = safeOrigin(payload.requestingPlatform); + if (origin) { + subscriptions = subscriptions.filter( + (sub) => safeOrigin(sub.targetUrl) !== origin, + ); + } + } - const deliveryRepo = AppDataSource.getRepository(Delivery); - const rows = subscriptions.map((sub) => - deliveryRepo.create({ - subscriptionId: sub.id, - packetId: packet.id, - contentHash: hash, - payload: deliveryPayload, - status: "pending", - attempts: 0, - nextAttemptAt: new Date(), - }), - ); + if (subscriptions.length === 0) { + return { + packetId: payload.id, + eventId, + duplicate: false, + deliveriesQueued: 0, + }; + } - // orIgnore skips deliveries that already exist for this - // (subscription, packet, content) triple, so an evault-core retry of - // POST /ingest with unchanged data does not double-deliver. An updated - // payload has a different contentHash and is inserted as a new delivery. - const result = await deliveryRepo - .createQueryBuilder() - .insert() - .into(Delivery) - .values(rows) - .orIgnore() - .execute(); + const hash = contentHash(deliveryPayload); + const deliveryRepo = manager.getRepository(Delivery); + const rows = subscriptions.map((sub) => + deliveryRepo.create({ + subscriptionId: sub.id, + packetId: payload.id, + eventId, + contentHash: hash, + payload: deliveryPayload, + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + retryStartedAt: new Date(), + }), + ); + const result = await deliveryRepo + .createQueryBuilder() + .insert() + .values(rows) + .orIgnore() + .execute(); - return { - packetId: packet.id, - deliveriesQueued: result.identifiers.filter(Boolean).length, - }; + return { + packetId: payload.id, + eventId, + duplicate: false, + deliveriesQueued: result.identifiers.filter(Boolean).length, + }; + }); } } diff --git a/services/awareness-service/api/src/services/SeedService.ts b/services/awareness-service/api/src/services/SeedService.ts index 5f12ecba2..94577cc5c 100644 --- a/services/awareness-service/api/src/services/SeedService.ts +++ b/services/awareness-service/api/src/services/SeedService.ts @@ -2,7 +2,20 @@ import axios from "axios"; import { AppDataSource } from "../database/data-source"; import { Consumer } from "../database/entities/Consumer"; import { Subscription } from "../database/entities/Subscription"; +import { AwarenessEvent } from "../database/entities/AwarenessEvent"; +import { Delivery } from "../database/entities/Delivery"; import { config } from "../config"; +import { contentHash } from "../utils/contentHash"; +import type { AwarenessPayload } from "../types"; + +function safeOrigin(url: string | null): string | null { + if (!url) return null; + try { + return new URL(url).origin; + } catch { + return null; + } +} /** * Backward-compat seeding. Before AaaS, evault-core fanned out every webhook to @@ -74,7 +87,9 @@ export class SeedService { host = new URL(platformUrl).host; targetUrl = new URL("/api/webhook", platformUrl).toString(); } catch { - console.warn(`[seed] skipping invalid platform: ${platformUrl}`); + console.warn( + `[seed] skipping invalid platform: ${platformUrl}`, + ); continue; } @@ -108,7 +123,7 @@ export class SeedService { }, }); if (!existing) { - await subRepo.save( + const created = await subRepo.save( subRepo.create({ consumerId: consumer.id, targetUrl, @@ -118,6 +133,7 @@ export class SeedService { active: true, }), ); + await this.queueLookback(created); seeded += 1; } else if ( !existing.active || @@ -128,6 +144,7 @@ export class SeedService { existing.ontologyFilter = []; existing.evaultFilter = []; await subRepo.save(existing); + await this.queueLookback(existing); seeded += 1; } } @@ -135,7 +152,7 @@ export class SeedService { const managedSubscriptions = await subRepo .createQueryBuilder("s") .innerJoin(Consumer, "c", "c.id = s.consumerId") - .addSelect('c.ename', "consumerEname") + .addSelect("c.ename", "consumerEname") .where("s.isCatchAll = true") .andWhere("s.active = true") .andWhere("c.ename LIKE :prefix", { prefix: "catchall:%" }) @@ -161,4 +178,75 @@ export class SeedService { ); return { seeded, total: platforms.length }; } + + /** Fill the registry reconciliation window without replaying old history. */ + private async queueLookback(subscription: Subscription): Promise { + const since = new Date(Date.now() - config.deliveryRetryWindowMs); + const targetOrigin = safeOrigin(subscription.targetUrl); + const deliveryRepo = AppDataSource.getRepository(Delivery); + let cursorReceivedAt: Date | null = null; + let cursorEventId: string | null = null; + + for (;;) { + const query = AppDataSource.getRepository(AwarenessEvent) + .createQueryBuilder("e") + .where("e.receivedAt >= :since", { since }) + .orderBy("e.receivedAt", "ASC") + .addOrderBy("e.eventId", "ASC") + .take(500); + if (cursorReceivedAt && cursorEventId) { + query.andWhere( + `(e.receivedAt > :cursorReceivedAt OR + (e.receivedAt = :cursorReceivedAt AND e.eventId > :cursorEventId))`, + { cursorReceivedAt, cursorEventId }, + ); + } + const events = await query.getMany(); + if (events.length === 0) break; + + const rows = events + .filter( + (event) => + !targetOrigin || + safeOrigin(event.requestingPlatform) !== targetOrigin, + ) + .map((event) => { + const payload: AwarenessPayload = { + eventId: event.eventId, + id: event.packetId, + w3id: event.w3id, + evaultPublicKey: event.evaultPublicKey, + data: event.data, + schemaId: event.ontology, + operation: event.operation, + streamVersion: event.streamVersion, + occurredAt: event.occurredAt.toISOString(), + }; + return deliveryRepo.create({ + subscriptionId: subscription.id, + packetId: event.packetId, + eventId: event.eventId, + contentHash: contentHash(payload), + payload, + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + retryStartedAt: new Date(), + }); + }); + if (rows.length > 0) { + await deliveryRepo + .createQueryBuilder() + .insert() + .values(rows) + .orIgnore() + .execute(); + } + + const last = events.at(-1)!; + cursorReceivedAt = last.receivedAt; + cursorEventId = last.eventId; + if (events.length < 500) break; + } + } } diff --git a/services/awareness-service/api/src/services/SubscriptionMatcher.ts b/services/awareness-service/api/src/services/SubscriptionMatcher.ts index a05169114..53ccd5a15 100644 --- a/services/awareness-service/api/src/services/SubscriptionMatcher.ts +++ b/services/awareness-service/api/src/services/SubscriptionMatcher.ts @@ -1,4 +1,5 @@ import { AppDataSource } from "../database/data-source"; +import type { EntityManager } from "typeorm"; import { Subscription } from "../database/entities/Subscription"; import type { Packet } from "../database/entities/Packet"; @@ -10,12 +11,16 @@ import type { Packet } from "../database/entities/Packet"; * - its evaultFilter is empty OR contains the packet's w3id / evaultPublicKey. */ export class SubscriptionMatcher { - async match(packet: Packet): Promise { + async match( + packet: Packet, + manager: EntityManager = AppDataSource.manager, + ): Promise { const evaultIds = [packet.w3id, packet.evaultPublicKey].filter( (v): v is string => Boolean(v), ); - return AppDataSource.getRepository(Subscription) + return manager + .getRepository(Subscription) .createQueryBuilder("s") .innerJoin("consumers", "c", "c.id = s.consumerId") .where("s.active = true") diff --git a/services/awareness-service/api/src/types.ts b/services/awareness-service/api/src/types.ts index 31147d7be..78589cbcc 100644 --- a/services/awareness-service/api/src/types.ts +++ b/services/awareness-service/api/src/types.ts @@ -2,15 +2,20 @@ import type { Consumer } from "./database/entities/Consumer"; /** The raw payload evault-core POSTs to /ingest (and the body delivered to subscribers). */ export interface AwarenessPayload { + /** Stable source-generated event id. Required for new producers. */ + eventId?: string; id: string; w3id?: string | null; evaultPublicKey?: string | null; data?: Record | null; schemaId: string; operation?: "create" | "update" | "delete"; + streamVersion?: number | string | null; + occurredAt?: string; /** * The platform that triggered the change, if known. Used only to skip - * delivering the packet back to its origin; never persisted or delivered. + * delivering the packet back to its origin. Retained in event history for + * audit and reconciliation, but never included in subscriber payloads. */ requestingPlatform?: string | null; } diff --git a/services/awareness-service/api/src/utils/contentHash.ts b/services/awareness-service/api/src/utils/contentHash.ts index 4738403ff..b21c6993a 100644 --- a/services/awareness-service/api/src/utils/contentHash.ts +++ b/services/awareness-service/api/src/utils/contentHash.ts @@ -26,8 +26,9 @@ export function stableStringify(value: unknown): string { /** * SHA-256 of a packet's payload, computed over a stable serialisation so the - * hash depends only on content, not key ordering. Used to dedupe deliveries by - * content: identical re-ingests share a hash, a changed payload yields a new one. + * hash depends only on content, not key ordering. This remains useful as an + * audit fingerprint and as a compatibility event ID for legacy producers that + * do not yet send one. Delivery identity is the source eventId, never this hash. */ export function contentHash(data: unknown): string { return crypto From 67059186b7e983cd66b4f2143ed2ddf344d59847 Mon Sep 17 00:00:00 2001 From: coodos Date: Tue, 15 Sep 2026 03:49:53 +0800 Subject: [PATCH 2/2] test(awareness): cover recovery and document deployment --- .github/workflows/tests-awareness-service.yml | 63 +++ docs/docs/Infrastructure/eVault.md | 27 +- docs/docs/Services/Awareness-as-a-Service.md | 50 +- docs/docs/W3DS Basics/Data-Ownership-Rules.md | 10 +- docs/docs/W3DS Basics/getting-started.md | 12 +- docs/docs/W3DS Protocol/Awareness-Protocol.md | 77 +-- .../src/core/protocol/graphql-server.spec.ts | 526 +++++++++++------- .../protocol/uploadFile-awareness.spec.ts | 207 +++---- services/awareness-service/README.md | 28 +- .../api/src/services/DeliveryEngine.spec.ts | 44 ++ .../IngestService.integration.spec.ts | 204 +++++++ skills/w3ds/SKILL.md | 8 +- skills/w3ds/reference/dev-setup.md | 9 +- skills/w3ds/reference/evault.md | 17 +- skills/w3ds/reference/protocols.md | 44 +- skills/w3ds/reference/w3ds-native.md | 6 +- 16 files changed, 857 insertions(+), 475 deletions(-) create mode 100644 .github/workflows/tests-awareness-service.yml create mode 100644 services/awareness-service/api/src/services/DeliveryEngine.spec.ts create mode 100644 services/awareness-service/api/src/services/IngestService.integration.spec.ts diff --git a/.github/workflows/tests-awareness-service.yml b/.github/workflows/tests-awareness-service.yml new file mode 100644 index 000000000..1d4e47594 --- /dev/null +++ b/.github/workflows/tests-awareness-service.yml @@ -0,0 +1,63 @@ +name: Tests [awareness-service] + +on: + push: + branches: [main] + pull_request: + branches: [main] + paths: + - "services/awareness-service/**" + - "infrastructure/evault-core/src/core/awareness/**" + - "infrastructure/evault-core/src/core/db/**" + - "infrastructure/evault-core/src/core/protocol/**" + - "pnpm-lock.yaml" + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.25.0 + run_install: false + + - name: Set up Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential python3 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build affected services + run: | + pnpm --filter awareness-service-api build + pnpm --filter evault-core build + + - name: Run AaaS reliability tests + env: + CI: true + DOCKER_HOST: unix:///var/run/docker.sock + TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock + TESTCONTAINERS_HOST_OVERRIDE: localhost + run: pnpm --filter awareness-service-api test + + - name: Run eVault awareness outbox tests + env: + CI: true + DOCKER_HOST: unix:///var/run/docker.sock + TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock + TESTCONTAINERS_HOST_OVERRIDE: localhost + run: | + pnpm --filter evault-core exec vitest run src/core/protocol/graphql-server.spec.ts src/core/protocol/uploadFile-awareness.spec.ts --silent diff --git a/docs/docs/Infrastructure/eVault.md b/docs/docs/Infrastructure/eVault.md index 7e0c56019..efec83d0f 100644 --- a/docs/docs/Infrastructure/eVault.md +++ b/docs/docs/Infrastructure/eVault.md @@ -649,20 +649,22 @@ A valid Registry-issued platform token satisfies the *legacy* path — but it do ## Webhook Delivery -When data is stored or updated, eVault automatically sends webhooks to all registered platforms. +When data is created, updated, or deleted, eVault atomically records an +awareness outbox event beside the mutation. A restart-safe dispatcher sends it +to AaaS, which owns subscription matching and webhook delivery. ### Webhook Process -1. **Data Stored**: MetaEnvelope is stored in Neo4j -2. **Wait 3 Seconds**: Delay prevents webhook ping-pong (same platform receiving its own webhook) -3. **Get Active Platforms**: Query [Registry](/docs/Infrastructure/Registry) for list of active platforms -4. **Filter Requesting Platform**: Exclude the platform that made the request -5. **Send Webhooks**: POST to each platform's `/api/webhook` endpoint (see [Webhook Controller Guide](/docs/Post%20Platform%20Guide/webhook-controller)) +1. **Atomic capture**: The MetaEnvelope mutation and immutable outbox event commit in one Neo4j transaction. +2. **Durable ingest**: The outbox dispatcher retries `POST AWARENESS_SERVICE_URL/ingest` until AaaS acknowledges persistence. +3. **Match and filter**: AaaS matches subscriptions and excludes the requesting platform. +4. **Send webhooks**: AaaS posts to each matching endpoint (see [Webhook Controller Guide](/docs/Post%20Platform%20Guide/webhook-controller)). ### Webhook Payload ```json { + "eventId": "7fd6c06c-80ae-4137-9d62-c15af53f92cf", "id": "global-id-123", "w3id": "@user-a.w3id", "schemaId": "550e8400-e29b-41d4-a716-446655440001", @@ -672,16 +674,19 @@ When data is stored or updated, eVault automatically sends webhooks to all regis "authorId": "...", "createdAt": "2025-01-24T10:00:00Z" }, - "evaultPublicKey": "z..." + "evaultPublicKey": "z...", + "operation": "update", + "streamVersion": 2, + "occurredAt": "2026-09-15T03:00:00.000Z" } ``` ### Webhook Delivery Details -- **Timeout**: 5 seconds per webhook -- **Retry**: No automatic retries (fire-and-forget) -- **Error Handling**: Logs failures but doesn't block the operation -- **Ordering**: Webhooks are sent in parallel to all platforms - sending to platform A does not block sending to platform B. All webhook POST requests are initiated concurrently. +- **Timeout**: 5 seconds per network attempt. +- **Retry**: eVault-to-AaaS retries until acknowledged; AaaS-to-subscriber retries with backoff for 24 hours and then dead-letters. +- **Idempotency**: Subscribers must deduplicate by `eventId` because delivery is at least once. +- **Ordering**: Events are ordered per subscription and MetaEnvelope while unrelated streams are delivered concurrently. ## Key Binding Certificates diff --git a/docs/docs/Services/Awareness-as-a-Service.md b/docs/docs/Services/Awareness-as-a-Service.md index ddd635450..f82c75171 100644 --- a/docs/docs/Services/Awareness-as-a-Service.md +++ b/docs/docs/Services/Awareness-as-a-Service.md @@ -22,16 +22,17 @@ to all of them. That design had three problems: - **Ungoverned** — any registered platform received everything; there was no access gate. -AaaS fixes all three. evault-core now makes **one** POST per change to -`AWARENESS_SERVICE_URL/ingest`, and AaaS owns persistence, polling, subscription -matching, and retrying delivery. +AaaS fixes all three. Every eVault mutation now commits an immutable event to a +Neo4j transactional outbox alongside the user's data. The outbox retries +`AWARENESS_SERVICE_URL/ingest` until AaaS atomically commits the event and its +matching deliveries; AaaS then owns polling and retrying subscriber delivery. ## Architecture ``` ┌─────────────────────────────┐ - evault-core ──POST───▶ │ AaaS /ingest │ - (per change) │ • persist packet │ + eVault outbox ─POST───▶ │ AaaS /ingest │ + (retry-until-ack) │ • persist immutable event │ │ • match subscriptions │ │ • queue deliveries │ └──────────────┬──────────────┘ @@ -63,9 +64,14 @@ existing receivers need no changes: } ``` -`/ingest` additionally accepts a `requestingPlatform` field, used only to skip +New producers add `eventId`, `streamVersion`, and `occurredAt`. `eventId` is the +stable idempotency key across outbox retries, while `id` remains the +MetaEnvelope id. These fields are delivered additively to existing receivers. + +`/ingest` additionally accepts a `requestingPlatform` field, used to skip delivering a packet back to its origin (the ping-pong guard the old fanout -enforced). It is never persisted or delivered. +enforced). It is retained in immutable event history for audit/reconciliation, +but is not included in subscriber payloads. ### File uploads @@ -113,16 +119,16 @@ subscription has a `secret`, each delivery carries an `x-aaas-signature` header (HMAC-SHA256 of the body). Because catch-all subscriptions receive every ontology, a receiver **must ack -packets it does not consume with a 200**. There is no 4xx short-circuit in the -delivery engine: a 400 on an unknown `schemaId` is retried up to -`AWARENESS_MAX_ATTEMPTS` and then dead-lettered. +packets it does not consume with a 200**. All non-2xx responses remain retryable +for the 24-hour window, after which the event is dead-lettered and alerted. ### 3. Retrying delivery + dead-letters -A background engine drains the delivery queue. Failed deliveries are retried -with exponential backoff (30s → 1m → 2m → 5m → 15m → 1h → 6h → 24h). After -`AWARENESS_MAX_ATTEMPTS` attempts the delivery is moved to a **dead-letter** -table, visible to admins in the portal, where it can be replayed. +A lease-based worker drains the delivery queue. Every Postgres operation and +batch has a deadline, so a poisoned connection cannot permanently wedge the +polling loop. Failed deliveries use jittered exponential backoff for 24 hours; +expired leases are reclaimed after crashes. After the retry window the delivery +moves to a **dead-letter** table, visible to admins in the portal for replay. ### 4. Public access portal @@ -157,14 +163,16 @@ AaaS is designed to be dropped in with **zero receiver-side changes**: 1. **Backfill.** AaaS runs on the same node as evault-core's Neo4j. The `backfill` script reads existing MetaEnvelopes straight from the graph and - seeds the `packets` table (history only — it does not queue deliveries). + seeds both immutable query history and the latest-state projection. It does + not queue deliveries. 2. **Catch-all reconciliation.** On every launch and once per configured sync interval, AaaS ensures each platform currently in the registry has an approved consumer and an active catch-all subscription pointing at `/api/webhook`. Existing and newly registered platforms therefore keep receiving every packet exactly as before. -3. **evault-core switch.** evault-core's `deliverWebhooks`/`getActivePlatforms` - are removed; a single `notifyAwareness` POST forwards each packet to AaaS. +3. **eVault transactional outbox.** Every mutation and its awareness event + commit together in Neo4j. A dispatcher retries ingestion until AaaS returns a + durable acknowledgement, including across eVault and AaaS restarts. ## Configuration @@ -177,8 +185,12 @@ AaaS is designed to be dropped in with **zero receiver-side changes**: | `AWARENESS_SERVICE_URL` | (evault-core) where to POST packets | | `AAAS_ADMIN_ENAMES` | Comma-separated admin eNames | | `AAAS_JWT_SECRET` | Signs portal session JWTs | -| `AWARENESS_MAX_ATTEMPTS` | Delivery attempts before dead-lettering (default 3) | | `AWARENESS_DELIVERY_POLL_MS` | Delivery engine poll interval (default 2000) | +| `AWARENESS_DELIVERY_LEASE_MS` | Expiring worker lease duration (default 30000) | +| `AWARENESS_DELIVERY_BATCH_TIMEOUT_MS` | Hard batch deadline (default 25000) | +| `AWARENESS_DELIVERY_RETRY_WINDOW_MS` | Subscriber retry window (default 24 hours) | +| `AWARENESS_DB_STATEMENT_TIMEOUT_MS` / `AWARENESS_DB_QUERY_TIMEOUT_MS` / `AWARENESS_DB_LOCK_TIMEOUT_MS` | Postgres anti-wedge deadlines | +| `AWARENESS_OUTBOX_POLL_MS` / `AWARENESS_OUTBOX_LEASE_MS` / `AWARENESS_OUTBOX_DB_TIMEOUT_MS` / `AWARENESS_OUTBOX_RETENTION_MS` | Durable eVault outbox tuning | | `AWARENESS_REGISTRY_SYNC_MS` | Registry catch-all reconciliation interval (default 60000; 0 disables periodic sync) | | `NEO4J_URI` / `NEO4J_USER` / `NEO4J_PASSWORD` | Standard eVault Neo4j vars — reused by the one-time backfill | | `PUBLIC_AWARENESS_API_URL` | (portal) AaaS API base URL | @@ -190,6 +202,6 @@ AaaS is designed to be dropped in with **zero receiver-side changes**: pnpm --filter awareness-service-api build pnpm --filter awareness-service-api migration:run pnpm --filter awareness-service-api backfill # one-time, from Neo4j -pnpm --filter awareness-service-api dev # API (keeps registry catch-alls synced) +pnpm --filter awareness-service-api dev # API + worker in one process pnpm --filter awareness-portal dev # portal ``` diff --git a/docs/docs/W3DS Basics/Data-Ownership-Rules.md b/docs/docs/W3DS Basics/Data-Ownership-Rules.md index 140eaa057..83944a4f3 100644 --- a/docs/docs/W3DS Basics/Data-Ownership-Rules.md +++ b/docs/docs/W3DS Basics/Data-Ownership-Rules.md @@ -78,14 +78,14 @@ Duplication is its own failure. [File URIs](/docs/W3DS%20Protocol/File-URIs) mak ## Bounds on how much a projection can be trusted -Synchronisation is eventual, and the [Awareness Protocol](/docs/W3DS%20Protocol/Awareness-Protocol) is prototype-level. Design the projection to tolerate all of this: +Synchronisation is eventual, and the [Awareness Protocol](/docs/W3DS%20Protocol/Awareness-Protocol) is delivered at least once. Design the projection to tolerate all of this: - **Last-write-wins.** No merge, no CRDT. -- **No ordering guarantee**, and no at-least-once delivery. -- **Fire-and-forget fanout** with no retries at the protocol level. The requesting platform is excluded from its own fanout. -- **A delay after creation** before fanout, to prevent ping-pong; updates fan out immediately. +- **Per-stream ordering, not global ordering.** Events for one subscription and MetaEnvelope are ordered; independent streams are concurrent. +- **At-least-once delivery.** Retries and crash recovery can produce duplicates; the requesting platform is excluded from its own fanout. +- **A bounded subscriber retry window.** AaaS retries for 24 hours and then requires dead-letter replay. -Consequences for your code: webhook handling must be **idempotent on the global `id`**, reads must tolerate a record that has not arrived yet, and nothing user-visible should depend on two platforms agreeing at the same instant. +Consequences for your code: webhook handling must deduplicate by **`eventId`** (not the MetaEnvelope `id`, which is shared by legitimate updates), reads must tolerate a record that has not arrived yet, and nothing user-visible should depend on two platforms agreeing at the same instant. ## Stateless applications diff --git a/docs/docs/W3DS Basics/getting-started.md b/docs/docs/W3DS Basics/getting-started.md index e4330a29d..f3a862fcb 100644 --- a/docs/docs/W3DS Basics/getting-started.md +++ b/docs/docs/W3DS Basics/getting-started.md @@ -149,13 +149,16 @@ mutation CreateMetaEnvelope($input: MetaEnvelopeInput!) { The [eVault](/docs/Infrastructure/eVault) stores the data as a [MetaEnvelope](/docs/Infrastructure/eVault#data-model), which is a flat graph structure of Envelopes. Each field becomes a separate Envelope node in Neo4j. -#### 6. Webhook Delivery (After 3 Second Delay) +#### 6. Durable Webhook Delivery -After a 3-second delay (to prevent webhook ping-pong), the [eVault](/docs/Infrastructure/eVault) sends webhooks to all registered platforms (see [Registry](/docs/Infrastructure/Registry)) **except** the one that made the request (Blabsy). +The eVault commits an outbox event with the MetaEnvelope write. AaaS ingests it, +then sends it to matching platforms **except** the one that made the request +(Blabsy). Both handoffs are restart-safe and retry automatically. The webhook payload contains: ```json { + "eventId": "7fd6c06c-80ae-4137-9d62-c15af53f92cf", "id": "global-id-123", "w3id": "@user-a.w3id", "schemaId": "550e8400-e29b-41d4-a716-446655440001", @@ -164,7 +167,10 @@ The webhook payload contains: "mediaUrls": [], "authorId": "...", "createdAt": "2025-01-24T10:00:00Z" - } + }, + "operation": "create", + "streamVersion": 1, + "occurredAt": "2026-09-15T03:00:00.000Z" } ``` diff --git a/docs/docs/W3DS Protocol/Awareness-Protocol.md b/docs/docs/W3DS Protocol/Awareness-Protocol.md index 7818dd0c8..fcd9c9111 100644 --- a/docs/docs/W3DS Protocol/Awareness-Protocol.md +++ b/docs/docs/W3DS Protocol/Awareness-Protocol.md @@ -4,29 +4,32 @@ sidebar_position: 4 # Awareness Protocol -:::warning Prototype-level implementation -The Awareness Protocol described here is **prototype-level**. The packet format, delivery behavior, and platform contract are subject to change in upcoming updates +:::info Delivery guarantee +Awareness delivery is durable and **at least once**. Receivers must deduplicate +on `eventId`; the same event can be sent again after a timeout or worker crash. ::: -The Awareness Protocol is the webhook delivery mechanism in W3DS. When data in an [eVault](/docs/Infrastructure/eVault) changes (create or update), the eVault notifies every registered platform so they can stay in sync. "Awareness" means platforms become aware of changes that happened elsewhere. +The Awareness Protocol is the webhook delivery mechanism in W3DS. When data in an [eVault](/docs/Infrastructure/eVault) changes, the mutation atomically creates an awareness outbox event. [Awareness as a Service](/docs/Services/Awareness-as-a-Service) (AaaS) persists that immutable event and delivers it to matching subscriptions. ## Overview -Platforms do not poll eVaults for changes. Instead, [eVault](/docs/Infrastructure/eVault) Core pushes change notifications to every registered platform (except the one that originated the change) via HTTP POST to each platform's `/api/webhook` endpoint. The payload is called an **awareness protocol packet**. Platforms use this packet to apply the same change locally (e.g. create or update an entity in their database) and to maintain ID mappings between global and local IDs. +Platforms can receive changes by webhook or poll AaaS history. Compatibility subscriptions send every change to each registered platform's `/api/webhook` endpoint except the platform that originated the change. Granular consumers can instead subscribe by ontology and eVault. ### Key Properties -- **Push-based**: eVault initiates delivery; platforms do not poll. -- **Fire-and-forget**: The eVault does not wait for platforms to acknowledge; failures are logged but do not block the store/update operation. -- **No retries**: There are no automatic retries for failed webhook deliveries in the current implementation. +- **Transactional source capture**: User data and its source outbox event commit together in Neo4j. +- **At-least-once delivery**: eVault retries ingestion until AaaS acknowledges it; AaaS retries subscribers with backoff for 24 hours. +- **Restart-safe**: Pending work and expiring worker leases live in Neo4j/Postgres, not process memory. +- **Ordered per stream**: Events for one subscription and MetaEnvelope are delivered in source order; unrelated streams run concurrently. - **Requestor excluded**: The platform that made the GraphQL request (store/update) is excluded from the list of recipients to avoid "webhook ping-pong." ## When the Protocol Runs -The Awareness Protocol is triggered after: +An awareness event is committed by: -1. **[storeMetaEnvelope](/docs/Infrastructure/eVault#graphql-api)** (create): After the new MetaEnvelope is stored in the eVault, webhooks are **scheduled with a 3-second delay** to ensure the requesting platform can be reliably identified and excluded from recipients, preventing the same platform from receiving its own write-back and creating a feedback loop ("webhook ping-pong"). -2. **[updateMetaEnvelopeById](/docs/Infrastructure/eVault#graphql-api)** (update): After the MetaEnvelope is updated, webhooks are sent **immediately** (fire-and-forget, no delay). +1. **[storeMetaEnvelope](/docs/Infrastructure/eVault#graphql-api)** and bulk/file/binding-document creates. +2. **[updateMetaEnvelopeById](/docs/Infrastructure/eVault#graphql-api)** and individual envelope updates. +3. **deleteMetaEnvelope**, as a tombstone with `operation: "delete"` and `data: null`. ## Mechanism @@ -34,28 +37,25 @@ The Awareness Protocol is triggered after: sequenceDiagram participant PlatformA as Platform A participant EVault as eVault Core - participant Registry as Registry + participant AaaS as AaaS participant PlatformB as Platform B participant PlatformC as Platform C PlatformA->>EVault: storeMetaEnvelope / updateMetaEnvelopeById - EVault->>EVault: Persist to Neo4j - alt storeMetaEnvelope - Note over EVault: Wait 3 seconds - end - EVault->>Registry: GET /platforms - Registry-->>EVault: List of platform base URLs - EVault->>EVault: Filter out requesting platform - EVault->>PlatformB: POST /api/webhook (payload) - EVault->>PlatformC: POST /api/webhook (payload) - Note over EVault,PlatformC: All requests in parallel, 5s timeout each, no retries + EVault->>EVault: Atomically persist data + outbox event + EVault->>AaaS: POST /ingest (retry until acknowledged) + AaaS->>AaaS: Atomically persist event + delivery rows + AaaS->>PlatformB: POST /api/webhook + AaaS->>PlatformC: POST /api/webhook + Note over AaaS,PlatformC: Retry non-2xx/timeouts for 24h; then dead-letter ``` ### Step-by-Step -1. **Platform list**: [eVault](/docs/Infrastructure/eVault) calls the [Registry](/docs/Infrastructure/Registry) with `GET /platforms` and receives a list of platform base URLs (e.g. `["https://blabsy.example.com", "https://pictique.example.com", ...]`). -2. **Filter**: The requesting platform is removed from the list. The requestor is identified from the Bearer token's `platform` claim (the platform URL that was certified when the token was issued). URL comparison is normalized so that equivalent URLs are treated as the same. -3. **Delivery**: For each remaining URL, the eVault sends `POST {platformUrl}/api/webhook` with the awareness protocol payload. Each request has a 5-second timeout. Requests are sent in parallel (`Promise.allSettled`); if one fails, others still run, and failures are logged without blocking the mutation. +1. **Capture**: eVault stores the data mutation and immutable `AwarenessOutbox` event in one Neo4j transaction. +2. **Ingest**: The eVault dispatcher sends the event to AaaS. Network and service failures remain queued across restarts and retry until acknowledged. +3. **Match**: AaaS atomically stores the immutable event, updates its latest-state projection, and queues every matching subscription. The requesting platform is excluded by normalized origin. +4. **Deliver**: Lease-based workers POST to subscribers concurrently across independent streams. Timeouts and every non-2xx response retry with jittered backoff for 24 hours, then move to the dead-letter queue for replay. ## Packet Format (Awareness Protocol Payload) @@ -63,10 +63,15 @@ The body of each webhook request is JSON with the following fields: | Field | Description | |-------|-------------| +| `eventId` | Stable, globally unique idempotency key for this mutation event. | | `id` | MetaEnvelope ID (W3ID). | | `w3id` | Owner eName (eVault owner W3ID). | +| `evaultPublicKey` | Public key of the source eVault, when available. | | `schemaId` | [Ontology](/docs/Infrastructure/Ontology) schema W3ID (identifies the type of entity and which mapping the platform should use). | -| `data` | The entity payload in the **global ontology** shape (parsed key-value structure). See [eVault](/docs/Infrastructure/eVault) for MetaEnvelope storage. | +| `data` | Full entity payload in the **global ontology** shape, or `null` for a delete tombstone. | +| `operation` | `create`, `update`, or `delete`. | +| `streamVersion` | Monotonic version within this MetaEnvelope's event stream. | +| `occurredAt` | Source mutation timestamp in ISO-8601 format. | In the current version of the implementation the entire payload is sent in plain text to any registered platform, so all data is sent to every platform and @@ -81,6 +86,7 @@ updated MetaEnvelope by reference instead of value. ```json { + "eventId": "7fd6c06c-80ae-4137-9d62-c15af53f92cf", "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "w3id": "@e4d909c2-5d2f-4a7d-9473-b34b6c0f1a5a", "schemaId": "550e8400-e29b-41d4-a716-446655440001", @@ -89,7 +95,10 @@ updated MetaEnvelope by reference instead of value. "mediaUrls": [], "authorId": "@e4d909c2-5d2f-4a7d-9473-b34b6c0f1a5a", "createdAt": "2025-01-24T10:00:00Z" - } + }, + "operation": "update", + "streamVersion": 4, + "occurredAt": "2026-09-15T03:00:00.000Z" } ``` @@ -100,21 +109,17 @@ Platforms that participate in W3DS must implement an HTTP endpoint that accepts - **Method and path**: `POST /api/webhook` - **Request**: JSON body as described above. - **Behavior**: The platform should (1) use `schemaId` to find the correct mapping from global ontology to local schema, (2) transform `data` from global to local format (e.g. using the [Web3 Adapter](/docs/Infrastructure/Web3-Adapter#fromglobal)'s `fromGlobal`), (3) resolve or create the local entity and store the global-ID-to-local-ID mapping, (4) return HTTP 200 on success. -- **Idempotency**: Implementors are encouraged to treat the same `id` (global ID) as idempotent (create or update the same local entity) so that duplicate or retried deliveries do not create duplicates. +- **Idempotency**: Persist processed `eventId` values and acknowledge repeats without applying them twice. Do **not** deduplicate on `id`: create and later updates intentionally share the same MetaEnvelope id. - **Unknown ontologies**: Delivery is a broadcast — a platform receives packets for ontologies it has no mapping for, such as the `w3ds-file-v1` envelopes emitted by `uploadFile`. Log and **return HTTP 200**; do not return 4xx. AaaS has no 4xx short-circuit, so an error response is retried and then dead-lettered even though nothing was wrong. For a step-by-step implementation guide, see the [Webhook Controller Guide](/docs/Post%20Platform%20Guide/webhook-controller) in the Post Platform Guide. -## Limitations and Extension Points +## Remaining delivery semantics -The current protocol has limitations which are going to be improved in subsequent versions: - -- **No retries**: Failed webhooks are not retried. A more robust system could add retries with backoff or a dead-letter queue. -- **No ordering guarantee**: Webhooks to different platforms are sent in parallel; there is no guarantee of order across platforms or across multiple mutations. -- **No at-least-once guarantee**: Because delivery is fire-and-forget and there are no retries, a platform might never receive a given update. At-least-once delivery would require acknowledgments and retries (and possibly idempotency keys). -- **Platform list from Registry**: The set of recipients is whatever the Registry returns for `GET /platforms`. This is a prototype level shortcut and will be phased out. - -Designing retries, ordering, or delivery guarantees would be natural extension points for a production-grade awareness mechanism. +- **Duplicates are possible**: At-least-once delivery deliberately prefers a duplicate over a lost event. Receivers own `eventId` deduplication. +- **Automatic retry is bounded downstream**: Subscriber delivery retries for 24 hours, then requires dead-letter replay. Source eVault-to-AaaS ingestion retries without that cutoff. +- **Ordering is stream-local**: One MetaEnvelope is ordered for one subscription. There is intentionally no global order across independent envelopes or subscribers. +- **Registry compatibility remains**: AaaS still reconciles catch-all subscriptions from the Registry for existing platforms; new consumers can use granular subscriptions. ## References diff --git a/infrastructure/evault-core/src/core/protocol/graphql-server.spec.ts b/infrastructure/evault-core/src/core/protocol/graphql-server.spec.ts index 539bbdae2..0f0498afc 100644 --- a/infrastructure/evault-core/src/core/protocol/graphql-server.spec.ts +++ b/infrastructure/evault-core/src/core/protocol/graphql-server.spec.ts @@ -1,5 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from "vitest"; -import axios from "axios"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import * as jose from "jose"; import { setupE2ETestServer, @@ -10,20 +9,75 @@ import { type ProvisionedEVault, } from "../../test-utils/e2e-setup"; import { getSharedTestKeyPair } from "../../test-utils/shared-test-keys"; - -// Store original axios functions before any spying happens -const originalAxiosPost = axios.post; - -// evault-core forwards every awareness packet to AaaS at -// AWARENESS_SERVICE_URL/ingest; point it somewhere the spy can intercept. -process.env.AWARENESS_SERVICE_URL = "http://localhost:9999"; - -describe("GraphQLServer Awareness Ingest Payload W3ID", () => { +import { AwarenessOutboxDispatcher } from "../awareness/awareness-outbox-dispatcher"; +import { createServer } from "node:http"; +import { createServer as createNetServer } from "node:net"; +import type { AddressInfo } from "node:net"; + +interface OutboxPayload { + eventId: string; + packetId: string; + w3id: string; + schemaId: string; + data: Record | null; + operation: string; + requestingPlatform: string | null; + streamVersion: number; + status: string; +} + +async function outboxPayloads(server: E2ETestServer): Promise { + const session = server.neo4jDriver.session(); + try { + const result = await session.run( + "MATCH (a:AwarenessOutbox) RETURN a ORDER BY a.createdAt", + ); + return result.records.map((record) => { + const p = record.get("a").properties; + return { + eventId: p.eventId, + packetId: p.packetId, + w3id: p.w3id, + schemaId: p.schemaId, + data: JSON.parse(p.dataJson), + operation: p.operation, + requestingPlatform: p.requestingPlatform ?? null, + streamVersion: p.streamVersion.toNumber(), + status: p.status, + }; + }); + } finally { + await session.close(); + } +} + +async function availablePort(): Promise { + return new Promise((resolve, reject) => { + const socket = createNetServer(); + socket.once("error", reject); + socket.listen(0, "127.0.0.1", () => { + const port = (socket.address() as AddressInfo).port; + socket.close((error) => (error ? reject(error) : resolve(port))); + }); + }); +} + +async function waitFor( + predicate: () => Promise, + timeoutMs = 5_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("condition not reached before timeout"); +} + +describe("GraphQL transactional awareness outbox", () => { let server: E2ETestServer; let evault1: ProvisionedEVault; let evault2: ProvisionedEVault; - const evaultW3ID = "evault-w3id-123"; - let axiosPostSpy: any; beforeAll(async () => { server = await setupE2ETestServer(); @@ -31,234 +85,288 @@ describe("GraphQLServer Awareness Ingest Payload W3ID", () => { evault2 = await provisionTestEVault(server); }, 120000); - afterAll(async () => { - await teardownE2ETestServer(server); - if (axiosPostSpy) { - axiosPostSpy.mockRestore(); - } - }); + afterAll(async () => teardownE2ETestServer(server)); - beforeEach(() => { - if (axiosPostSpy) { - axiosPostSpy.mockRestore(); + beforeEach(async () => { + const session = server.neo4jDriver.session(); + try { + await session.run("MATCH (a:AwarenessOutbox) DETACH DELETE a"); + } finally { + await session.close(); } + }); - vi.clearAllMocks(); - - // Spy on axios.post to capture the awareness ingest payload. - axiosPostSpy = vi.spyOn(axios, "post").mockImplementation((url: string | any, data?: any, config?: any) => { - // If it's the AaaS ingest call, capture it and return success. - if (typeof url === "string" && url.includes("/ingest")) { - console.log("Ingest intercepted:", { url, data }); - return Promise.resolve({ status: 200, data: { ok: true } }) as any; - } - // For GraphQL and other requests, call through to original. - return originalAxiosPost.call(axios, url, data, config); + it("atomically records the owner's W3ID and payload on create", async () => { + const data = { field: "value", test: "store-test" }; + const result = await makeGraphQLRequest( + server, + `mutation Store($input: MetaEnvelopeInput!) { + storeMetaEnvelope(input: $input) { metaEnvelope { id ontology } } + }`, + { input: { ontology: "OutboxCreate", payload: data, acl: ["*"] } }, + { "X-ENAME": evault1.w3id }, + ); + + const events = await outboxPayloads(server); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + packetId: result.storeMetaEnvelope.metaEnvelope.id, + w3id: evault1.w3id, + schemaId: "OutboxCreate", + data, + operation: "create", + streamVersion: 1, + status: "pending", }); + expect(events[0].eventId).toBeTruthy(); }); - describe("storeMetaEnvelope ingest payload", () => { - it("should include X-ENAME in the ingest payload", async () => { - const testData = { field: "value", test: "store-test" }; - const testOntology = "WebhookTestOntology"; - - // Make GraphQL mutation with user's W3ID in X-ENAME header - const mutation = ` - mutation StoreMetaEnvelope($input: MetaEnvelopeInput!) { - storeMetaEnvelope(input: $input) { - metaEnvelope { - id - ontology - } - } - } - `; - - await makeGraphQLRequest(server, mutation, { + it("keeps events for different owners distinct", async () => { + const mutation = `mutation Store($input: MetaEnvelopeInput!) { + storeMetaEnvelope(input: $input) { metaEnvelope { id } } + }`; + await makeGraphQLRequest( + server, + mutation, + { input: { - ontology: testOntology, - payload: testData, + ontology: "OutboxOwner", + payload: { user: 1 }, acl: ["*"], }, - }, { - "X-ENAME": evault1.w3id, - }); - - // notifyAwareness is fire-and-forget; give it a moment to run. - await new Promise(resolve => setTimeout(resolve, 1000)); - - // Verify axios.post was called (awareness ingest) - expect(axios.post).toHaveBeenCalled(); - - // Get the ingest payload from the axios.post call - const ingestCalls = (axios.post as any).mock.calls; - const ingestCall = ingestCalls.find((call: any[]) => - typeof call[0] === "string" && call[0].includes("/ingest") - ); - - expect(ingestCall).toBeDefined(); - const ingestPayload = ingestCall[1]; // Second argument is the payload - - console.log("Ingest payload:", JSON.stringify(ingestPayload, null, 2)); - console.log("Expected w3id:", evault1.w3id); - - // Verify the payload contains the user's W3ID, not the eVault's W3ID - expect(ingestPayload).toBeDefined(); - expect(ingestPayload.w3id).toBe(evault1.w3id); - expect(ingestPayload.w3id).not.toBe(evaultW3ID); - expect(ingestPayload.data).toEqual(testData); - expect(ingestPayload.schemaId).toBe(testOntology); - }); - - it("should use different W3IDs for different users in ingest payloads", async () => { - const testData1 = { user: "1", data: "test1" }; - const testData2 = { user: "2", data: "test2" }; - const testOntology = "MultiUserWebhookTest"; - - const mutation = ` - mutation StoreMetaEnvelope($input: MetaEnvelopeInput!) { - storeMetaEnvelope(input: $input) { - metaEnvelope { - id - ontology - } - } - } - `; - - // Store for user1 - await makeGraphQLRequest(server, mutation, { + }, + { "X-ENAME": evault1.w3id }, + ); + await makeGraphQLRequest( + server, + mutation, + { input: { - ontology: testOntology, - payload: testData1, + ontology: "OutboxOwner", + payload: { user: 2 }, acl: ["*"], }, - }, { - "X-ENAME": evault1.w3id, - }); + }, + { "X-ENAME": evault2.w3id }, + ); + + const events = await outboxPayloads(server); + expect(events.map((event) => event.w3id)).toEqual([ + evault1.w3id, + evault2.w3id, + ]); + expect(new Set(events.map((event) => event.eventId)).size).toBe(2); + }); - // Store for user2 - await makeGraphQLRequest(server, mutation, { + it("records ordered full-state updates and origin metadata", async () => { + const create = await makeGraphQLRequest( + server, + `mutation Store($input: MetaEnvelopeInput!) { + storeMetaEnvelope(input: $input) { metaEnvelope { id } } + }`, + { input: { - ontology: testOntology, - payload: testData2, + ontology: "OutboxUpdate", + payload: { field: "initial", preserved: true }, acl: ["*"], }, - }, { - "X-ENAME": evault2.w3id, - }); - - // Give the fire-and-forget ingest calls a moment to run. - await new Promise(resolve => setTimeout(resolve, 1000)); - - // Get all ingest calls - const ingestCalls = (axios.post as any).mock.calls.filter((call: any[]) => - typeof call[0] === "string" && call[0].includes("/ingest") - ); - - expect(ingestCalls.length).toBeGreaterThanOrEqual(2); - - // Find payloads by their data - const payload1 = ingestCalls.find((call: any[]) => - call[1]?.data?.user === "1" - )?.[1]; - const payload2 = ingestCalls.find((call: any[]) => - call[1]?.data?.user === "2" - )?.[1]; - - expect(payload1).toBeDefined(); - expect(payload1.w3id).toBe(evault1.w3id); - expect(payload2).toBeDefined(); - expect(payload2.w3id).toBe(evault2.w3id); - expect(payload1.w3id).not.toBe(payload2.w3id); + }, + { "X-ENAME": evault1.w3id }, + ); + const id = create.storeMetaEnvelope.metaEnvelope.id; + const { privateKey } = await getSharedTestKeyPair(); + const platform = "http://localhost:3000"; + const token = await new jose.SignJWT({ platform }) + .setProtectedHeader({ alg: "ES256", kid: "entropy-key-1" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(privateKey); + + await makeGraphQLRequest( + server, + `mutation Update($id: String!, $input: MetaEnvelopeInput!) { + updateMetaEnvelopeById(id: $id, input: $input) { metaEnvelope { id } } + }`, + { + id, + input: { + ontology: "OutboxUpdate", + payload: { field: "updated" }, + acl: ["*"], + }, + }, + { "X-ENAME": evault1.w3id, Authorization: `Bearer ${token}` }, + ); + + const events = await outboxPayloads(server); + expect(events).toHaveLength(2); + expect(events[1]).toMatchObject({ + packetId: id, + operation: "update", + streamVersion: 2, + requestingPlatform: platform, + data: { field: "updated", preserved: true }, }); }); - describe("updateMetaEnvelopeById ingest payload", () => { - it("should include user's W3ID (eName) in the ingest payload, not eVault's W3ID", async () => { - const testData = { field: "updated-value", test: "update-test" }; - const testOntology = "UpdateWebhookTestOntology"; - - // First, create an envelope - const createMutation = ` - mutation StoreMetaEnvelope($input: MetaEnvelopeInput!) { - storeMetaEnvelope(input: $input) { - metaEnvelope { - id - ontology - } - } - } - `; - - const createResult = await makeGraphQLRequest(server, createMutation, { + it("records a delete tombstone and keeps stream versions monotonic after recreation", async () => { + const create = await makeGraphQLRequest( + server, + `mutation Store($input: MetaEnvelopeInput!) { + storeMetaEnvelope(input: $input) { metaEnvelope { id } } + }`, + { input: { - ontology: testOntology, - payload: { field: "initial-value" }, + ontology: "OutboxDelete", + payload: { value: "gone" }, acl: ["*"], }, - }, { + }, + { "X-ENAME": evault1.w3id }, + ); + const id = create.storeMetaEnvelope.metaEnvelope.id; + const { privateKey } = await getSharedTestKeyPair(); + const token = await new jose.SignJWT({ + platform: "http://localhost:3000", + }) + .setProtectedHeader({ alg: "ES256", kid: "entropy-key-1" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(privateKey); + await makeGraphQLRequest( + server, + `mutation Delete($id: String!) { deleteMetaEnvelope(id: $id) }`, + { id }, + { "X-ENAME": evault1.w3id, - }); - - const envelopeId = createResult.storeMetaEnvelope.metaEnvelope.id; - - // Clear previous ingest calls - (axios.post as any).mockClear(); + Authorization: `Bearer ${token}`, + }, + ); + + const events = await outboxPayloads(server); + expect(events.at(-1)).toMatchObject({ + packetId: id, + schemaId: "OutboxDelete", + operation: "delete", + data: null, + streamVersion: 2, + }); - // Now update the envelope - const updateMutation = ` - mutation UpdateMetaEnvelopeById($id: String!, $input: MetaEnvelopeInput!) { - updateMetaEnvelopeById(id: $id, input: $input) { - metaEnvelope { - id - ontology - } - } + const recreate = await makeGraphQLRequest( + server, + `mutation Recreate($inputs: [BulkMetaEnvelopeInput!]!) { + bulkCreateMetaEnvelopes(inputs: $inputs) { + successCount } - `; - - // Create a valid Bearer token for authentication. - const { privateKey } = await getSharedTestKeyPair(); - const testToken = await new jose.SignJWT({ platform: "http://localhost:3000" }) - .setProtectedHeader({ alg: "ES256", kid: "entropy-key-1" }) - .setIssuedAt() - .setExpirationTime("1h") - .sign(privateKey); + }`, + { + inputs: [ + { + id, + ontology: "OutboxDelete", + payload: { value: "back" }, + acl: ["*"], + }, + ], + }, + { + "X-ENAME": evault1.w3id, + Authorization: `Bearer ${token}`, + }, + ); + expect(recreate.bulkCreateMetaEnvelopes.successCount).toBe(1); + expect((await outboxPayloads(server)).at(-1)).toMatchObject({ + packetId: id, + operation: "create", + data: { value: "back" }, + streamVersion: 3, + }); + }); - await makeGraphQLRequest(server, updateMutation, { - id: envelopeId, + it("resumes a failed outbox event after dispatcher restart", async () => { + await makeGraphQLRequest( + server, + `mutation Store($input: MetaEnvelopeInput!) { + storeMetaEnvelope(input: $input) { metaEnvelope { id } } + }`, + { input: { - ontology: testOntology, - payload: testData, + ontology: "OutboxRestart", + payload: { durable: true }, acl: ["*"], }, - }, { - "X-ENAME": evault1.w3id, - "Authorization": `Bearer ${testToken}`, + }, + { "X-ENAME": evault1.w3id }, + ); + + const port = await availablePort(); + const previousUrl = process.env.AWARENESS_SERVICE_URL; + const previousPollMs = process.env.AWARENESS_OUTBOX_POLL_MS; + process.env.AWARENESS_SERVICE_URL = `http://127.0.0.1:${port}`; + process.env.AWARENESS_OUTBOX_POLL_MS = "20"; + + let received: Record | null = null; + let first: AwarenessOutboxDispatcher | undefined; + let second: AwarenessOutboxDispatcher | undefined; + let inlet: ReturnType | undefined; + try { + first = new AwarenessOutboxDispatcher(server.neo4jDriver); + first.start(); + await waitFor(async () => { + const event = (await outboxPayloads(server))[0]; + return event?.status === "failed"; }); - - // Give the fire-and-forget ingest call a moment to run. - await new Promise(resolve => setTimeout(resolve, 1000)); - - // Verify axios.post was called (awareness ingest) - expect(axios.post).toHaveBeenCalled(); - - // Get the ingest payload - const ingestCalls = (axios.post as any).mock.calls.filter((call: any[]) => - typeof call[0] === "string" && call[0].includes("/ingest") + await first.stop(); + first = undefined; + + inlet = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + request.on("end", () => { + received = JSON.parse( + Buffer.concat(chunks).toString("utf8"), + ); + response.writeHead(200, { + "content-type": "application/json", + }); + response.end('{"ok":true}'); + }); + }); + await new Promise((resolve) => + inlet!.listen(port, "127.0.0.1", () => resolve()), ); - expect(ingestCalls.length).toBeGreaterThan(0); - const ingestPayload = ingestCalls[0][1]; + second = new AwarenessOutboxDispatcher(server.neo4jDriver); + second.start(); + await waitFor(async () => { + const event = (await outboxPayloads(server))[0]; + return event?.status === "delivered"; + }); - // Verify the payload contains the user's W3ID, not the eVault's W3ID - expect(ingestPayload).toBeDefined(); - expect(ingestPayload.w3id).toBe(evault1.w3id); - expect(ingestPayload.w3id).not.toBe(evaultW3ID); - expect(ingestPayload.id).toBe(envelopeId); - expect(ingestPayload.data).toEqual(testData); - expect(ingestPayload.schemaId).toBe(testOntology); - }); + expect(received).toMatchObject({ + eventId: expect.any(String), + schemaId: "OutboxRestart", + data: { durable: true }, + streamVersion: 1, + }); + } finally { + await first?.stop(); + await second?.stop(); + if (inlet?.listening) { + await new Promise((resolve) => + inlet!.close(() => resolve()), + ); + } + if (previousUrl === undefined) { + delete process.env.AWARENESS_SERVICE_URL; + } else { + process.env.AWARENESS_SERVICE_URL = previousUrl; + } + if (previousPollMs === undefined) { + delete process.env.AWARENESS_OUTBOX_POLL_MS; + } else { + process.env.AWARENESS_OUTBOX_POLL_MS = previousPollMs; + } + } }); }); diff --git a/infrastructure/evault-core/src/core/protocol/uploadFile-awareness.spec.ts b/infrastructure/evault-core/src/core/protocol/uploadFile-awareness.spec.ts index 17d6f6091..2e07ea521 100644 --- a/infrastructure/evault-core/src/core/protocol/uploadFile-awareness.spec.ts +++ b/infrastructure/evault-core/src/core/protocol/uploadFile-awareness.spec.ts @@ -1,123 +1,94 @@ -import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from "vitest"; -import axios from "axios"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import * as jose from "jose"; import { + makeGraphQLRequest, + provisionTestEVault, setupE2ETestServer, teardownE2ETestServer, - provisionTestEVault, - makeGraphQLRequest, type E2ETestServer, type ProvisionedEVault, } from "../../test-utils/e2e-setup"; import { getSharedTestKeyPair } from "../../test-utils/shared-test-keys"; import { FILE_SCHEMA_ID } from "../utils/w3ds-uri"; -// Keep a handle on the real axios.post: the spy below must still let the -// GraphQL requests through to the test server. -const originalAxiosPost = axios.post; - -// evault-core forwards every awareness packet to AaaS at -// AWARENESS_SERVICE_URL/ingest; point it somewhere the spy can intercept. -process.env.AWARENESS_SERVICE_URL = "http://localhost:9999"; - -// StorageService.isConfigured() gates the uploadFile resolver, and its -// constructor throws without these. Set them before the module is imported. process.env.DO_SPACES_ENDPOINT = "https://ams3.digitaloceanspaces.com"; process.env.DO_SPACES_REGION = "ams3"; process.env.DO_SPACES_KEY = "test-key"; process.env.DO_SPACES_SECRET = "test-secret"; process.env.DO_SPACES_BUCKET = "test-bucket"; -// vi.mock is hoisted above every const in this module, so the shared spy has to -// be created inside vi.hoisted or the factory would hit it in the TDZ. const { s3Send } = vi.hoisted(() => ({ s3Send: vi.fn() })); - -// Stub the S3 transport so uploads never leave the process, while leaving -// StorageService itself real (buildKey and the public URL are what we assert). vi.mock("@aws-sdk/client-s3", () => ({ S3Client: vi.fn().mockImplementation(() => ({ send: s3Send })), PutObjectCommand: vi.fn().mockImplementation((input) => ({ input })), DeleteObjectCommand: vi.fn().mockImplementation((input) => ({ input })), })); -const UPLOAD_FILE = ` - mutation UploadFile($input: UploadFileInput!) { - uploadFile(input: $input) { - uri - metaEnvelopeId - publicUrl - errors { field message code } - } +const UPLOAD_FILE = `mutation Upload($input: UploadFileInput!) { + uploadFile(input: $input) { + uri metaEnvelopeId publicUrl errors { field message code } } -`; - -// The platform claim the test Bearer token carries. evault-core passes it to -// AaaS as requestingPlatform so the packet is not delivered back to its origin. +}`; const TEST_PLATFORM = "http://localhost:3000"; -/** Every /ingest call the spy captured, in order. */ -function ingestCalls() { - return (axios.post as any).mock.calls.filter( - (call: any[]) => - typeof call[0] === "string" && call[0].includes("/ingest"), - ); +async function outboxEvents(server: E2ETestServer) { + const session = server.neo4jDriver.session(); + try { + const result = await session.run( + "MATCH (a:AwarenessOutbox) RETURN a ORDER BY a.createdAt", + ); + return result.records.map((record) => { + const p = record.get("a").properties; + return { ...p, data: JSON.parse(p.dataJson) }; + }); + } finally { + await session.close(); + } } -/** - * uploadFile used to be the only write mutation that never dispatched an - * awareness packet, so uploaded blobs were invisible to AaaS. Consumers worked - * around it by mirroring every upload as a second envelope under a different - * ontology. These tests pin the dispatch in place. - */ -describe("uploadFile awareness ingest", () => { +describe("uploadFile transactional awareness outbox", () => { let server: E2ETestServer; let evault: ProvisionedEVault; let authHeaders: Record; - let axiosPostSpy: any; beforeAll(async () => { server = await setupE2ETestServer(); evault = await provisionTestEVault(server); - const { privateKey } = await getSharedTestKeyPair(); const token = await new jose.SignJWT({ platform: TEST_PLATFORM }) .setProtectedHeader({ alg: "ES256", kid: "entropy-key-1" }) .setIssuedAt() .setExpirationTime("1h") .sign(privateKey); - authHeaders = { "X-ENAME": evault.w3id, Authorization: `Bearer ${token}`, }; }, 120000); - afterAll(async () => { - await teardownE2ETestServer(server); - if (axiosPostSpy) axiosPostSpy.mockRestore(); - }); + afterAll(async () => teardownE2ETestServer(server)); - beforeEach(() => { - if (axiosPostSpy) axiosPostSpy.mockRestore(); + beforeEach(async () => { vi.clearAllMocks(); s3Send.mockResolvedValue({}); - - axiosPostSpy = vi - .spyOn(axios, "post") - .mockImplementation((url: string | any, data?: any, config?: any) => { - if (typeof url === "string" && url.includes("/ingest")) { - return Promise.resolve({ - status: 200, - data: { ok: true }, - }) as any; - } - return originalAxiosPost.call(axios, url, data, config); - }); + const session = server.neo4jDriver.session(); + try { + await session.run("MATCH (a:AwarenessOutbox) DETACH DELETE a"); + } finally { + await session.close(); + } }); - it("dispatches an ingest packet stamped w3ds-file-v1", async () => { - const content = Buffer.from("hello world").toString("base64"); - + it("atomically records the stored w3ds-file-v1 payload", async () => { + const body = "hello world"; const result = await makeGraphQLRequest( server, UPLOAD_FILE, @@ -125,81 +96,35 @@ describe("uploadFile awareness ingest", () => { input: { filename: "greeting.txt", contentType: "text/plain", - content, + content: Buffer.from(body).toString("base64"), acl: ["*"], }, }, authHeaders, ); - expect(result.uploadFile.errors ?? []).toEqual([]); - const metaEnvelopeId = result.uploadFile.metaEnvelopeId; - expect(metaEnvelopeId).toBeTruthy(); - - // notifyAwareness is fire-and-forget; give it a moment to run. - await new Promise((resolve) => setTimeout(resolve, 1000)); - - const calls = ingestCalls(); - expect(calls.length).toBeGreaterThan(0); - - const payload = calls[0][1]; - expect(payload.schemaId).toBe(FILE_SCHEMA_ID); - expect(payload.schemaId).toBe("w3ds-file-v1"); - expect(payload.w3id).toBe(evault.w3id); - expect(payload.operation).toBe("create"); - // The packet id is the MetaEnvelope id, so a consumer can address the - // blob as w3ds://file?id=/ without another round trip. - expect(payload.id).toBe(metaEnvelopeId); - // Origin is forwarded so AaaS can skip delivering back to the uploader. - expect(payload.requestingPlatform).toBe(TEST_PLATFORM); - }); - it("sends the stored payload verbatim, including blobKey", async () => { - const body = "second file"; - const content = Buffer.from(body).toString("base64"); - - const result = await makeGraphQLRequest( - server, - UPLOAD_FILE, - { - input: { - filename: "notes.txt", - contentType: "text/plain", - content, - acl: ["*"], - }, + const events = await outboxEvents(server); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + packetId: result.uploadFile.metaEnvelopeId, + schemaId: FILE_SCHEMA_ID, + w3id: evault.w3id, + operation: "create", + requestingPlatform: TEST_PLATFORM, + status: "pending", + data: { + filename: "greeting.txt", + contentType: "text/plain", + size: Buffer.byteLength(body), + blobKey: expect.stringContaining("greeting.txt"), + publicUrl: result.uploadFile.publicUrl, + uploadedAt: expect.any(String), }, - authHeaders, - ); - - const { metaEnvelopeId, publicUrl } = result.uploadFile; - await new Promise((resolve) => setTimeout(resolve, 1000)); - - const payload = ingestCalls()[0][1]; - - // Packet data must equal what a consumer reads back via - // metaEnvelope(id) — any divergence is a trap for consumers that diff - // the two, and would muddy the contentHash dedupe in AaaS. - expect(payload.data).toEqual({ - filename: "notes.txt", - contentType: "text/plain", - size: Buffer.byteLength(body), - blobKey: expect.stringContaining("notes.txt"), - publicUrl, - uploadedAt: expect.any(String), }); - - const stored = await makeGraphQLRequest( - server, - `query Get($id: ID!) { metaEnvelope(id: $id) { id ontology parsed } }`, - { id: metaEnvelopeId }, - authHeaders, - ); - expect(stored.metaEnvelope.ontology).toBe(FILE_SCHEMA_ID); - expect(stored.metaEnvelope.parsed).toEqual(payload.data); }); - it("does not dispatch when the upload is rejected", async () => { + it("does not create an event when input validation rejects the upload", async () => { const result = await makeGraphQLRequest( server, UPLOAD_FILE, @@ -213,34 +138,26 @@ describe("uploadFile awareness ingest", () => { }, authHeaders, ); - expect(result.uploadFile.errors?.[0]?.code).toBe("INVALID_CONTENT"); - expect(result.uploadFile.metaEnvelopeId).toBeFalsy(); - - await new Promise((resolve) => setTimeout(resolve, 1000)); - expect(ingestCalls()).toHaveLength(0); + expect(await outboxEvents(server)).toHaveLength(0); }); - it("does not dispatch when the object store write fails", async () => { + it("does not create an event when object storage fails first", async () => { s3Send.mockRejectedValueOnce(new Error("spaces unavailable")); - const result = await makeGraphQLRequest( server, UPLOAD_FILE, { input: { - filename: "doomed.txt", + filename: "failed.txt", contentType: "text/plain", - content: Buffer.from("nope").toString("base64"), + content: Buffer.from("body").toString("base64"), acl: ["*"], }, }, authHeaders, ); - expect(result.uploadFile.errors?.[0]?.code).toBe("UPLOAD_FAILED"); - - await new Promise((resolve) => setTimeout(resolve, 1000)); - expect(ingestCalls()).toHaveLength(0); + expect(await outboxEvents(server)).toHaveLength(0); }); }); diff --git a/services/awareness-service/README.md b/services/awareness-service/README.md index c9f9f8d70..fca1694de 100644 --- a/services/awareness-service/README.md +++ b/services/awareness-service/README.md @@ -12,16 +12,17 @@ subscription matching and retrying webhook delivery. ## What it does -1. **Ingest** — `POST /ingest` receives every awareness packet from evault-core - (shared-secret auth) and persists it. +1. **Ingest** — eVault's transactional Neo4j outbox retries `POST /ingest` until + AaaS atomically persists the immutable event and all matching deliveries. 2. **Poll** — `GET /api/packets` lets approved consumers query packet history by ontology, eVault and time range, with cursor pagination. A single packet can also be fetched directly by its MetaEnvelope id with `GET /api/packets/:id`. 3. **Subscribe** — `/api/subscriptions` registers webhook subscriptions filtered - by ontology and eVault. Delivered payloads match the legacy evault-core - webhook format exactly. -4. **Deliver** — a background engine drains the delivery queue with exponential - backoff; exhausted deliveries land in a dead-letter table. + by ontology and eVault. Delivered payloads preserve the legacy evault-core + fields and add event identity, operation, version, and timestamp metadata. +4. **Deliver** — a lease-based background worker drains the queue with bounded + database operations and exponential backoff for 24 hours; failures then land + in a dead-letter table. 5. **Portal** — platforms log in with W3DS, apply for access, and admins (`AAAS_ADMIN_ENAMES`) approve them. Approved consumers get API keys. @@ -36,9 +37,12 @@ pnpm --filter awareness-service-api migration:run # 3. One-time backfill from evault-core's Neo4j (same node) pnpm --filter awareness-service-api backfill -# 4. Start the API (also seeds catch-all subscriptions on launch) +# 4. Start AaaS. The API and delivery worker run in this one process. pnpm --filter awareness-service-api dev +# Production: +pnpm --filter awareness-service-api start + # 5. Start the portal pnpm --filter awareness-portal dev ``` @@ -46,6 +50,10 @@ pnpm --filter awareness-portal dev Then set `AWARENESS_SERVICE_URL` and `AWARENESS_INGEST_SECRET` for evault-core so it forwards packets here. +`GET /ready` verifies Postgres, migration state, and worker heartbeat. `GET +/metrics` exposes queue age, queue states, expired leases, and heartbeat age in +Prometheus format. Deployments must run migrations before either process starts. + ## API documentation The running API serves an interactive [Scalar](https://github.com/scalar/scalar) @@ -63,5 +71,7 @@ On launch and periodically thereafter, AaaS reconciles a catch-all subscription for every platform currently in the registry, so existing and newly registered webhook receivers keep getting every packet at `/api/webhook` with no change. `AWARENESS_REGISTRY_SYNC_MS` controls the interval (default 60000; set -to 0 to disable periodic reconciliation). Non-registry consumers can narrow -their own subscriptions to specific ontologies / eVaults. +to 0 to disable periodic reconciliation). A repaired or newly created catch-all +also receives the previous 24 hours, closing the registry-sync race. +Non-registry consumers can narrow their own subscriptions to specific +ontologies / eVaults. diff --git a/services/awareness-service/api/src/services/DeliveryEngine.spec.ts b/services/awareness-service/api/src/services/DeliveryEngine.spec.ts new file mode 100644 index 000000000..96ccc3178 --- /dev/null +++ b/services/awareness-service/api/src/services/DeliveryEngine.spec.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { config } from "../config"; +import { DeliveryEngine } from "./DeliveryEngine"; + +class ProbeEngine extends DeliveryEngine { + ticks = 0; + + protected override async writeHeartbeat(): Promise { + // This test isolates scheduler recovery from the database. + } + + protected override async tick(): Promise { + this.ticks += 1; + if (this.ticks === 1) { + await new Promise(() => undefined); + } + } +} + +describe("DeliveryEngine scheduler", () => { + const originalPoll = config.deliveryPollMs; + const originalDeadline = config.deliveryBatchTimeoutMs; + + afterEach(() => { + config.deliveryPollMs = originalPoll; + config.deliveryBatchTimeoutMs = originalDeadline; + vi.restoreAllMocks(); + }); + + it("continues polling after a tick promise never resolves", async () => { + config.deliveryPollMs = 2; + config.deliveryBatchTimeoutMs = 10; + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + const engine = new ProbeEngine(); + engine.start(); + await new Promise((resolve) => setTimeout(resolve, 40)); + await engine.stop(); + + // The old global `running` latch stayed true forever after tick 1. + expect(engine.ticks).toBeGreaterThan(1); + }); +}); diff --git a/services/awareness-service/api/src/services/IngestService.integration.spec.ts b/services/awareness-service/api/src/services/IngestService.integration.spec.ts new file mode 100644 index 000000000..f613e06c6 --- /dev/null +++ b/services/awareness-service/api/src/services/IngestService.integration.spec.ts @@ -0,0 +1,204 @@ +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { StartedPostgreSqlContainer } from "@testcontainers/postgresql"; + +describe("durable awareness ingest", () => { + let container: StartedPostgreSqlContainer; + let dataSource: any; + let IngestService: any; + let Consumer: any; + let Subscription: any; + let AwarenessEvent: any; + let Delivery: any; + let EventIdConflictError: any; + let queueStats: () => Promise; + let DeliveryEngine: any; + + beforeAll(async () => { + container = await new PostgreSqlContainer("postgres:16-alpine").start(); + process.env.AWARENESS_DATABASE_URL = container.getConnectionUri(); + process.env.PUBLIC_REGISTRY_URL = ""; + ({ AppDataSource: dataSource } = await import( + "../database/data-source" + )); + ({ IngestService, EventIdConflictError } = await import( + "./IngestService" + )); + ({ queueStats } = await import("../controllers/SystemController")); + ({ DeliveryEngine } = await import("./DeliveryEngine")); + ({ Consumer } = await import("../database/entities/Consumer")); + ({ Subscription } = await import("../database/entities/Subscription")); + ({ AwarenessEvent } = await import( + "../database/entities/AwarenessEvent" + )); + ({ Delivery } = await import("../database/entities/Delivery")); + await dataSource.initialize(); + await dataSource.runMigrations(); + + const consumer = await dataSource.getRepository(Consumer).save({ + ename: "catchall:test.example", + name: "test", + status: "approved", + webhookBaseUrl: "https://test.example", + approvedAt: new Date(), + }); + await dataSource.getRepository(Subscription).save({ + consumerId: consumer.id, + targetUrl: "https://test.example/api/webhook", + ontologyFilter: [], + evaultFilter: [], + isCatchAll: true, + active: true, + secret: null, + }); + }, 120_000); + + afterAll(async () => { + if (dataSource?.isInitialized) await dataSource.destroy(); + if (container) await container.stop(); + }); + + it("migrates a fresh database and commits event plus deliveries atomically", async () => { + const service = new IngestService(); + const result = await service.ingest({ + eventId: "event-atomic-1", + id: "envelope-atomic", + schemaId: "schema-test", + w3id: "@owner", + data: { state: "A" }, + operation: "create", + streamVersion: 1, + occurredAt: new Date().toISOString(), + }); + + expect(result).toMatchObject({ + eventId: "event-atomic-1", + duplicate: false, + deliveriesQueued: 1, + }); + expect(await dataSource.getRepository(AwarenessEvent).count()).toBe(1); + expect(await dataSource.getRepository(Delivery).count()).toBe(1); + + const duplicate = await service.ingest({ + eventId: "event-atomic-1", + id: "envelope-atomic", + schemaId: "schema-test", + w3id: "@owner", + data: { state: "A" }, + operation: "create", + streamVersion: 1, + }); + expect(duplicate.duplicate).toBe(true); + expect(await dataSource.getRepository(Delivery).count()).toBe(1); + + await expect( + service.ingest({ + eventId: "event-atomic-1", + id: "envelope-atomic", + schemaId: "schema-test", + w3id: "@owner", + data: { state: "different" }, + operation: "create", + streamVersion: 1, + }), + ).rejects.toBeInstanceOf(EventIdConflictError); + + const stats = await queueStats(); + expect(Number(stats.pending)).toBeGreaterThan(0); + expect(Number(stats.oldest_pending_seconds)).toBeGreaterThanOrEqual(0); + }); + + it("does not collapse a legitimate A to B to A event sequence", async () => { + const service = new IngestService(); + for (const [index, state] of ["A", "B", "A"].entries()) { + await service.ingest({ + eventId: `event-sequence-${index + 1}`, + id: "envelope-sequence", + schemaId: "schema-test", + data: { state }, + operation: index === 0 ? "create" : "update", + streamVersion: index + 1, + }); + } + const events = await dataSource.getRepository(AwarenessEvent).find({ + where: { packetId: "envelope-sequence" }, + }); + const deliveries = await dataSource.getRepository(Delivery).find({ + where: { packetId: "envelope-sequence" }, + }); + expect(events).toHaveLength(3); + expect(deliveries).toHaveLength(3); + }); + + it("reclaims expired leases with token fencing and preserves stream order", async () => { + const service = new IngestService(); + await service.ingest({ + eventId: "event-lease-1", + id: "envelope-lease", + schemaId: "schema-test", + data: { version: 1 }, + streamVersion: 1, + }); + await service.ingest({ + eventId: "event-lease-2", + id: "envelope-lease", + schemaId: "schema-test", + data: { version: 2 }, + operation: "update", + streamVersion: 2, + }); + + const deliveryRepo = dataSource.getRepository(Delivery); + const [first, second] = await deliveryRepo.find({ + where: { packetId: "envelope-lease" }, + order: { createdAt: "ASC" }, + }); + const staleToken = "00000000-0000-4000-8000-000000000001"; + await deliveryRepo.update(first.id, { + createdAt: new Date(Date.now() - 2_000), + status: "delivering", + leaseToken: staleToken, + leaseOwner: "dead-worker", + leaseExpiresAt: new Date(Date.now() - 1_000), + }); + await deliveryRepo.update(second.id, { + createdAt: new Date(Date.now() - 1_000), + status: "pending", + nextAttemptAt: new Date(Date.now() - 1_000), + }); + + const engine = new DeliveryEngine() as any; + const firstClaim = await engine.claimBatch(); + const streamClaims = firstClaim.filter( + (delivery: any) => delivery.packetId === "envelope-lease", + ); + expect(streamClaims.map((delivery: any) => delivery.eventId)).toEqual([ + "event-lease-1", + ]); + + const staleCompletion = await deliveryRepo.update( + { id: first.id, leaseToken: staleToken }, + { status: "delivered" }, + ); + expect(staleCompletion.affected).toBe(0); + + const activeLease = streamClaims[0].leaseToken; + await deliveryRepo.update( + { id: first.id, leaseToken: activeLease }, + { + status: "delivered", + leaseToken: null, + leaseOwner: null, + leaseExpiresAt: null, + }, + ); + const secondClaim = await engine.claimBatch(); + expect( + secondClaim + .filter( + (delivery: any) => delivery.packetId === "envelope-lease", + ) + .map((delivery: any) => delivery.eventId), + ).toEqual(["event-lease-2"]); + }); +}); diff --git a/skills/w3ds/SKILL.md b/skills/w3ds/SKILL.md index baff1f5e5..d66ac7708 100644 --- a/skills/w3ds/SKILL.md +++ b/skills/w3ds/SKILL.md @@ -72,7 +72,7 @@ Before reporting a W3DS task complete, check every line: - [ ] Every `schemaId` was resolved from the Ontology service in this session, not recalled. - [ ] Every new entity type has an ontology, a resolving `ownerEnamePath`, and a write path to the owner's eVault. - [ ] `handleChange` is called after every write to a mapped table — including writes from migrations, seeds, admin paths and background jobs. -- [ ] The webhook controller is idempotent on the global `id`, and returns 200 for ontologies the platform does not consume. +- [ ] The webhook controller deduplicates on `eventId`, upserts by global `id`, and returns 200 for ontologies the platform does not consume. - [ ] Nothing was invented: no UUID, endpoint path, GraphQL field, mapping directive or ACL verb that was not verified — or, if unverifiable, each is flagged in the response and marked in code. - [ ] The reconstructability test was applied to anything newly persisted, and the answer stated. - [ ] If the work touched a platform repository: no managed `.w3ds/platform.json` field was hand-edited, no history was rewritten, and no key material was committed. @@ -149,7 +149,7 @@ Uncertain? Fetch the relevant page from `https://docs.w3ds.metastate.foundation` - **Ontology vs schema**: "Ontology" here means a specific JSON Schema published by the Ontology service and referenced by its `schemaId` (a W3ID). Not the semantic-web sense of the word. - **Platform vs post-platform**: A platform participates in W3DS via a Web3 Adapter and a `/api/webhook` endpoint. A post-platform operates in "dataless" mode — it doesn't own the data, users' eVaults do. - **`w3ds-file-v1` vs `File` ontology**: `w3ds-file-v1` is the low-level storage envelope created by `uploadFile` for blob dereferencing. The `File` ontology is a higher-level platform record for file-manager / esigner style apps. Not interchangeable — different field names, different layer. Detail in [reference/protocols.md](reference/protocols.md). -- **Awareness Protocol vs AaaS**: Awareness Protocol is the prototype-level fire-and-forget fanout from eVault-core. AaaS is the production-grade replacement with subscriptions, persistence, retries, and a dead-letter queue. +- **Awareness Protocol vs AaaS**: Awareness Protocol is the packet/receiver contract. AaaS is its durable delivery system: eVault transactional outbox, immutable events, subscriptions, at-least-once retries, and dead letters. - **`storeMetaEnvelope` / `updateMetaEnvelopeById`**: Legacy GraphQL mutation names, still used internally by the Web3 Adapter's `EVaultClient`. External integrations should use `createMetaEnvelope` / `updateMetaEnvelope` / `removeMetaEnvelope`. ## Working style @@ -157,8 +157,8 @@ Uncertain? Fetch the relevant page from `https://docs.w3ds.metastate.foundation` - Always resolve the eVault URL for a user via the Registry before hitting `/graphql` or `/whois`. Never hardcode eVault URLs; cache the resolution, revalidate it, and evict on a failed `HEAD /whois`. - Every GraphQL and HTTP call to eVault needs `X-ENAME`. Missing this header is the most common cause of 400s. - Two ACL models coexist. The `_acl` block gives per-verb grants (READ/CREATE/UPDATE/DELETE bitmask), denials, and ontology conditions, and is authoritative where present. The legacy `acl` string array is all-or-nothing except `["*"]` and still applies to records with no `_acl`. Do not describe ACLs as all-or-nothing without that distinction — see [reference/evault.md](reference/evault.md). -- Webhook delivery is fire-and-forget and prototype-level: no retries, no ordering, no at-least-once. Make the webhook controller **idempotent** on global `id`. -- After `storeMetaEnvelope` there is a 3-second delay before webhook fanout to prevent ping-pong. `updateMetaEnvelopeById` fanout is immediate. +- Webhook delivery is at least once. Make the webhook controller **idempotent on `eventId`**, not global MetaEnvelope `id`; distinct updates intentionally share the same `id`. +- Delivery is ordered per subscription and MetaEnvelope, not globally. eVault retries AaaS ingestion until acknowledged; AaaS retries subscribers for 24 hours before dead-lettering. - Do not mirror what you can already observe. If a record reaches you through the Awareness Protocol, subscribe to it rather than writing a second envelope to make it visible. - Building a platform they intend to publish? Say early that it belongs in a GitW3 repository — a plain repository import is not the same as the guided port flow, and retrofitting an identity after the fact is worse than starting there. - Never commit `w3ds-deployment-key.json`, a platform token, a migration proof or a personal access token. If asked to paste key material anywhere, stop and say why. diff --git a/skills/w3ds/reference/dev-setup.md b/skills/w3ds/reference/dev-setup.md index 7e703e36f..91bf3380f 100644 --- a/skills/w3ds/reference/dev-setup.md +++ b/skills/w3ds/reference/dev-setup.md @@ -123,10 +123,11 @@ Same pattern for `w3ds://sign` — paste the URI, click Perform, watch your call Check in order: -1. Is your platform registered? Query `GET http://localhost:4321/list` and confirm your platform's URL is in the response. -2. If the write is a **create** (not update), remember there is a **3-second delay** before fanout. Wait, then re-check. -3. Is your `/api/webhook` endpoint publicly reachable from eVault-core? (In local dev, `localhost` works. In containers, use the service name or host.docker.internal.) -4. Does the packet's `schemaId` match a mapping in your Web3 Adapter? If not, your controller correctly drops it — that's expected. +1. Does eVault `GET /ready` report an active/configured awareness dispatcher? Check `evault_awareness_outbox_*` metrics for a source backlog. +2. Does AaaS `GET /ready` report current migrations and a fresh worker heartbeat? Check `aaas_oldest_pending_seconds` and `aaas_expired_leases`. +3. Is your platform registered and does AaaS have an active catch-all or matching granular subscription? +4. Is your `/api/webhook` endpoint reachable from the AaaS worker? (In containers, use the service name or `host.docker.internal`.) +5. Does the packet's `schemaId` match a mapping in your Web3 Adapter? Unknown ontologies should be acknowledged with 200. ### Duplicate entities on sync diff --git a/skills/w3ds/reference/evault.md b/skills/w3ds/reference/evault.md index 5f86ef654..9b25fa53c 100644 --- a/skills/w3ds/reference/evault.md +++ b/skills/w3ds/reference/evault.md @@ -328,18 +328,18 @@ Special cases: ## Webhook delivery (Awareness Protocol) -After a `createMetaEnvelope` (or legacy `storeMetaEnvelope`), eVault: +After a create, update, delete, file upload, or binding-document mutation: -1. Persists to Neo4j. -2. Waits **3 seconds** (create only — `updateMetaEnvelope` fires immediately). -3. `GET /platforms` on the Registry → list of platform base URLs. -4. Filters out the requesting platform (identified from the Bearer token's `platform` claim, URL-normalized). -5. `POST /api/webhook` on every remaining platform in parallel. 5s timeout per call. No retries. Fire-and-forget. +1. eVault atomically persists the data and a uniquely identified awareness outbox event in Neo4j. +2. Its dispatcher retries `POST /ingest` until AaaS durably acknowledges the event. +3. AaaS atomically records the immutable event and matching delivery rows, excluding the requesting platform by normalized origin. +4. Lease-based AaaS workers send `POST /api/webhook`. Non-2xx responses and timeouts retry with backoff for 24 hours, then dead-letter. Payload: ```json { + "eventId": "7fd6c06c-80ae-4137-9d62-c15af53f92cf", "id": "a1b2c3d4-...", "w3id": "@user-a.w3id", "schemaId": "", @@ -349,7 +349,10 @@ Payload: "authorId": "@e4d909c2-...", "createdAt": "2025-01-24T10:00:00Z" }, - "evaultPublicKey": "z..." + "evaultPublicKey": "z...", + "operation": "update", + "streamVersion": 4, + "occurredAt": "2026-09-15T03:00:00.000Z" } ``` diff --git a/skills/w3ds/reference/protocols.md b/skills/w3ds/reference/protocols.md index dad5ba270..a4e41b470 100644 --- a/skills/w3ds/reference/protocols.md +++ b/skills/w3ds/reference/protocols.md @@ -111,26 +111,27 @@ Detail: [Signing](https://docs.w3ds.metastate.foundation/docs/W3DS%20Protocol/Si ## Awareness Protocol (webhooks) -Prototype-level fanout from eVault-core to every registered platform after a write. Fire-and-forget. Source: [Awareness Protocol](https://docs.w3ds.metastate.foundation/docs/W3DS%20Protocol/Awareness-Protocol). +Durable at-least-once change delivery from eVault through AaaS to matching platform subscriptions. Source: [Awareness Protocol](https://docs.w3ds.metastate.foundation/docs/W3DS%20Protocol/Awareness-Protocol). ### When it fires -- After `createMetaEnvelope` (legacy `storeMetaEnvelope`): **3-second delay**, then fanout. The delay gives eVault time to reliably identify the requesting platform (from the Bearer token's `platform` claim) so it can exclude that platform from the fanout list. Without the delay you get "webhook ping-pong." -- After `updateMetaEnvelope` (legacy `updateMetaEnvelopeById`): **immediate** fanout. +- Create, update, delete, file upload, and binding-document mutations atomically create outbox events. +- The requesting platform comes from the Bearer token's `platform` claim and is excluded by normalized origin to prevent webhook ping-pong. ### Delivery mechanics -1. eVault `GET /platforms` on the Registry → list of platform base URLs. -2. Filter out the requesting platform (normalized URL compare). -3. `POST {platformUrl}/api/webhook` on each remaining platform in parallel. -4. 5-second timeout per call. -5. `Promise.allSettled` — one failure does not affect others. -6. No retries. Failures are logged but do not block the mutation. +1. eVault commits user data and `AwarenessOutbox` event in one Neo4j transaction. +2. An expiring-lease dispatcher retries AaaS `/ingest` until acknowledged, including across restarts. +3. AaaS commits the immutable event and matching Postgres delivery rows in one transaction. +4. Expiring-lease workers deliver independent streams concurrently with a 5-second request timeout. +5. Non-2xx/timeouts retry with jittered backoff for 24 hours, then dead-letter for admin replay. +6. Events are ordered per subscription and MetaEnvelope. There is no global cross-stream order. ### Packet format ```json { + "eventId": "7fd6c06c-80ae-4137-9d62-c15af53f92cf", "id": "a1b2c3d4-...", "w3id": "@e4d909c2-...", "schemaId": "", @@ -139,11 +140,14 @@ Prototype-level fanout from eVault-core to every registered platform after a wri "mediaUrls": [], "authorId": "@e4d909c2-...", "createdAt": "2025-01-24T10:00:00Z" - } + }, + "operation": "update", + "streamVersion": 4, + "occurredAt": "2026-09-15T03:00:00.000Z" } ``` -Every platform receives every packet (broadcast). It is the platform's responsibility to inspect `schemaId` and drop packets it doesn't consume. A future revision will support ontology subscriptions and by-reference delivery. +Registry-managed compatibility subscriptions receive every packet. Other consumers can filter subscriptions by ontology and eVault. Receivers must inspect `schemaId` and return 200 for packets they do not consume. ### Platform contract @@ -151,23 +155,23 @@ Every platform participating in W3DS MUST implement `POST /api/webhook` and: 1. Find the mapping whose `schemaId` matches the packet's `schemaId`. 2. `adapter.fromGlobal({ data: body.data, mapping })` → local-shaped data. -3. Look up existing local ID for `body.id`; if found, update; otherwise create and persist the `(globalId, localId)` mapping. -4. Return 200. +3. Deduplicate the event by `body.eventId`. +4. Look up existing local ID for `body.id`; if found, apply the update/delete, otherwise create and persist the `(globalId, localId)` mapping. +5. Return 200, including for duplicate events and unknown ontologies. -**Idempotency required**: the same `body.id` may arrive more than once (network retries, misbehaving eVault). Never create a second local entity for the same global ID. +**Idempotency required**: the same `body.eventId` may arrive more than once. Never apply one event twice. Do not suppress all repeats of `body.id`: legitimate create/update/delete events for one MetaEnvelope share that id. Full webhook controller code in [platform.md § Webhook controller](platform.md#webhook-controller). -### Limitations to know +### Semantics to know -- No retries. No ordering. No at-least-once guarantee. -- Recipient set = whatever `GET /platforms` returns. That is a prototype shortcut. - -For production, use Awareness-as-a-Service. +- At-least-once delivery can duplicate events; dedupe with `eventId`. +- Ordering is local to a subscription/MetaEnvelope stream, not global. +- Automatic subscriber retries stop after 24 hours and require dead-letter replay. ### Awareness-as-a-Service (AaaS) -Production-grade replacement layer. Source: [Awareness as a Service (AaaS)](https://docs.w3ds.metastate.foundation/docs/Services/Awareness-as-a-Service). Key differences vs raw Awareness Protocol: +The production delivery layer for the Awareness Protocol. Source: [Awareness as a Service (AaaS)](https://docs.w3ds.metastate.foundation/docs/Services/Awareness-as-a-Service). Key capabilities: - `POST /ingest` accepts packets from eVault-core. - `GET /api/packets` — poll query with filters (ontology, eVault, time). diff --git a/skills/w3ds/reference/w3ds-native.md b/skills/w3ds/reference/w3ds-native.md index 7bc6d3db0..2495c6ce8 100644 --- a/skills/w3ds/reference/w3ds-native.md +++ b/skills/w3ds/reference/w3ds-native.md @@ -120,9 +120,9 @@ Common thread: none of it is data about a user that a user would expect to take **Wrong** — write locally, then immediately read back the eVault-derived version and assume it is there. -**Right** — design for last-write-wins, no ordering, no at-least-once delivery, a delay after create before fanout (immediate on update), and the requesting platform excluded from its own fanout. Idempotent on the global `id`, tolerant of a record that has not arrived. +**Right** — design for last-write-wins and eventual consistency. Delivery is at least once, ordered only within a subscription/MetaEnvelope stream, and excludes the requesting platform. Deduplicate on `eventId` and tolerate a record that has not arrived yet. -**Why** — the Awareness Protocol is prototype-level and fire-and-forget. Anything user-visible that assumes otherwise breaks intermittently and unreproducibly. +**Why** — durable retries prevent silent loss, but cross-stream timing is still asynchronous and duplicate delivery is an intentional consequence of at-least-once semantics. ## Proposing a new ontology @@ -175,5 +175,5 @@ For an existing platform, in order: 2. For each, find the `mapping.json`. No mapping → is it operational state, or claimed user data? 3. For each mapping, check `ownerEnamePath` resolves to the data subject for every row, not just the common case. 4. Find every write path per mapped table — migrations, seeds, admin endpoints, background jobs included — and confirm each reaches `handleChange`. -5. Check the webhook controller is idempotent on global `id` and 200s ontologies it does not consume. +5. Check the webhook controller deduplicates on `eventId`, upserts by global `id`, and 200s ontologies it does not consume. 6. Run the reconstructability test over the whole set and state what would be lost.