From eed9d3ef6bc4dbb5e1c055b0c4ac5937e9177ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:10:43 +0100 Subject: [PATCH 01/19] feat(core): share one lease among the background jobs and start them over after a restore --- packages/core/src/adapter/sql/person-store.ts | 14 ++ .../src/adapter/sql/personal-key-store.ts | 16 ++ .../core/src/adapter/storage/person-store.ts | 9 + .../src/adapter/storage/personal-key-store.ts | 9 + .../testing/checkpoint-store.contract.ts | 87 +++++++ .../adapter/testing/person-store.contract.ts | 19 ++ .../testing/personal-key-store.contract.ts | 23 ++ packages/core/src/kernel/app.ts | 6 + .../src/kernel/dead-letters/retention.test.ts | 78 +++++- .../core/src/kernel/dead-letters/retention.ts | 46 +++- .../core/src/kernel/dispatch/dispatcher.ts | 8 + .../personal-data/backfill-walk.test.ts | 7 +- .../core/src/kernel/personal-data/backfill.ts | 62 ++--- .../kernel/personal-data/event-store.test.ts | 4 + .../kernel/personal-data/reconciliation.ts | 51 ++++ .../kernel/personal-data/restriction.test.ts | 2 + .../kernel/personal-data/retention.test.ts | 126 ++++++++++ .../src/kernel/personal-data/retention.ts | 60 ++++- .../src/kernel/personal-data/rotation.test.ts | 163 ++++++++++++- .../core/src/kernel/personal-data/rotation.ts | 78 +++++- .../src/kernel/personal-data/wrapped-keys.ts | 2 + packages/core/src/kernel/scheduler/worker.ts | 39 ++- packages/core/src/kernel/shared/lease.ts | 97 ++++++++ .../src/kernel/shared/restores-app.test.ts | 227 ++++++++++++++++++ packages/core/src/kernel/shared/restores.ts | 100 ++++++++ packages/core/src/memory/person-store.ts | 15 ++ .../core/src/memory/personal-key-store.ts | 11 + packages/core/src/memory/transaction.ts | 6 + 28 files changed, 1310 insertions(+), 55 deletions(-) create mode 100644 packages/core/src/kernel/shared/lease.ts create mode 100644 packages/core/src/kernel/shared/restores-app.test.ts create mode 100644 packages/core/src/kernel/shared/restores.ts diff --git a/packages/core/src/adapter/sql/person-store.ts b/packages/core/src/adapter/sql/person-store.ts index a8106686..da199a40 100644 --- a/packages/core/src/adapter/sql/person-store.ts +++ b/packages/core/src/adapter/sql/person-store.ts @@ -101,5 +101,19 @@ export const createSqlPersonStore: CreateSqlPersonStoreFunction = ({ db, table, [after.toISOString()], ), ), + countIdle: async ({ activeBefore }) => { + const [row] = await db.all( + `SELECT count(*) AS "count" FROM ${table} WHERE ${IDLE}${activeBefore === undefined ? "" : ` AND "last_active_at" <= ${p(1)}`}`, + activeBefore === undefined ? [] : [activeBefore.toISOString()], + ); + return Number(row?.count ?? 0); + }, + releaseHolds: async () => + ( + await db.all( + `UPDATE ${table} SET "held_until" = NULL, "attempts" = 0 WHERE "held_until" IS NOT NULL RETURNING "person"`, + [], + ) + ).length, }; }; diff --git a/packages/core/src/adapter/sql/personal-key-store.ts b/packages/core/src/adapter/sql/personal-key-store.ts index 4d347c59..6df06240 100644 --- a/packages/core/src/adapter/sql/personal-key-store.ts +++ b/packages/core/src/adapter/sql/personal-key-store.ts @@ -107,5 +107,21 @@ export const createSqlPersonalKeyStore: CreateSqlPersonalKeyStoreFunction = ({ ), ); }, + countWrappedUnder: async ({ kids }) => { + const under = wrappedUnder(kids); + if (under === null) return 0; + const [row] = await db.all( + `SELECT count(*) AS "count" FROM ${table} WHERE ${under.condition}`, + under.values, + ); + return Number(row?.count ?? 0); + }, + releaseHolds: async () => + ( + await db.all( + `UPDATE ${table} SET "held_until" = NULL, "attempts" = 0 WHERE "held_until" IS NOT NULL RETURNING "person"`, + [], + ) + ).length, }; }; diff --git a/packages/core/src/adapter/storage/person-store.ts b/packages/core/src/adapter/storage/person-store.ts index 2af291e7..92d9ea48 100644 --- a/packages/core/src/adapter/storage/person-store.ts +++ b/packages/core/src/adapter/storage/person-store.ts @@ -98,4 +98,13 @@ export interface PersonStore { * is held past it. */ nextHoldEnd(after: Date): Promise; + /** + * How many persons `listIdle` lists with `activeBefore`, whatever their holds. + */ + countIdle(args: Pick): Promise; + /** + * Clears every person's hold, and resolves to how many it cleared: what the runtime does once the + * database went back to a copy, whose holds count failures the copy undid. + */ + releaseHolds(): Promise; } diff --git a/packages/core/src/adapter/storage/personal-key-store.ts b/packages/core/src/adapter/storage/personal-key-store.ts index 3ffd0885..36f71bde 100644 --- a/packages/core/src/adapter/storage/personal-key-store.ts +++ b/packages/core/src/adapter/storage/personal-key-store.ts @@ -117,4 +117,13 @@ export interface PersonalKeyStore { rewrap(person: string, from: StoredPersonalKey, to: RewrappedKey): Promise; hold(person: string, from: StoredPersonalKey, hold: Hold): Promise; nextHoldEnd(args: NextKeyHoldEndArgs): Promise; + /** + * How many keys are wrapped under one of `kids`, whatever their holds. + */ + countWrappedUnder(args: Pick): Promise; + /** + * Clears every key's hold, and resolves to how many it cleared: what the runtime does once the + * database went back to a copy, whose holds count failures the copy undid. + */ + releaseHolds(): Promise; } diff --git a/packages/core/src/adapter/testing/checkpoint-store.contract.ts b/packages/core/src/adapter/testing/checkpoint-store.contract.ts index c142eceb..40878bcf 100644 --- a/packages/core/src/adapter/testing/checkpoint-store.contract.ts +++ b/packages/core/src/adapter/testing/checkpoint-store.contract.ts @@ -1,4 +1,13 @@ import { beforeEach, describe, expect, it } from "vitest"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { silentLogger } from "../../contracts/logger.ts"; +import { createLease, LEASE_MS } from "../../kernel/shared/lease.ts"; +import { + createRestoreWatch, + GENERATION_INTERVAL_MS, + RESTORE_GENERATION, + RESTORES_FOUND, +} from "../../kernel/shared/restores.ts"; import type { CheckpointStore } from "../storage/checkpoint-store.ts"; export interface CheckpointStoreContractArgs { @@ -74,5 +83,83 @@ export const checkpointStoreContract: CheckpointStoreContractFunction = ({ creat expect(await store.compareAndSet("projection:order-summary", 40, 12)).toBe(true); expect(await store.get("projection:order-summary")).toBe(12); }); + + // What the runtime's background jobs build on the store: a lease that one of several + // instances holds, and a watch that tells a database gone back to an earlier copy. + describe("for the runtime's background jobs", () => { + const start = new Date("2026-01-01T00:00:00.000Z").getTime(); + const leaseOver = (clock = createFixedClock(new Date(start))) => + createLease({ checkpoints: store, row: "job:lease", clock, logger: silentLogger }); + + it("lets one of two instances taking a lease at once hold it", async () => { + const [one, other] = [leaseOver(), leaseOver()]; + const taken = await Promise.all([one.take(start), other.take(start)]); + expect(taken.filter(Boolean)).toHaveLength(1); + const loser = taken[0] ? other : one; + expect(await loser.take(start)).toBe(false); + expect(loser.heldElsewhereUntil()).toBe(start + LEASE_MS); + }); + + it("hands a lease that lapsed mid-way to another, and stops the instance that held it", async () => { + const one = leaseOver(); + const other = leaseOver(); + expect(await one.take(start)).toBe(true); + expect(await other.take(start + LEASE_MS - 1)).toBe(false); + expect(await other.take(start + LEASE_MS)).toBe(true); + expect(await one.take(start + LEASE_MS + 1)).toBe(false); + const renewed = start + LEASE_MS + LEASE_MS / 2 + 1; + expect(await other.take(renewed)).toBe(true); + expect(await store.get("job:lease")).toBe(renewed + LEASE_MS); + }); + + it("gives a lease back so another takes it at once", async () => { + const one = leaseOver(); + expect(await one.take(start)).toBe(true); + await one.release(); + expect(await store.get("job:lease")).toBe(start); + expect(await leaseOver().take(start)).toBe(true); + }); + + it("takes back as its own a lease of its own a restored copy brought back", async () => { + const one = leaseOver(); + expect(await one.take(start)).toBe(true); + const renewed = start + LEASE_MS / 2 + 1; + expect(await one.take(renewed)).toBe(true); + await store.set("job:lease", start + LEASE_MS); + one.reset(); + expect(await one.take(renewed + 1)).toBe(true); + expect(await leaseOver().take(renewed + 1)).toBe(false); + }); + + it("tells every instance that the database went back to an earlier copy, and one that it was first", async () => { + const one = createRestoreWatch({ checkpoints: store, logger: silentLogger }); + const other = createRestoreWatch({ checkpoints: store, logger: silentLogger }); + const at = (ms: number) => new Date(start + ms); + expect(await one.check(at(0))).toBeNull(); + expect(await other.check(at(0))).toBeNull(); + const copy = await store.get(RESTORE_GENERATION); + expect(await one.check(at(GENERATION_INTERVAL_MS))).toBeNull(); + expect(await other.check(at(GENERATION_INTERVAL_MS))).toBeNull(); + expect(await store.get(RESTORE_GENERATION)).toBe(copy + 2); + await store.set(RESTORE_GENERATION, copy); + expect(await one.check(at(GENERATION_INTERVAL_MS + 1))).toEqual({ first: true }); + expect(await store.get(RESTORES_FOUND)).toBe(1); + expect(await other.check(at(GENERATION_INTERVAL_MS + 1))).toEqual({ first: false }); + expect(await one.check(at(2 * GENERATION_INTERVAL_MS + 1))).toBeNull(); + expect(await other.check(at(2 * GENERATION_INTERVAL_MS + 1))).toBeNull(); + expect(await one.found()).toBe(1); + }); + + it("tells a restore that took the count of restores back too", async () => { + const one = createRestoreWatch({ checkpoints: store, logger: silentLogger }); + await store.set(RESTORES_FOUND, 3); + await store.set(RESTORE_GENERATION, 10); + expect(await one.check(new Date(start))).toBeNull(); + await store.set(RESTORES_FOUND, 2); + await store.set(RESTORE_GENERATION, 11); + expect(await one.check(new Date(start + 1))).toEqual({ first: true }); + expect(await store.get(RESTORES_FOUND)).toBe(4); + }); + }); }); }; diff --git a/packages/core/src/adapter/testing/person-store.contract.ts b/packages/core/src/adapter/testing/person-store.contract.ts index 0c28c457..39b3d609 100644 --- a/packages/core/src/adapter/testing/person-store.contract.ts +++ b/packages/core/src/adapter/testing/person-store.contract.ts @@ -180,6 +180,25 @@ export const personStoreContract: PersonStoreContractFunction = ({ create }) => expect(await store.nextHoldEnd(new Date(at(0)))).toBeNull(); }); + it("counts the persons a retention may forget, held or not, and clears every hold at once", async () => { + expect(await store.countIdle({})).toBe(0); + await store.record({ person: "c-1", position: 1, lastActiveAt: at(1), forgottenAt: null }); + await store.record({ person: "c-2", position: 2, lastActiveAt: at(2), forgottenAt: null }); + await store.record({ person: "c-3", position: 3, lastActiveAt: at(3), forgottenAt: null }); + await store.record({ person: "c-4", position: 4, lastActiveAt: at(1), forgottenAt: null }); + await store.restrict("c-4", since); + await store.hold("c-1", at(1), { until: at(5), attempts: 2 }); + await store.hold("c-2", at(2), { until: at(6), attempts: 1 }); + expect(await store.countIdle({})).toBe(3); + expect(await store.countIdle({ activeBefore: new Date(at(2)) })).toBe(2); + expect(await store.countIdle({ activeBefore: new Date(at(0)) })).toBe(0); + expect(await store.releaseHolds()).toBe(2); + expect(await store.get("c-1")).toMatchObject({ lastActiveAt: at(1), hold: null }); + expect(await store.get("c-2")).toMatchObject({ lastActiveAt: at(2), hold: null }); + expect(await store.nextHoldEnd(new Date(at(0)))).toBeNull(); + expect(await store.releaseHolds()).toBe(0); + }); + it("keeps a hold through a restriction, and names no hold of a person a sweep would not list", async () => { await store.record({ person: "c-1", position: 1, lastActiveAt: at(1), forgottenAt: null }); await store.record({ person: "c-2", position: 2, lastActiveAt: at(1), forgottenAt: null }); diff --git a/packages/core/src/adapter/testing/personal-key-store.contract.ts b/packages/core/src/adapter/testing/personal-key-store.contract.ts index 80f74835..b2467d49 100644 --- a/packages/core/src/adapter/testing/personal-key-store.contract.ts +++ b/packages/core/src/adapter/testing/personal-key-store.contract.ts @@ -151,6 +151,29 @@ export const personalKeyStoreContract: PersonalKeyStoreContractFunction = ({ cre expect(await store.nextHoldEnd({ kids: ["kid-1", "kid-2"], after: before })).toBeNull(); }); + it("counts the keys wrapped under the ids asked for, held or not, and clears every hold at once", async () => { + await store.add("c-1", wrapped("1", "kid-1")); + await store.add("c-2", wrapped("2", "kid-1")); + await store.add("c-3", stored("3")); + await store.add("c-4", wrapped("4", "kid-2")); + await store.hold("c-1", wrapped("1", "kid-1"), { + until: "2026-02-01T00:00:30.000Z", + attempts: 2, + }); + await store.hold("c-3", stored("3"), { until: "2026-02-01T00:00:20.000Z", attempts: 1 }); + expect(await store.countWrappedUnder({ kids: ["kid-1"] })).toBe(2); + expect(await store.countWrappedUnder({ kids: ["kid-1", null] })).toBe(3); + expect(await store.countWrappedUnder({ kids: [null] })).toBe(1); + expect(await store.countWrappedUnder({ kids: [] })).toBe(0); + expect(await store.releaseHolds()).toBe(2); + expect(await store.listWrappedUnder({ kids: ["kid-1", null], limit: 10 })).toEqual([ + { person: "c-1", key: wrapped("1", "kid-1"), hold: null }, + { person: "c-2", key: wrapped("2", "kid-1"), hold: null }, + { person: "c-3", key: stored("3"), hold: null }, + ]); + expect(await store.releaseHolds()).toBe(0); + }); + it("never holds a key forgotten or replaced since it was read, and forgets a hold with its key", async () => { const read = wrapped("first", "kid-1"); const hold = { until: "2026-02-01T00:00:30.000Z", attempts: 1 }; diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 007bec50..6805a8e7 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -68,6 +68,7 @@ import { import { createScheduledCommandWorker } from "./scheduler/worker.ts"; import { guardedLogger } from "./shared/guarded-logger.ts"; import { ignoredRetries, type PendingRetries } from "./shared/pending-retries.ts"; +import { createRestoreWatch } from "./shared/restores.ts"; import { createSnapshotSubscriber, takesSnapshots } from "./snapshot/subscriber.ts"; import { ATTRIBUTES, METRICS, meter, tracedOperation } from "./telemetry.ts"; @@ -593,7 +594,9 @@ export const assembleApp: AssembleAppFunction = async ({ ? undefined : createDeadLetterRetention({ store: written.deadLetterStore, + checkpoints: written.checkpointStore, retentionMs: letterRetentionMs, + clock, logger, }); const { concurrencyRetries } = config.runtime.commands; @@ -638,6 +641,7 @@ export const assembleApp: AssembleAppFunction = async ({ logger, errorText, }); + const restores = createRestoreWatch({ checkpoints: written.checkpointStore, logger }); const dispatcher = createDispatcher({ eventStore: storage.eventStore, checkpointStore: storage.checkpointStore, @@ -695,6 +699,8 @@ export const assembleApp: AssembleAppFunction = async ({ rotation, reconciliation, backfill, + restores, + onRestored: dispatcher.restored, reactive: following.map((subscriber) => subscriber.name), config, ids, diff --git a/packages/core/src/kernel/dead-letters/retention.test.ts b/packages/core/src/kernel/dead-letters/retention.test.ts index 63ee5e57..628b31e2 100644 --- a/packages/core/src/kernel/dead-letters/retention.test.ts +++ b/packages/core/src/kernel/dead-letters/retention.test.ts @@ -1,23 +1,44 @@ import { describe, expect, it } from "vitest"; import { testDeadLetter } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; import { memory } from "../../memory/index.ts"; import { HOLD_MS } from "../personal-data/hold.ts"; +import { LEASE_MS } from "../shared/lease.ts"; import { createRecordingLogger } from "../test-support.ts"; -import { createDeadLetterRetention, DEAD_LETTER_PURGE_LIMIT } from "./retention.ts"; +import { + createDeadLetterRetention, + DEAD_LETTER_PURGE_LEASE, + DEAD_LETTER_PURGE_LIMIT, +} from "./retention.ts"; const DAY = 86_400_000; const start = new Date("2026-01-01T00:00:00.000Z"); const later = (ms: number): Date => new Date(start.getTime() + ms); const setUp = async () => { - const { deadLetterStore: store } = await memory().createStorage(); + const { deadLetterStore: store, checkpointStore: checkpoints } = await memory().createStorage(); const { logger, entries: logs } = createRecordingLogger(); - const retention = createDeadLetterRetention({ store, retentionMs: 7 * DAY, logger }); + const retention = createDeadLetterRetention({ + store, + checkpoints, + retentionMs: 7 * DAY, + clock: createFixedClock(start), + logger, + }); const resolve = async (id: string, at: Date, status: "retried" | "discarded" = "retried") => { await store.add(testDeadLetter(id)); await store.updateStatus(id, status, at); }; - return { store, logs, retention, resolve }; + // Another instance over the same store. + const other = () => + createDeadLetterRetention({ + store, + checkpoints, + retentionMs: 7 * DAY, + clock: createFixedClock(start), + logger, + }); + return { store, checkpoints, logs, retention, resolve, other }; }; describe("the retention of dead letters", () => { @@ -142,3 +163,52 @@ describe("the retention of dead letters", () => { expect(await retention.nextDueAt()).toEqual(new Date(next + HOLD_MS)); }); }); + +describe("the retention of dead letters across instances", () => { + it("purges each batch once while two instances run it: the one holding the lease", async () => { + const { store, checkpoints, retention, resolve, other } = await setUp(); + for (let at = 0; at <= DEAD_LETTER_PURGE_LIMIT; at += 1) await resolve(`l-${at}`, later(at)); + const second = other(); + const due = later(8 * DAY); + expect(await retention.sweep(due)).toBe(DEAD_LETTER_PURGE_LIMIT); + expect(await second.sweep(due)).toBe(0); + expect(await store.count()).toBe(1); + expect(await checkpoints.get(DEAD_LETTER_PURGE_LEASE)).toBe(due.getTime() + LEASE_MS); + expect(await second.nextDueAt()).toEqual(new Date(due.getTime() + LEASE_MS)); + await retention.release(); + expect(await second.sweep(due)).toBe(1); + }); + + it("hands the purge to another instance once its lease lapsed, and purges nothing more itself", async () => { + const { store, retention, resolve, other } = await setUp(); + for (let at = 0; at < 2 * DEAD_LETTER_PURGE_LIMIT; at += 1) await resolve(`l-${at}`, later(at)); + const due = later(8 * DAY); + expect(await retention.sweep(due)).toBe(DEAD_LETTER_PURGE_LIMIT); + const lapsed = new Date(due.getTime() + LEASE_MS); + const second = other(); + expect(await second.sweep(lapsed)).toBe(DEAD_LETTER_PURGE_LIMIT); + await resolve("late", start); + expect(await retention.sweep(new Date(lapsed.getTime() + 1))).toBe(0); + expect(await store.count()).toBe(1); + }); + + it("reads the store again at once after a restore", async () => { + const { store, retention, resolve } = await setUp(); + await retention.sweep(start); + await resolve("a", later(-8 * DAY)); + expect(await retention.sweep(later(DAY))).toBe(0); + retention.restored(); + expect(await retention.sweep(later(DAY))).toBe(1); + expect(await store.count()).toBe(0); + }); + + it("says when the earliest resolved letter comes due", async () => { + const { retention, resolve } = await setUp(); + expect(await retention.status()).toEqual({ retentionMs: 7 * DAY, nextDueAt: null }); + await resolve("a", later(DAY)); + expect(await retention.status()).toEqual({ + retentionMs: 7 * DAY, + nextDueAt: later(8 * DAY).toISOString(), + }); + }); +}); diff --git a/packages/core/src/kernel/dead-letters/retention.ts b/packages/core/src/kernel/dead-letters/retention.ts index b6037d2c..b3b32592 100644 --- a/packages/core/src/kernel/dead-letters/retention.ts +++ b/packages/core/src/kernel/dead-letters/retention.ts @@ -1,10 +1,27 @@ +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; import type { DeadLetterStore } from "../../adapter/storage/dead-letter-store.ts"; +import type { Clock } from "../../contracts/clock.ts"; import type { Logger } from "../../contracts/logger.ts"; import { createPause } from "../personal-data/hold.ts"; +import { createLease } from "../shared/lease.ts"; // How many resolved letters a pass of the worker removes at most. export const DEAD_LETTER_PURGE_LIMIT = 100; +/** + * The checkpoint row of the lease the purge runs under: one instance purges at a time. + */ +export const DEAD_LETTER_PURGE_LEASE = "dead-letters:purge:lease"; + +/** + * Where the purge stands: when the earliest resolved letter comes due, `null` when none is + * resolved. + */ +export interface DeadLetterRetentionStatus { + readonly retentionMs: number; + readonly nextDueAt: string | null; +} + export interface DeadLetterRetention { /** * Removes up to `DEAD_LETTER_PURGE_LIMIT` letters resolved longer than the retention ago at @@ -16,11 +33,22 @@ export interface DeadLetterRetention { * When the earliest resolved letter outlives the retention, `null` when no letter is resolved. */ nextDueAt(): Promise; + status(): Promise; + /** + * Gives the lease back when this instance holds it. Never rejects. + */ + release(): Promise; + /** + * After the database went back to a copy: forgets what this instance knew of the purge. + */ + restored(): void; } export interface CreateDeadLetterRetentionArgs { readonly store: DeadLetterStore; + readonly checkpoints: CheckpointStore; readonly retentionMs: number; + readonly clock: Clock; readonly logger: Logger; } @@ -35,10 +63,13 @@ export interface CreateDeadLetterRetentionFunction { */ export const createDeadLetterRetention: CreateDeadLetterRetentionFunction = ({ store, + checkpoints, retentionMs, + clock, logger, }) => { const pause = createPause(); + const lease = createLease({ checkpoints, row: DEAD_LETTER_PURGE_LEASE, clock, logger }); const dueAt = (resolvedAt: Date | null): number | null => resolvedAt === null ? null : resolvedAt.getTime() + retentionMs; let unreadUntil = Number.NEGATIVE_INFINITY; @@ -54,6 +85,7 @@ export const createDeadLetterRetention: CreateDeadLetterRetentionFunction = ({ pause.resume(); return 0; } + if (!(await lease.take(at))) return 0; const purged = await store.purge({ resolvedBy: new Date(at - retentionMs), limit: DEAD_LETTER_PURGE_LIMIT, @@ -69,7 +101,19 @@ export const createDeadLetterRetention: CreateDeadLetterRetentionFunction = ({ }, nextDueAt: async () => { const due = dueAt(await store.firstResolvedAt()); - return due === null ? null : new Date(pause.notBefore(due)); + return due === null + ? null + : new Date(pause.notBefore(Math.max(due, lease.heldElsewhereUntil()))); + }, + status: async () => { + const due = dueAt(await store.firstResolvedAt()); + return { retentionMs, nextDueAt: due === null ? null : new Date(due).toISOString() }; + }, + release: lease.release, + restored: () => { + unreadUntil = Number.NEGATIVE_INFINITY; + pause.resume(); + lease.reset(); }, }; }; diff --git a/packages/core/src/kernel/dispatch/dispatcher.ts b/packages/core/src/kernel/dispatch/dispatcher.ts index 4b597c17..ee3a1ab9 100644 --- a/packages/core/src/kernel/dispatch/dispatcher.ts +++ b/packages/core/src/kernel/dispatch/dispatcher.ts @@ -106,6 +106,11 @@ export interface Dispatcher { */ catchUpThrough(args: CatchUpThroughArgs): Promise; getLag(): Promise; + /** + * Forgets the failures this instance met, as after the database went back to an earlier copy: + * every subscriber is tried again at once, its attempts counted from the first. + */ + restored(): void; } interface FailingState { @@ -344,6 +349,9 @@ export const createDispatcher: CreateDispatcherFunction = ({ }); return false; }, + restored: () => { + failing.clear(); + }, getLag: async () => { const lastPosition = await eventStore.lastPosition(); const lags = await Promise.all( diff --git a/packages/core/src/kernel/personal-data/backfill-walk.test.ts b/packages/core/src/kernel/personal-data/backfill-walk.test.ts index ec2a6612..f63a8c78 100644 --- a/packages/core/src/kernel/personal-data/backfill-walk.test.ts +++ b/packages/core/src/kernel/personal-data/backfill-walk.test.ts @@ -575,8 +575,11 @@ describe("the backfill's walk", () => { await backfill.release(); expect(logs).toContainEqual([ "error", - "personal data backfill could not give its lease back", - expect.objectContaining({ error: expect.objectContaining({ message: "cannot write" }) }), + "background job could not give its lease back", + expect.objectContaining({ + lease: `${BACKFILL_PREFIX}m1:lease`, + error: expect.objectContaining({ message: "cannot write" }), + }), ]); }); diff --git a/packages/core/src/kernel/personal-data/backfill.ts b/packages/core/src/kernel/personal-data/backfill.ts index 0b3787b0..926d2f06 100644 --- a/packages/core/src/kernel/personal-data/backfill.ts +++ b/packages/core/src/kernel/personal-data/backfill.ts @@ -6,6 +6,7 @@ import type { Clock } from "../../contracts/clock.ts"; import type { Logger } from "../../contracts/logger.ts"; import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { ProcessesRuntime } from "../process/build-processes.ts"; +import { createLease, LEASE_MS } from "../shared/lease.ts"; import { ATTRIBUTES, traced } from "../telemetry.ts"; import { createPause, holdAfter } from "./hold.ts"; import { @@ -32,8 +33,8 @@ export const BACKFILL_PREFIX = "personal-data:backfill:"; // late. export const BACKFILL_FOLLOW_INTERVAL_MS = 60_000; -// How long the instance walking holds the walk without renewing it, which it does on every pass. -export const BACKFILL_LEASE_MS: number = 2 * BACKFILL_FOLLOW_INTERVAL_MS; +// How long the instance walking holds the walk without renewing it. +export const BACKFILL_LEASE_MS: number = LEASE_MS; // How long the lease of other marks must have lapsed before their rows are removed: marks some // instance still runs renew theirs at least every lease. @@ -68,6 +69,12 @@ export interface PersonalDataBackfill { * for it to lapse. Never rejects. */ release(): Promise; + /** + * After the database went back to a copy: forgets what this instance knew of the walk, and, + * when `first`, clears the hold the copy brought back, so the walk starts again at once from + * where the copy stood. Never rejects. + */ + restored(first: boolean): Promise; } export interface CreatePersonalDataBackfillArgs { @@ -142,10 +149,7 @@ export const createPersonalDataBackfill: CreatePersonalDataBackfillFunction = ({ const rows = rowsOf(marks); const pause = createPause(); let restUntil = Number.NEGATIVE_INFINITY; - // Until when another instance's lease keeps the walk, for `nextDueAt`: the passes read the lease - // again, so a lease given back is taken at once. - let leasedUntil = Number.NEGATIVE_INFINITY; - let lease: number | null = null; + const lease = createLease({ checkpoints: checkpointStore, row: rows.lease, clock, logger }); let caughtUp = false; // This instance's tally since it started, for the line it logs when the walk catches up. let tally = { events: 0, discarded: 0, unsealable: 0, scheduledCommands: 0, deadLetters: 0 }; @@ -170,7 +174,7 @@ export const createPersonalDataBackfill: CreatePersonalDataBackfillFunction = ({ for (;;) { let reading = NOTHING_SEALED; for (let offset = 0; ; offset += batchSize) { - if (!(await takeLease(clock.now().getTime()))) return null; + if (!(await lease.take(clock.now().getTime()))) return null; const page = await list({ limit: batchSize, offset }); reading = addSealed(reading, await sealPage(page)); if (page.length < batchSize) break; @@ -209,27 +213,6 @@ export const createPersonalDataBackfill: CreatePersonalDataBackfillFunction = ({ await checkpointStore.remove(rows.heldUntil); }; - // Resolves whether this instance walks: it holds the lease, renewed once half of it is spent, or - // took it once it lapsed. A lease more than a lease ahead was written by a clock that runs ahead, - // and is taken. - const takeLease = async (at: number): Promise => { - const held = await checkpointStore.get(rows.lease); - if (held === lease && held - at > BACKFILL_LEASE_MS / 2) return true; - if (held !== lease && held > at && held <= at + BACKFILL_LEASE_MS) { - lease = null; - leasedUntil = held; - return false; - } - const until = at + BACKFILL_LEASE_MS; - if (!(await checkpointStore.compareAndSet(rows.lease, held, until))) { - lease = null; - return false; - } - lease = until; - leasedUntil = Number.NEGATIVE_INFINITY; - return true; - }; - // The rows of other marks whose lease lapsed long ago: no instance runs those marks any more. const removeStale = async (at: number): Promise => { const others = new Map(); @@ -359,7 +342,7 @@ export const createPersonalDataBackfill: CreatePersonalDataBackfillFunction = ({ restUntil = heldUntil; return 0; } - if (!(await takeLease(at))) return 0; + if (!(await lease.take(at))) return 0; let started = await checkpointStore.get(rows.started); if (started === 0) { await checkpointStore.compareAndSet(rows.started, 0, at); @@ -409,17 +392,20 @@ export const createPersonalDataBackfill: CreatePersonalDataBackfillFunction = ({ if (caughtUp) return null; caughtUp = (await status()).caughtUp; if (caughtUp) return null; - return new Date(pause.notBefore(Math.max(clock.now().getTime(), restUntil, leasedUntil))); + return new Date( + pause.notBefore(Math.max(clock.now().getTime(), restUntil, lease.heldElsewhereUntil())), + ); }, status, - // Lapsed now rather than removed, so the rows of these marks stay while instances may run them. - release: async () => { - if (lease === null) return; - const held = lease; - lease = null; - const lapsed = Math.min(held, clock.now().getTime()); - await checkpointStore.compareAndSet(rows.lease, held, lapsed).catch((error: unknown) => { - logger.error("personal data backfill could not give its lease back", { error }); + release: lease.release, + restored: async (first) => { + caughtUp = false; + restUntil = Number.NEGATIVE_INFINITY; + pause.resume(); + lease.reset(); + if (!first) return; + await releaseHold().catch((error: unknown) => { + logger.error("personal data backfill's hold could not be cleared", { error }); }); }, }; diff --git a/packages/core/src/kernel/personal-data/event-store.test.ts b/packages/core/src/kernel/personal-data/event-store.test.ts index fdd5bf08..13f1b404 100644 --- a/packages/core/src/kernel/personal-data/event-store.test.ts +++ b/packages/core/src/kernel/personal-data/event-store.test.ts @@ -72,6 +72,8 @@ const setup = async () => { rewrap: (person, from, to) => storage.personalKeys.rewrap(person, from, to), hold: (person, from, hold) => storage.personalKeys.hold(person, from, hold), nextHoldEnd: (args) => storage.personalKeys.nextHoldEnd(args), + countWrappedUnder: (args) => storage.personalKeys.countWrappedUnder(args), + releaseHolds: () => storage.personalKeys.releaseHolds(), }; const store = withPersonalData({ eventStore: storage.eventStore, @@ -474,6 +476,8 @@ describe("withPersonalData", () => { rewrap: (person, from, to) => storage.personalKeys.rewrap(person, from, to), hold: (person, from, hold) => storage.personalKeys.hold(person, from, hold), nextHoldEnd: (args) => storage.personalKeys.nextHoldEnd(args), + countWrappedUnder: (args) => storage.personalKeys.countWrappedUnder(args), + releaseHolds: () => storage.personalKeys.releaseHolds(), }; await expect( withPersonalData({ diff --git a/packages/core/src/kernel/personal-data/reconciliation.ts b/packages/core/src/kernel/personal-data/reconciliation.ts index 4d98c4ec..7b511554 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.ts @@ -117,6 +117,26 @@ export interface ReconciliationFailed { readonly failed: unknown; } +/** + * Where the reconciliation stands, as the database's rows say, beside whether this instance's + * last one read the ledger through. + */ +export interface ForgetLedgerReconciliationStatus { + readonly reconciled: boolean; + /** + * The time of the last ledger entry the database has applied, `null` before the first. + */ + readonly appliedThrough: string | null; + /** + * When an instance last took the periodic reconciliation, or finished one at start. + */ + readonly lastRunAt: string | null; + /** + * The hold after reconciliations that failed in a row, `null` without one. + */ + readonly held: { readonly until: string; readonly attempts: number } | null; +} + export interface ForgetLedgerReconciliation { /** * Reads the ledger past the database's cursor and forgets again whom `forgetAgain` finds; then @@ -141,6 +161,13 @@ export interface ForgetLedgerReconciliation { * and after one that failed. */ reconciled(): boolean; + status(): Promise; + /** + * After the database went back to a copy: due at once, since the copy may hold persons + * forgotten after it was taken; when `first`, clears the hold the copy brought back. Never + * rejects. + */ + restored(first: boolean): Promise; } export interface CreateForgetLedgerReconciliationArgs { @@ -369,5 +396,29 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF }, nextDueAt: () => (dueAt === null ? null : new Date(dueAt)), reconciled: () => done, + status: async () => { + const [cursor, lastRun, held] = await Promise.all([ + checkpointStore.get(FORGET_LEDGER_CURSOR), + checkpointStore.get(FORGET_LEDGER_LAST_RUN), + storedHold(), + ]); + const timeOf = (at: number): string | null => (at === 0 ? null : new Date(at).toISOString()); + return { + reconciled: done, + appliedThrough: timeOf(cursor), + lastRunAt: timeOf(lastRun), + held: held.attempts === 0 && Date.parse(held.until) === 0 ? null : held, + }; + }, + restored: async (first) => { + done = false; + dueAt = clock.now().getTime(); + // A hold the copy brought back is not waited for: what it held was failing before the copy. + local = null; + if (!first) return; + await release().catch((error: unknown) => + logger.error("forget ledger's reconciliation hold could not be cleared", { error }), + ); + }, }; }; diff --git a/packages/core/src/kernel/personal-data/restriction.test.ts b/packages/core/src/kernel/personal-data/restriction.test.ts index 9e3886d6..7938595f 100644 --- a/packages/core/src/kernel/personal-data/restriction.test.ts +++ b/packages/core/src/kernel/personal-data/restriction.test.ts @@ -232,6 +232,8 @@ describe("refuseRestricted", () => { listIdle: vi.fn(), hold: vi.fn(), nextHoldEnd: vi.fn(), + countIdle: vi.fn(), + releaseHolds: vi.fn(), }; await refuseRestricted({ persons, person: null }); expect(persons.get).not.toHaveBeenCalled(); diff --git a/packages/core/src/kernel/personal-data/retention.test.ts b/packages/core/src/kernel/personal-data/retention.test.ts index 2e17f794..c1c5fb91 100644 --- a/packages/core/src/kernel/personal-data/retention.test.ts +++ b/packages/core/src/kernel/personal-data/retention.test.ts @@ -6,6 +6,7 @@ import type { StoredEvent } from "../../contracts/event.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { memory } from "../../memory/index.ts"; import { createKeptPersonStore } from "../../memory/person-store.ts"; +import { LEASE_MS } from "../shared/lease.ts"; import { tracer } from "../telemetry.ts"; import { createRecordingLogger } from "../test-support.ts"; import { forgetPersonalData } from "./forget.ts"; @@ -14,6 +15,7 @@ import { restrictProcessing } from "./restriction.ts"; import { createRetentionSubscriber, createRetentionSweep, + RETENTION_LEASE, RETENTION_SUBSCRIBER, RETENTION_SWEEP_LIMIT, } from "./retention.ts"; @@ -584,6 +586,8 @@ describe("createRetentionSweep", () => { advance(31); const failedAt = clock.now().getTime(); expect(await failing.sweep(clock.now())).toBe(0); + // Stopped, as an instance gives its lease back when it stops. + await failing.release(); expect((await storage.persons.get("c-1"))?.hold).toEqual({ until: new Date(failedAt + HOLD_MS).toISOString(), attempts: 1, @@ -614,6 +618,7 @@ describe("createRetentionSweep", () => { clock.advance(HOLD_MS); await before.sweep(clock.now()); expect(await before.nextDueAt()).toEqual(new Date(clock.now().getTime() + 2 * HOLD_MS)); + await before.release(); const restarted = sweepOver(broken); expect(await restarted.nextDueAt()).toEqual(new Date(clock.now().getTime() + 2 * HOLD_MS)); expect(await restarted.sweep(clock.now())).toBe(0); @@ -760,3 +765,124 @@ describe("createRetentionSweep", () => { expect(asked).toEqual([]); }); }); + +describe("the retention across instances", () => { + // A store that runs `after` once each forget commits. + const committing = (live: Storage, after: () => Promise): Storage => ({ + ...live, + transact: async (work) => { + const result = await live.transact(work); + await after(); + return result; + }, + }); + + const idle = async (persons: readonly string[]) => { + const set = await setUp(); + for (const person of persons) await set.touch(person); + await set.record(); + set.advance(31); + return set; + }; + + it("forgets each person once while two instances sweep: the one holding the lease", async () => { + const { storage, sweepOver, forgottenFor, clock } = await idle(["c-1", "c-2", "c-3"]); + const one = sweepOver(storage); + const other = sweepOver(storage); + expect(await one.sweep(clock.now())).toBe(3); + expect(await other.sweep(clock.now())).toBe(0); + expect(await forgottenFor()).toEqual([ + ["c-1", "retention"], + ["c-2", "retention"], + ["c-3", "retention"], + ]); + expect(await storage.checkpointStore.get(RETENTION_LEASE)).toBe( + clock.now().getTime() + LEASE_MS, + ); + }); + + it("leaves the persons to the instance holding the lease, and names when it lapses", async () => { + const { storage, sweepOver, touch, record, advance, clock } = await idle(["c-1"]); + const one = sweepOver(storage); + const leased = clock.now().getTime() + LEASE_MS; + expect(await one.sweep(clock.now())).toBe(1); + advance(-31); + await touch("c-2"); + await record(); + advance(31); + const other = sweepOver(storage); + expect(await other.sweep(clock.now())).toBe(0); + expect(await other.nextDueAt()).toEqual(new Date(leased)); + await one.release(); + expect(await other.sweep(clock.now())).toBe(1); + }); + + it("hands the rest to another instance when its lease lapses mid-way, forgetting nobody twice", async () => { + const { storage, sweepOver, forgottenFor, clock } = await idle(["c-1", "c-2", "c-3", "c-4"]); + const other = sweepOver(storage); + let commits = 0; + let handedOver = 0; + // After its second forget the instance stalls past its lease, and the other runs a pass. + const one = sweepOver( + committing(storage, async () => { + commits += 1; + if (commits !== 2) return; + clock.set(new Date(clock.now().getTime() + LEASE_MS)); + handedOver = await other.sweep(clock.now()); + }), + ); + expect(await one.sweep(clock.now())).toBe(2); + expect(handedOver).toBe(2); + expect((await forgottenFor()).map(([person]) => person)).toEqual(["c-1", "c-2", "c-3", "c-4"]); + }); + + it("clears the holds a restore brought back, on the instance that found it first", async () => { + const { storage, sweepOver, clock } = await idle(["c-1"]); + const hold = { + until: new Date(clock.now().getTime() + HOLD_MAX_MS).toISOString(), + attempts: 5, + }; + await storage.persons.hold("c-1", day(0), hold); + await sweepOver(storage).restored(false); + expect((await storage.persons.get("c-1"))?.hold).toEqual(hold); + const sweep = sweepOver(storage); + expect(await sweep.sweep(clock.now())).toBe(0); + await sweep.restored(true); + expect((await storage.persons.get("c-1"))?.hold).toBeNull(); + expect(await sweep.nextDueAt()).toEqual(new Date(Date.parse(day(0)) + 30 * DAY)); + expect(await sweep.sweep(clock.now())).toBe(1); + }); + + it("logs holds it could not clear after a restore, and goes on", async () => { + const { storage, sweepOver, entries } = await idle([]); + await sweepOver({ + ...storage, + persons: { + ...storage.persons, + releaseHolds: () => Promise.reject(new Error("cannot write")), + }, + }).restored(true); + expect(entries).toContainEqual({ + level: "error", + message: "the holds of persons past the retention could not be cleared", + fields: expect.objectContaining({ errorMessage: "cannot write" }), + }); + }); + + it("says how many persons are past the retention and when the next one comes due", async () => { + const { storage, sweepOver, touch, record, advance, clock } = await setUp(); + const sweep = sweepOver(storage); + expect(await sweep.status()).toEqual({ retentionMs: 30 * DAY, due: 0, nextDueAt: null }); + await touch("c-1"); + advance(1); + await touch("c-2"); + await record(); + advance(29.5); + expect(await sweep.status()).toEqual({ + retentionMs: 30 * DAY, + due: 1, + nextDueAt: new Date(Date.parse(day(0)) + 30 * DAY).toISOString(), + }); + expect(clock.now().getTime()).toBe(Date.parse(day(30)) + DAY / 2); + }); +}); diff --git a/packages/core/src/kernel/personal-data/retention.ts b/packages/core/src/kernel/personal-data/retention.ts index df9b0f3c..a9a4c7e6 100644 --- a/packages/core/src/kernel/personal-data/retention.ts +++ b/packages/core/src/kernel/personal-data/retention.ts @@ -12,6 +12,7 @@ import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; import type { Subscriber } from "../dispatch/delivery.ts"; +import { createLease } from "../shared/lease.ts"; import { forgetWithin, logForgotten } from "./forget.ts"; import { createPause, holdAfter } from "./hold.ts"; import { transactRetrying } from "./transact.ts"; @@ -21,6 +22,21 @@ export const RETENTION_SUBSCRIBER: "retention" = "retention"; // How many persons a pass of the worker forgets at most. export const RETENTION_SWEEP_LIMIT = 50; +/** + * The checkpoint row of the lease the sweep runs under: one instance forgets at a time. + */ +export const RETENTION_LEASE = "personal-data:retention:lease"; + +/** + * Where the retention stands: how many persons are idle past it now, held ones included, and when + * the next one comes due. + */ +export interface RetentionStatus { + readonly retentionMs: number; + readonly due: number; + readonly nextDueAt: string | null; +} + const isActivity = (event: StoredEvent): boolean => event.aggregateType !== BOUNDA_AGGREGATE; // Asked only of the runtime's own events, once `isActivity` said no. @@ -86,6 +102,16 @@ export interface RetentionSweep { * ends; `null` when there is none. */ nextDueAt(): Promise; + status(): Promise; + /** + * Gives the lease back when this instance holds it. Never rejects. + */ + release(): Promise; + /** + * After the database went back to a copy: forgets what this instance knew of the sweep, and, + * when `first`, clears the holds of the persons the copy brought back. Never rejects. + */ + restored(first: boolean): Promise; } export interface CreateRetentionSweepArgs { @@ -182,6 +208,12 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ // When the store itself fails a pass, so no row can keep the wait, the sweep stops in this // instance for a while. const pause = createPause(); + const lease = createLease({ + checkpoints: storage.checkpointStore, + row: RETENTION_LEASE, + clock, + logger, + }); // A person left undone waits on their own row, so one whose forget keeps failing never holds // the others back, every instance skips them, and a host that wakes for `nextDueAt` never wakes @@ -237,6 +269,8 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ let forgotten = 0; let kept = true; for (const person of due) { + // A lease that lapsed while this instance stalled was taken by another, which goes on. + if (!(await lease.take(clock.now().getTime()))) break; const outcome = await attempt(person, at, through); if (outcome.forgotten) forgotten += 1; kept &&= outcome.kept; @@ -278,7 +312,31 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ ]); next = Math.min(deadlineOf(free), heldUntil?.getTime() ?? Number.POSITIVE_INFINITY); } - return next === Number.POSITIVE_INFINITY ? null : new Date(pause.notBefore(next)); + return next === Number.POSITIVE_INFINITY + ? null + : new Date(pause.notBefore(Math.max(next, lease.heldElsewhereUntil()))); + }, + status: async () => { + const now = clock.now(); + const [due, next] = await Promise.all([ + storage.persons.countIdle({ activeBefore: new Date(now.getTime() - retentionMs) }), + storage.persons.listIdle({ limit: 1 }), + ]); + const deadline = deadlineOf(next[0]); + return { + retentionMs, + due, + nextDueAt: deadline === Number.POSITIVE_INFINITY ? null : new Date(deadline).toISOString(), + }; + }, + release: lease.release, + restored: async (first) => { + pause.resume(); + lease.reset(); + if (!first) return; + await storage.persons.releaseHolds().catch((error: unknown) => { + logger.error("the holds of persons past the retention could not be cleared", { error }); + }); }, }; }; diff --git a/packages/core/src/kernel/personal-data/rotation.test.ts b/packages/core/src/kernel/personal-data/rotation.test.ts index 7561cc51..ae6bec18 100644 --- a/packages/core/src/kernel/personal-data/rotation.test.ts +++ b/packages/core/src/kernel/personal-data/rotation.test.ts @@ -4,6 +4,7 @@ import type { PersonalKeyStore } from "../../adapter/storage/personal-key-store. import { createFixedClock } from "../../contracts/clock.ts"; import type { Logger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; +import { LEASE_MS } from "../shared/lease.ts"; import { HOLD_MAX_MS, HOLD_MS } from "./hold.ts"; import { type CryptoKeyWrapper, cryptoKeyWrapperOver } from "./key-wrapper.ts"; import { @@ -11,6 +12,7 @@ import { ROTATED_PREFIX, ROTATION_BATCH, ROTATION_CHECK_INTERVAL_MS, + ROTATION_LEASE_PREFIX, ROTATION_MARK_EXPIRY_MS, } from "./rotation.ts"; import { createPersonalKeyWrapping } from "./wrapped-keys.ts"; @@ -522,9 +524,10 @@ describe("createKeyRotation", () => { }; await addUnder(store, "old", "c-1"); const failedAt = clock.now().getTime(); - expect( - await rotationOver({ keys: store, checkpoints, wrapper: failing }).sweep(clock.now()), - ).toBe(0); + const first = rotationOver({ keys: store, checkpoints, wrapper: failing }); + expect(await first.sweep(clock.now())).toBe(0); + // Stopped, as an instance gives its lease back when it stops. + await first.release(); const [row] = await store.listWrappedUnder({ kids: ["old"], limit: 1 }); expect(row?.hold).toEqual({ until: new Date(failedAt + HOLD_MS).toISOString(), attempts: 1 }); const healthy = rotationOver({ keys: store, checkpoints, wrapper }); @@ -534,6 +537,7 @@ describe("createKeyRotation", () => { const restarted = rotationOver({ keys: store, checkpoints, wrapper: failing }); expect(await restarted.sweep(clock.now())).toBe(0); expect(await restarted.nextDueAt()).toEqual(new Date(clock.now().getTime() + 2 * HOLD_MS)); + await restarted.release(); clock.advance(2 * HOLD_MS); expect(await healthy.sweep(clock.now())).toBe(1); expect(await store.listWrappedUnder({ kids: ["new"], limit: 1 })).toEqual([ @@ -689,3 +693,156 @@ describe("createKeyRotation", () => { }); const toText = (bytes: Uint8Array): string => Buffer.from(bytes).toString("base64"); + +describe("the key rotation across instances", () => { + const wrapper = () => fakeWrapper("new", ["old"]); + + // Counts the rewraps each key had, by person, whichever instance wrote them. + const countingRewraps = ( + store: PersonalKeyStore, + counts: Map, + afterEach: () => Promise = async () => undefined, + ): PersonalKeyStore => ({ + ...store, + rewrap: async (person, from, to) => { + counts.set(person, (counts.get(person) ?? 0) + 1); + const rewrapped = await store.rewrap(person, from, to); + await afterEach(); + return rewrapped; + }, + }); + + it("rewraps each key once while two instances run it: the one holding the lease", async () => { + const { store, checkpoints } = await setUp(); + const persons = ["c-1", "c-2", "c-3"]; + for (const person of persons) await addUnder(store, "old", person); + const counts = new Map(); + const keys = countingRewraps(store, counts); + const one = rotationOver({ keys, checkpoints, wrapper: wrapper() }); + const other = rotationOver({ keys, checkpoints, wrapper: wrapper() }); + expect(await one.sweep(clock.now())).toBe(3); + expect(await other.sweep(clock.now())).toBe(0); + expect(await kidsOf(store, persons)).toEqual(["new", "new", "new"]); + expect([...counts.values()]).toEqual([1, 1, 1]); + const leases = (await checkpoints.list()).filter(({ subscriber }) => + subscriber.startsWith(ROTATION_LEASE_PREFIX), + ); + expect(leases.map(({ position }) => position)).toEqual([clock.now().getTime() + LEASE_MS]); + }); + + it("leaves another instance's keys to it while it holds the lease, and names when it lapses", async () => { + const { store, checkpoints } = await setUp(); + await addUnder(store, "old", "c-1"); + const one = rotationOver({ keys: store, checkpoints, wrapper: wrapper() }); + const leased = clock.now().getTime() + LEASE_MS; + expect(await one.sweep(clock.now())).toBe(1); + await addUnder(store, "old", "c-2"); + clock.advance(ROTATION_CHECK_INTERVAL_MS); + const other = rotationOver({ keys: store, checkpoints, wrapper: wrapper() }); + expect(await other.sweep(clock.now())).toBe(0); + expect(await kidsOf(store, ["c-2"])).toEqual(["old"]); + expect(await other.nextDueAt()).toEqual(new Date(leased)); + await one.release(); + expect(await other.sweep(clock.now())).toBe(1); + }); + + it("hands the rest to another instance when its lease lapses mid-way, rewrapping no key twice", async () => { + const { store, checkpoints } = await setUp(); + const persons = ["c-1", "c-2", "c-3", "c-4"]; + for (const person of persons) await addUnder(store, "old", person); + const counts = new Map(); + let handedOver = 0; + const plain = countingRewraps(store, counts); + const other = rotationOver({ keys: plain, checkpoints, wrapper: wrapper() }); + // After its second key the instance stalls past its lease, and the other runs a pass. + const stalling = countingRewraps(store, counts, async () => { + if (counts.size !== 2 || handedOver > 0) return; + clock.advance(LEASE_MS); + handedOver = await other.sweep(clock.now()); + }); + const one = rotationOver({ keys: stalling, checkpoints, wrapper: wrapper() }); + expect(await one.sweep(clock.now())).toBe(2); + expect(handedOver).toBe(2); + expect(await kidsOf(store, persons)).toEqual(["new", "new", "new", "new"]); + expect([...counts.values()]).toEqual([1, 1, 1, 1]); + }); + + it("starts over at once after a restore taken mid-way, without the holds the copy brought back", async () => { + const { store, checkpoints } = await setUp(); + await addUnder(store, "old", "c-1"); + const held = await addUnder(store, "old", "c-2"); + const later = (ms: number) => new Date(clock.now().getTime() + ms).toISOString(); + await store.hold("c-2", held, { until: later(HOLD_MAX_MS), attempts: 7 }); + const rotation = rotationOver({ keys: store, checkpoints, wrapper: wrapper() }); + expect(await rotation.sweep(clock.now())).toBe(1); + // The rotation goes on to the end, then the copy taken at this point comes back: c-2 held + // again under the old key, and no mark. + await store.hold("c-2", held, { until: later(0), attempts: 7 }); + expect(await rotation.sweep(clock.now())).toBe(1); + expect(await rotation.sweep(clock.now())).toBe(0); + expect(await rotation.nextDueAt()).toBeNull(); + await store.remove("c-2"); + await store.add("c-2", held); + await store.hold("c-2", held, { until: later(HOLD_MAX_MS), attempts: 7 }); + for (const { subscriber } of await checkpoints.list()) await checkpoints.remove(subscriber); + await rotation.restored(true); + expect(await store.listWrappedUnder({ kids: ["old"], limit: 10 })).toEqual([ + { person: "c-2", key: held, hold: null }, + ]); + expect(await rotation.nextDueAt()).toEqual(clock.now()); + expect(await rotation.sweep(clock.now())).toBe(1); + expect(await kidsOf(store, ["c-1", "c-2"])).toEqual(["new", "new"]); + }); + + it("leaves the holds to the instance that found the restore first", async () => { + const { store, checkpoints } = await setUp(); + const held = await addUnder(store, "old", "c-1"); + const hold = { until: new Date(clock.now().getTime() + HOLD_MS).toISOString(), attempts: 2 }; + await store.hold("c-1", held, hold); + const rotation = rotationOver({ keys: store, checkpoints, wrapper: wrapper() }); + await rotation.restored(false); + expect((await store.listWrappedUnder({ kids: ["old"], limit: 1 }))[0]?.hold).toEqual(hold); + }); + + it("logs holds it could not clear after a restore, and goes on", async () => { + const { store, checkpoints } = await setUp(); + const logger = recordingLogger(); + const rotation = rotationOver({ + keys: { ...store, releaseHolds: () => Promise.reject(new Error("cannot write")) }, + checkpoints, + wrapper: wrapper(), + logger, + }); + await rotation.restored(true); + expect(logger.logged).toContainEqual({ + level: "error", + message: "personal keys' holds could not be cleared", + context: { error: expect.objectContaining({ message: "cannot write" }) }, + }); + }); + + it("says how many keys are left to rewrap, and when a pass last found none", async () => { + const { store, checkpoints } = await setUp(); + await addUnder(store, "old", "c-1"); + await addUnder(store, null, "c-2"); + const rotation = rotationOver({ + keys: store, + checkpoints, + wrapper: fakeWrapper("new", ["old", "new"]), + }); + expect(await rotation.status()).toEqual({ + kid: "new", + previous: ["old"], + pending: 2, + completeAt: null, + }); + await rotation.sweep(clock.now()); + await rotation.sweep(clock.now()); + expect(await rotation.status()).toEqual({ + kid: "new", + previous: ["old"], + pending: 0, + completeAt: clock.now().toISOString(), + }); + }); +}); diff --git a/packages/core/src/kernel/personal-data/rotation.ts b/packages/core/src/kernel/personal-data/rotation.ts index 2c0f2efc..769b3069 100644 --- a/packages/core/src/kernel/personal-data/rotation.ts +++ b/packages/core/src/kernel/personal-data/rotation.ts @@ -3,6 +3,7 @@ import type { PersonalKeyRow, PersonalKeyStore } from "../../adapter/storage/per import type { Clock } from "../../contracts/clock.ts"; import type { Logger } from "../../contracts/logger.ts"; import { digest } from "../read-model/fingerprint.ts"; +import { createLease } from "../shared/lease.ts"; import { createPause, holdAfter } from "./hold.ts"; import type { CryptoKeyWrapper } from "./key-wrapper.ts"; import type { PersonalKeyWrapping } from "./wrapped-keys.ts"; @@ -27,6 +28,28 @@ export const ROTATED_PREFIX = "personal-keys:rotated:"; // `ROTATION_CHECK_INTERVAL_MS`, so a mark this old is no running instance's. export const ROTATION_MARK_EXPIRY_MS: number = 10 * ROTATION_CHECK_INTERVAL_MS; +/** + * The checkpoint rows of the lease each set of wrapping keys rotates under, named after the set's + * digest as its mark is: one instance rewraps at a time. + */ +export const ROTATION_LEASE_PREFIX = "personal-keys:rotating:"; + +/** + * Where the rotation under one set of wrapping keys stands. + */ +export interface KeyRotationStatus { + readonly kid: string; + readonly previous: readonly string[]; + /** + * How many keys are left to rewrap, held ones included. + */ + readonly pending: number; + /** + * When a pass last found none left, `null` while one is. + */ + readonly completeAt: string | null; +} + export interface KeyRotation { /** * Rewraps under the wrapper's key up to `ROTATION_BATCH` keys wrapped under a previous one or @@ -39,6 +62,16 @@ export interface KeyRotation { * is none, and without reading the key table once the rotation is complete. */ nextDueAt(): Promise; + status(): Promise; + /** + * Gives the lease back when this instance holds it. Never rejects. + */ + release(): Promise; + /** + * After the database went back to a copy: forgets what this instance knew of the rotation, and, + * when `first`, clears the holds of the keys the copy brought back. Never rejects. + */ + restored(first: boolean): Promise; } export interface CreateKeyRotationArgs { @@ -57,8 +90,8 @@ export interface CreateKeyRotationFunction { (args: CreateKeyRotationArgs): KeyRotation; } -const markerOf = (kid: string, previous: readonly string[]): string => - `${ROTATED_PREFIX}${digest(JSON.stringify([kid, ...[...new Set(previous)].sort()]))}`; +const digestOf = (kid: string, previous: readonly string[]): string => + digest(JSON.stringify([kid, ...[...new Set(previous)].sort()])); interface Answer { readonly next: number | null; @@ -84,7 +117,14 @@ export const createKeyRotation: CreateKeyRotationFunction = ({ // A wrapper that names its own key among the previous ones would rewrap it on every pass. const previous = wrapper.previous.filter((kid) => kid !== wrapper.kid); const kids = [...previous, null]; - const marker = markerOf(wrapper.kid, previous); + const set = digestOf(wrapper.kid, previous); + const marker = `${ROTATED_PREFIX}${set}`; + const lease = createLease({ + checkpoints, + row: `${ROTATION_LEASE_PREFIX}${set}`, + clock, + logger, + }); // Until when a pass reads nothing: the mark stands, or every key left is held. let completeUntil = Number.NEGATIVE_INFINITY; let restUntil = Number.NEGATIVE_INFINITY; @@ -110,10 +150,12 @@ export const createKeyRotation: CreateKeyRotationFunction = ({ completeUntil = at + ROTATION_CHECK_INTERVAL_MS; if (!(await checkpoints.compareAndSet(marker, markedAt, at))) return; if (markedAt === 0) report(); + // The lease of another set lapsed at its position, as long ago as its mark was renewed. for (const { subscriber, position } of await checkpoints.list()) { if ( - subscriber.startsWith(ROTATED_PREFIX) && + (subscriber.startsWith(ROTATED_PREFIX) || subscriber.startsWith(ROTATION_LEASE_PREFIX)) && subscriber !== marker && + subscriber !== `${ROTATION_LEASE_PREFIX}${set}` && at - position >= ROTATION_MARK_EXPIRY_MS ) { await checkpoints.remove(subscriber); @@ -173,6 +215,8 @@ export const createKeyRotation: CreateKeyRotationFunction = ({ let rewrapped = 0; let kept = true; for (const row of due) { + // A lease that lapsed while this instance stalled was taken by another, which goes on. + if (!(await lease.take(clock.now().getTime()))) break; const outcome = await rewrap(row, at); if (outcome.rewrapped) rewrapped += 1; kept &&= outcome.kept; @@ -212,7 +256,31 @@ export const createKeyRotation: CreateKeyRotationFunction = ({ if (answer === null || at >= answer.until) answer = await read(now); return answer.next === null ? null - : new Date(pause.notBefore(Math.max(answer.next, restUntil))); + : new Date(pause.notBefore(Math.max(answer.next, restUntil, lease.heldElsewhereUntil()))); + }, + status: async () => { + const [pending, markedAt] = await Promise.all([ + keys.countWrappedUnder({ kids }), + checkpoints.get(marker), + ]); + return { + kid: wrapper.kid, + previous, + pending, + completeAt: markedAt === 0 ? null : new Date(markedAt).toISOString(), + }; + }, + release: lease.release, + restored: async (first) => { + completeUntil = Number.NEGATIVE_INFINITY; + restUntil = Number.NEGATIVE_INFINITY; + answer = null; + pause.resume(); + lease.reset(); + if (!first) return; + await keys.releaseHolds().catch((error: unknown) => { + logger.error("personal keys' holds could not be cleared", { error }); + }); }, }; }; diff --git a/packages/core/src/kernel/personal-data/wrapped-keys.ts b/packages/core/src/kernel/personal-data/wrapped-keys.ts index a03a2bed..2fe96488 100644 --- a/packages/core/src/kernel/personal-data/wrapped-keys.ts +++ b/packages/core/src/kernel/personal-data/wrapped-keys.ts @@ -289,6 +289,8 @@ export const createPersonalKeyWrapping: CreatePersonalKeyWrappingFunction = ({ rewrap: (person, from, to) => keys.rewrap(person, from, to), hold: (person, from, hold) => keys.hold(person, from, hold), nextHoldEnd: (args) => keys.nextHoldEnd(args), + countWrappedUnder: (args) => keys.countWrappedUnder(args), + releaseHolds: () => keys.releaseHolds(), }), close: () => { cancelExpiry?.(); diff --git a/packages/core/src/kernel/scheduler/worker.ts b/packages/core/src/kernel/scheduler/worker.ts index 73867caf..8109a2bf 100644 --- a/packages/core/src/kernel/scheduler/worker.ts +++ b/packages/core/src/kernel/scheduler/worker.ts @@ -32,6 +32,7 @@ import { PROCESS_DEADLINE_COMMAND, type ProcessDeadlinePayload } from "../proces import type { ProcessDeadlines } from "../process/deliver-deadline.ts"; import { PROCESSES_SUBSCRIBER } from "../process/runner.ts"; import { createMutex } from "../shared/mutex.ts"; +import type { RestoreWatch } from "../shared/restores.ts"; import { classifyFailure, errorIdentity, errorLabel, retryDelayMs } from "../shared/retry.ts"; import { appendSystemEvent, @@ -90,6 +91,15 @@ export interface CreateScheduledCommandWorkerArgs { * Swept on every pass; absent when no schema marks personal data. */ readonly backfill?: PersonalDataBackfill | undefined; + /** + * Checked on every pass: a database found gone back to an earlier copy has every background + * job start over from what it holds, the attempts the jobs counted with it. + */ + readonly restores?: RestoreWatch | undefined; + /** + * Called once a pass finds a restore, for what else counted attempts in this instance. + */ + readonly onRestored?: (() => void) | undefined; /** * The reactive subscribers the dispatcher runs, by name: what a deadline and a restricted * retry wait for before they run. @@ -165,6 +175,8 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction rotation, reconciliation, backfill, + restores, + onRestored, reactive, config, ids, @@ -540,8 +552,28 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction return storage.scheduler.claimDue({ now, limit: CLAIM_LIMIT, leaseMs }); }; + // The jobs' holds the copy brought back count failures it undid: the instance that found the + // restore first clears them, and every one forgets what it knew of the jobs. Each never rejects. + const restart = async (first: boolean): Promise => { + logger.warn("the database went back to an earlier copy; background jobs start over", { + first, + }); + purgedAt = Number.NEGATIVE_INFINITY; + waits.clear(); + onRestored?.(); + letterRetention?.restored(); + await Promise.all([ + retention?.restored(first), + rotation?.restored(first), + reconciliation?.restored(first), + backfill?.restored(first), + ]); + }; + const runOnce = (): Promise => mutex.run(async () => { + const restored = await restores?.check(clock.now()); + if (restored !== undefined && restored !== null) await restart(restored.first); const claimedAt = clock.now(); const due = await claimDue(claimedAt); // The batch is claimed already: an error escaping here would leave every entry to lapse, and @@ -605,7 +637,12 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction running = false; cancelWait?.(); await mutex.drain(); - await backfill?.release(); + await Promise.all([ + backfill?.release(), + retention?.release(), + rotation?.release(), + letterRetention?.release(), + ]); }, runOnce, leaseMs, diff --git a/packages/core/src/kernel/shared/lease.ts b/packages/core/src/kernel/shared/lease.ts new file mode 100644 index 00000000..96896307 --- /dev/null +++ b/packages/core/src/kernel/shared/lease.ts @@ -0,0 +1,97 @@ +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import type { Clock } from "../../contracts/clock.ts"; +import type { Logger } from "../../contracts/logger.ts"; + +// How long the instance running a background job holds it without renewing it, which it does +// whenever half of it is spent. +export const LEASE_MS = 120_000; + +export interface Lease { + /** + * Resolves whether this instance runs the job at `at`: it holds the lease, renewed once half of + * it is spent, or took it once it lapsed. Asked before each item of a pass, so an instance + * whose lease lapsed while it was stalled stops at the next one. + */ + take(at: number): Promise; + /** + * Until when another instance held the lease, as the last `take` read it; `-Infinity` when + * none did. + */ + heldElsewhereUntil(): number; + /** + * Gives the lease back when this instance holds it, so another runs the job at once instead of + * waiting for it to lapse. Never rejects. + */ + release(): Promise; + /** + * Forgets what this instance knew of the lease, as after the database went back to a copy: + * the next `take` reads it afresh, and takes as its own a lease of its own the copy brought back. + */ + reset(): void; +} + +export interface CreateLeaseArgs { + readonly checkpoints: CheckpointStore; + /** + * The checkpoint row that keeps, in milliseconds, until when the lease is held. + */ + readonly row: string; + readonly clock: Clock; + readonly logger: Logger; +} + +export interface CreateLeaseFunction { + (args: CreateLeaseArgs): Lease; +} + +// How many of the leases an instance wrote last it still knows as its own: a copy restored within a +// lease brings back one of them, at most this many renewals old. +const OWN_LEASES_KEPT = 4; + +/** + * One instance runs a job at a time: the one whose `compareAndSet` of the row took the lease. + * A lease more than a lease ahead was written by a clock that runs ahead, and is taken. + */ +export const createLease: CreateLeaseFunction = ({ checkpoints, row, clock, logger }) => { + let held: number | null = null; + let written: readonly number[] = []; + let elsewhereUntil = Number.NEGATIVE_INFINITY; + + return { + take: async (at) => { + const stored = await checkpoints.get(row); + if (stored === held && stored - at > LEASE_MS / 2) return true; + const ours = stored === held || written.includes(stored); + if (!ours && stored > at && stored <= at + LEASE_MS) { + held = null; + elsewhereUntil = stored; + return false; + } + const until = at + LEASE_MS; + if (!(await checkpoints.compareAndSet(row, stored, until))) { + held = null; + return false; + } + held = until; + written = [...written.slice(1 - OWN_LEASES_KEPT), until]; + elsewhereUntil = Number.NEGATIVE_INFINITY; + return true; + }, + heldElsewhereUntil: () => elsewhereUntil, + // Lapsed now rather than removed, so a row that names a configuration stays while instances + // may run it. + release: async () => { + if (held === null) return; + const until = held; + held = null; + const lapsed = Math.min(until, clock.now().getTime()); + await checkpoints.compareAndSet(row, until, lapsed).catch((error: unknown) => { + logger.error("background job could not give its lease back", { lease: row, error }); + }); + }, + reset: () => { + held = null; + elsewhereUntil = Number.NEGATIVE_INFINITY; + }, + }; +}; diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts new file mode 100644 index 00000000..876085b5 --- /dev/null +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -0,0 +1,227 @@ +import type { DatabaseSync, SQLInputValue } from "node:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; +import { createNodeSqliteAdapter } from "../../adapter/sqlite/node-sqlite.ts"; +import type { RuntimeConfig } from "../../config/types.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import { type LogFields, type Logger, silentLogger } from "../../contracts/logger.ts"; +import { memory } from "../../memory/index.ts"; +import type { PersonalDataForgottenPayload } from "../../modules/bounda-events.ts"; +import type { EventPayloadArgs, PayloadArgs } from "../../modules/payload.ts"; +import type { Registry } from "../../modules/registry.ts"; +import { type BoundaApp, createApp } from "../app.ts"; +import { HOLD_MAX_MS } from "../personal-data/hold.ts"; +import { masterKey } from "../personal-data/master-key.ts"; +import { ROTATION_CHECK_INTERVAL_MS } from "../personal-data/rotation.ts"; +import { GENERATION_INTERVAL_MS } from "./restores.ts"; + +const START = new Date("2026-04-01T00:00:00.000Z"); + +const registry = { + aggregates: { + customer: { + state: { initialState: {} }, + events: { + customerRegistered: { + payload: ({ z, personal }: EventPayloadArgs) => z.object({ email: personal(z.email()) }), + evolve: () => ({}), + }, + }, + commands: { + registerCustomer: { + module: { + payload: ({ z }: PayloadArgs) => z.object({ customerId: z.string(), email: z.email() }), + handler: ({ + command, + events, + }: { + command: { payload: { email: string } }; + events: Record unknown>; + }) => [events.customerRegistered?.({ email: command.payload.email })], + }, + }, + }, + policies: {}, + processes: {}, + }, + }, + readModels: {}, +} as unknown as Registry; + +const randomKey = (): string => + Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString("base64"); + +type Copy = ReadonlyMap[]>; + +// The Bounda tables as they are now, and putting them back while the app runs: what restoring a +// backup in place does to the instances still connected. +const copyOf = (db: DatabaseSync): Copy => + new Map( + ( + db + .prepare( + `SELECT "name" FROM sqlite_master WHERE "type" = 'table' AND "name" LIKE 'bounda_%'`, + ) + .all() as { name: string }[] + ).map(({ name }) => [ + name, + db.prepare(`SELECT * FROM "${name}"`).all() as Record[], + ]), + ); + +const restore = (db: DatabaseSync, copy: Copy): void => { + db.exec("BEGIN IMMEDIATE"); + for (const table of copy.keys()) db.exec(`DELETE FROM "${table}"`); + for (const [table, rows] of copy) { + for (const row of rows) { + const columns = Object.keys(row); + db.prepare( + `INSERT OR IGNORE INTO "${table}" (${columns.map((column) => `"${column}"`).join(", ")}) VALUES (${columns.map(() => "?").join(", ")})`, + ).run(...columns.map((column) => row[column] ?? null)); + } + } + db.exec("COMMIT"); +}; + +interface Logged { + readonly level: string; + readonly message: string; + readonly fields?: LogFields | undefined; +} + +const recordingLogger = (): Logger & { readonly logged: Logged[] } => { + const logged: Logged[] = []; + const at = (level: string) => (message: string, fields?: LogFields) => { + logged.push({ level, message, fields }); + }; + return { logged, debug: at("debug"), info: at("info"), warn: at("warn"), error: at("error") }; +}; + +const stops: (() => Promise)[] = []; + +afterEach(async () => { + for (const stop of stops.splice(0)) await stop(); +}); + +let started = 0; + +const start = async (args: { + readonly adapter: ReturnType["adapter"]; + readonly runtime?: RuntimeConfig; + readonly env?: Record; + readonly logger?: Logger; +}): Promise<{ app: BoundaApp; clock: ReturnType }> => { + started += 1; + const clock = createFixedClock(START); + const app = await createApp({ + registry, + config: { + storage: args.adapter, + ...(args.runtime === undefined ? {} : { runtime: args.runtime }), + }, + env: args.env ?? {}, + ids: createSequentialIdGenerator({ prefix: `app-${started}` }), + clock, + logger: args.logger ?? silentLogger, + }); + stops.push(() => app.stop()); + return { app, clock }; +}; + +const register = async (app: BoundaApp, customerId: string): Promise => { + await app.commands.registerCustomer?.({ customerId, email: `${customerId}@example.com` }); +}; + +const keyRows = (db: DatabaseSync) => + db + .prepare( + `SELECT "person", "kid", "held_until", "attempts" FROM "bounda_personal_keys" ORDER BY "person"`, + ) + .all() as { person: string; kid: string | null; held_until: string | null; attempts: number }[]; + +describe("a database restored while the app runs", () => { + it("has the rotation start over at once, without the holds the copy brought back", async () => { + const { adapter, db } = createNodeSqliteAdapter(); + const old = randomKey(); + const before = await start({ + adapter, + env: { BOUNDA_MASTER_KEY: old }, + runtime: { personalData: { keyWrapper: masterKey({ from: "BOUNDA_MASTER_KEY" }) } }, + }); + await register(before.app, "c-1"); + await register(before.app, "c-2"); + await before.app.stop(); + + const logger = recordingLogger(); + const { app, clock } = await start({ + adapter, + env: { BOUNDA_MASTER_KEY: randomKey(), BOUNDA_MASTER_KEY_PREVIOUS: old }, + runtime: { + personalData: { + keyWrapper: masterKey({ + from: "BOUNDA_MASTER_KEY", + previous: ["BOUNDA_MASTER_KEY_PREVIOUS"], + }), + }, + }, + logger, + }); + // A key that failed to rewrap for a while: held for long, with its attempts counted. + db.prepare( + `UPDATE "bounda_personal_keys" SET "held_until" = ?, "attempts" = 9 WHERE "person" = 'c-2'`, + ).run(new Date(START.getTime() + HOLD_MAX_MS).toISOString()); + await app.runUntilIdle(); + const [, oldKid] = keyRows(db).map(({ kid }) => kid); + // The backup: c-1 rewrapped, c-2 still under the old key and held. + const copy = copyOf(db); + db.prepare( + `UPDATE "bounda_personal_keys" SET "held_until" = NULL WHERE "person" = 'c-2'`, + ).run(); + clock.advance(ROTATION_CHECK_INTERVAL_MS); + await app.runUntilIdle(); + const newKid = keyRows(db)[0]?.kid; + expect(keyRows(db).map(({ kid }) => kid)).toEqual([newKid, newKid]); + expect(await app.nextDueAt()).toBeNull(); + + restore(db, copy); + expect(keyRows(db)[1]).toMatchObject({ kid: oldKid, attempts: 9 }); + await app.runUntilIdle(); + expect(keyRows(db)).toEqual([ + { person: "c-1", kid: newKid, held_until: null, attempts: 0 }, + { person: "c-2", kid: newKid, held_until: null, attempts: 0 }, + ]); + expect(logger.logged).toContainEqual({ + level: "warn", + message: "the database went back to an earlier copy; background jobs start over", + fields: { first: true }, + }); + expect(await app.status()).toMatchObject({ restoresFound: 1 }); + }); + + it("forgets again at once whom the copy brings back, without waiting for the periodic reconciliation", async () => { + const { adapter, db } = createNodeSqliteAdapter(); + const ledger = memory(); + const { app, clock } = await start({ + adapter, + runtime: { personalData: { forgetLedger: { store: ledger, reconcileEvery: "1h" } } }, + }); + await register(app, "c-1"); + await app.runUntilIdle(); + const copy = copyOf(db); + clock.advance(GENERATION_INTERVAL_MS); + await app.forget("c-1"); + await app.runUntilIdle(); + restore(db, copy); + expect(db.prepare(`SELECT count(*) AS "keys" FROM "bounda_personal_keys"`).get()).toEqual({ + keys: 1, + }); + await app.runUntilIdle(); + expect(db.prepare(`SELECT count(*) AS "keys" FROM "bounda_personal_keys"`).get()).toEqual({ + keys: 0, + }); + const reasons = (await app.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => (event.payload as PersonalDataForgottenPayload).reason); + expect(reasons).toEqual(["restored"]); + }); +}); diff --git a/packages/core/src/kernel/shared/restores.ts b/packages/core/src/kernel/shared/restores.ts new file mode 100644 index 00000000..754a8274 --- /dev/null +++ b/packages/core/src/kernel/shared/restores.ts @@ -0,0 +1,100 @@ +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import type { Logger } from "../../contracts/logger.ts"; + +/** + * The checkpoint row every running instance moves one forward, at most once every + * `GENERATION_INTERVAL_MS`: it only ever grows, so a database that holds less than an instance + * read from it went back to an earlier copy. + */ +export const RESTORE_GENERATION = "runtime:generation"; + +/** + * The checkpoint row that counts the restores the instances found, moved by the first to find + * one, so the others, whose own reading may not show it, learn of it too. + */ +export const RESTORES_FOUND = "runtime:restores"; + +export const GENERATION_INTERVAL_MS = 5_000; + +export interface Restored { + /** + * Whether this instance found it first: the one that clears what the copy brought back of the + * holds, which a copy holds once. + */ + readonly first: boolean; +} + +export interface RestoreWatch { + /** + * Reads the rows and resolves to what this instance learnt: `null` while the database holds + * at least what it read last. A database restored while this instance was stopped is not + * told apart from one left alone. Never rejects: a store that fails is logged, and the next + * check reads again. + */ + check(now: Date): Promise; + /** + * How many restores the instances found, as the database counts them. + */ + found(): Promise; +} + +export interface CreateRestoreWatchArgs { + readonly checkpoints: CheckpointStore; + readonly logger: Logger; +} + +export interface CreateRestoreWatchFunction { + (args: CreateRestoreWatchArgs): RestoreWatch; +} + +/** + * A generation that went back, or a count of restores that changed, means a restore. The first + * instance to see the generation go back moves the count past every value an instance can have + * read, so each other instance finds it changed, whether its own reading of the generation shows + * the restore or not. A copy taken within one interval of the restore may hold the generation + * every instance read last, and goes unseen: what it undid is that recent. + */ +export const createRestoreWatch: CreateRestoreWatchFunction = ({ checkpoints, logger }) => { + let seen: { readonly generation: number; readonly restores: number } | null = null; + let movedAt = Number.NEGATIVE_INFINITY; + + const check = async (now: Date): Promise => { + const at = now.getTime(); + const [generation, restores] = await Promise.all([ + checkpoints.get(RESTORE_GENERATION), + checkpoints.get(RESTORES_FOUND), + ]); + let restored: Restored | null = null; + let counted = restores; + if (seen !== null && (generation < seen.generation || restores !== seen.restores)) { + // A count past what this instance read was moved by the instance that found it first. + if (restores > seen.restores) { + restored = { first: false }; + } else { + counted = seen.restores + 1; + const first = await checkpoints.compareAndSet(RESTORES_FOUND, restores, counted); + if (!first) counted = await checkpoints.get(RESTORES_FOUND); + restored = { first }; + } + movedAt = Number.NEGATIVE_INFINITY; + } + let current = generation; + if (at - movedAt >= GENERATION_INTERVAL_MS) { + if (await checkpoints.compareAndSet(RESTORE_GENERATION, generation, generation + 1)) { + current = generation + 1; + } + movedAt = at; + } + seen = { generation: current, restores: counted }; + return restored; + }; + + return { + check: (now) => + check(now).catch((error: unknown) => { + logger.error("the database could not be checked for a restore", { error }); + return null; + }), + found: () => checkpoints.get(RESTORES_FOUND), + }; +}; diff --git a/packages/core/src/memory/person-store.ts b/packages/core/src/memory/person-store.ts index a623380b..d925554d 100644 --- a/packages/core/src/memory/person-store.ts +++ b/packages/core/src/memory/person-store.ts @@ -98,6 +98,21 @@ export const createKeptPersonStore: CreateKeptPersonStoreFunction = () => { [...persons.values()].filter(isIdle).map(({ hold }) => hold), after, ), + countIdle: async ({ activeBefore }) => { + const before = activeBefore?.toISOString(); + return [...persons.values()] + .filter(isIdle) + .filter((kept) => before === undefined || kept.lastActiveAt <= before).length; + }, + releaseHolds: async () => { + let released = 0; + for (const [person, kept] of persons) { + if (kept.hold === null) continue; + persons.set(person, { ...kept, hold: null }); + released += 1; + } + return released; + }, }, entries: createStoreEntries(persons, (person) => person), }; diff --git a/packages/core/src/memory/personal-key-store.ts b/packages/core/src/memory/personal-key-store.ts index 187bcae0..f40046d2 100644 --- a/packages/core/src/memory/personal-key-store.ts +++ b/packages/core/src/memory/personal-key-store.ts @@ -84,6 +84,17 @@ export const createKeptPersonalKeyStore: CreateKeptPersonalKeyStoreFunction = () [...keys.values()].filter((key) => kids.includes(key.kid)).map(({ hold }) => hold), after, ), + countWrappedUnder: async ({ kids }) => + [...keys.values()].filter((key) => kids.includes(key.kid)).length, + releaseHolds: async () => { + let released = 0; + for (const [person, kept] of keys) { + if (kept.hold === null) continue; + keys.set(person, { ...kept, hold: null }); + released += 1; + } + return released; + }, }, entries: createStoreEntries(keys, (person) => person), }; diff --git a/packages/core/src/memory/transaction.ts b/packages/core/src/memory/transaction.ts index 0ab7ac0b..36271fed 100644 --- a/packages/core/src/memory/transaction.ts +++ b/packages/core/src/memory/transaction.ts @@ -142,6 +142,9 @@ const trackedPersonalKeys = ( hold: (person, from, hold) => tracked(entries, person, () => personalKeys.hold(person, from, hold)), nextHoldEnd: (args) => personalKeys.nextHoldEnd(args), + countWrappedUnder: (args) => personalKeys.countWrappedUnder(args), + // Only after a restore, never inside a transaction. + releaseHolds: () => personalKeys.releaseHolds(), }; }; @@ -172,6 +175,9 @@ const trackedPersons = ( hold: (person, lastActiveAt, hold) => tracked(entries, person, () => persons.hold(person, lastActiveAt, hold)), nextHoldEnd: (after) => persons.nextHoldEnd(after), + countIdle: (args) => persons.countIdle(args), + // Only after a restore, never inside a transaction. + releaseHolds: () => persons.releaseHolds(), }; }; From ffcd6c8fdd640bb27dccd81b13cd5e8d217cf81b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:14:49 +0100 Subject: [PATCH 02/19] feat(core): end a process whose failure is discarded with ProcessAbandoned --- .../src/kernel/dead-letters/access.test.ts | 2 +- .../src/kernel/dead-letters/dead-letters.ts | 12 ++- .../src/kernel/process/dead-letter-retry.ts | 48 ++++++++- .../core/src/kernel/process/deliver-event.ts | 34 ++----- .../core/src/kernel/process/lifecycle.test.ts | 30 ++++++ packages/core/src/kernel/process/lifecycle.ts | 20 +++- .../src/kernel/process/parked.fuzz.test.ts | 30 +++--- .../core/src/kernel/process/parked.test.ts | 98 ++++++++++++++++++- packages/core/src/kernel/process/runner.ts | 1 + 9 files changed, 224 insertions(+), 51 deletions(-) diff --git a/packages/core/src/kernel/dead-letters/access.test.ts b/packages/core/src/kernel/dead-letters/access.test.ts index f15a8bca..e7c023ec 100644 --- a/packages/core/src/kernel/dead-letters/access.test.ts +++ b/packages/core/src/kernel/dead-letters/access.test.ts @@ -36,7 +36,7 @@ describe("the access log of dead letters", () => { const deadLetters = createDeadLetters({ storage, retries: {} as DeadLetterRetry, - processes: {} as Pick, + processes: {} as Pick, clock: createFixedClock(), logger: silentLogger, errorText: "omit", diff --git a/packages/core/src/kernel/dead-letters/dead-letters.ts b/packages/core/src/kernel/dead-letters/dead-letters.ts index 441ab1a3..98307b10 100644 --- a/packages/core/src/kernel/dead-letters/dead-letters.ts +++ b/packages/core/src/kernel/dead-letters/dead-letters.ts @@ -55,8 +55,11 @@ export interface DeadLetters { */ retry(id: string): Promise; /** - * Marks the letter `discarded`. Rejects a letter that is missing, and with - * `DeadLetterSettledError` one that is no longer `failed`. + * Marks the letter `discarded`. A process letter whose instance is still failed on it ends the + * instance with `ProcessAbandoned` in the same transaction, acted by the client's actor: the + * events parked behind it stay unhandled, and later ones are no longer parked. Rejects a letter + * that is missing, and with `DeadLetterSettledError` one that is no longer `failed`, or that a + * retry settled while this ran. */ discard(id: string): Promise; } @@ -64,7 +67,7 @@ export interface DeadLetters { export interface CreateDeadLettersArgs { readonly storage: AppStorage; readonly retries: DeadLetterRetry; - readonly processes: Pick; + readonly processes: Pick; readonly clock: Clock; readonly logger: Logger; readonly errorText: ErrorText; @@ -202,7 +205,8 @@ export const createDeadLetters: CreateDeadLettersFunction = ({ const letter = await failedLetter(id); return recordedLetter("deadLetters.discard", letter, async () => { const at = clock.now(); - await storage.deadLetterStore.updateStatus(id, "discarded", at); + if (letter.kind === "process") await processes.abandon({ letter, at, actor }); + else await storage.deadLetterStore.updateStatus(id, "discarded", at); logger.info("dead letter discarded", { id, kind: letter.kind, diff --git a/packages/core/src/kernel/process/dead-letter-retry.ts b/packages/core/src/kernel/process/dead-letter-retry.ts index ec79dcc1..9f0bdab4 100644 --- a/packages/core/src/kernel/process/dead-letter-retry.ts +++ b/packages/core/src/kernel/process/dead-letter-retry.ts @@ -8,6 +8,7 @@ import { import type { StoredEvent } from "../../contracts/event.ts"; import type { Logger } from "../../contracts/logger.ts"; import type { CausationContext } from "../../contracts/metadata.ts"; +import { actedBy } from "../shared/causation.ts"; import { runtimeText } from "../shared/runtime-text.ts"; import type { ProcessesRuntime, ProcessRuntime } from "./build-processes.ts"; import type { DeadlineStep } from "./deadline-step.ts"; @@ -15,7 +16,7 @@ import { PROCESS_DEADLINE_COMMAND, reachedKey } from "./deadlines.ts"; import type { DeadlineTarget } from "./deliver-deadline.ts"; import { blockedOn } from "./failures.ts"; import type { ProcessHandlers } from "./handlers.ts"; -import { lifecycleEntries, type ProcessInstance } from "./lifecycle.ts"; +import { instanceContext, lifecycleEntries, type ProcessInstance } from "./lifecycle.ts"; import type { ResumeParked } from "./resume.ts"; import { completesOn, handledEntries, handlerOf, letThrough, pendingFollowUp } from "./routes.ts"; import type { DeadlineSchedule } from "./schedule.ts"; @@ -78,6 +79,18 @@ export interface ProcessDeadLetters { * once it resumed. */ stillParked(letter: StoredDeadLetter): Promise; + /** + * Discards a process dead letter: marks it `discarded` at `at` and, while its instance is failed + * on it, ends the instance with `ProcessAbandoned`, acted by `actor`, in one unit. Rejects with + * `DeadLetterSettledError` a letter no longer `failed`, writing nothing. + */ + abandon(args: AbandonProcessArgs): Promise; +} + +export interface AbandonProcessArgs { + readonly letter: StoredDeadLetter; + readonly at: Date; + readonly actor?: string | undefined; } export interface CreateProcessRetryArgs { @@ -251,7 +264,11 @@ export const createProcessRetry: CreateProcessRetryFunction = ({ await resume.resumeParked(process, payload.aggregateId, letter, context.actor); }; - const instanceOfLetter = async (letter: StoredDeadLetter): Promise => { + // The process and instance a letter is about; `null` once the process left the registry or its + // event can no longer be read. + const subjectOfLetter = async ( + letter: StoredDeadLetter, + ): Promise<{ readonly process: ProcessRuntime; readonly instanceId: string } | null> => { const process = processes.byName[letter.handler]; if (letter.kind !== "process" || process === undefined) return null; const instanceId = @@ -266,7 +283,12 @@ export const createProcessRetry: CreateProcessRetryFunction = ({ }) .then((event) => process.instanceOf(event)) .catch(() => null); - return instanceId === null ? null : load(process, instanceId); + return instanceId === null ? null : { process, instanceId }; + }; + + const instanceOfLetter = async (letter: StoredDeadLetter): Promise => { + const subject = await subjectOfLetter(letter); + return subject === null ? null : load(subject.process, subject.instanceId); }; return { @@ -279,6 +301,26 @@ export const createProcessRetry: CreateProcessRetryFunction = ({ ? instance.parked.filter((parked) => parked.eventId !== failure.eventId).length : 0; }, + abandon: async ({ letter, at, actor }) => { + const subject = await subjectOfLetter(letter); + await units.commit(async (unit, within) => { + const stored = await unit.deadLetterStore.get(letter.id); + if (stored?.status !== "failed") throw new DeadLetterSettledError({ id: letter.id }); + await unit.deadLetterStore.updateStatus(letter.id, "discarded", at); + if (subject === null) return; + const { process, instanceId } = subject; + const current = await within.load(process, instanceId); + if (current.status !== "failed" || !blockedOn(current, letter.id)) return; + await within.append(process, instanceId, current, [ + lifecycleEntries.abandoned(letter.id, { + ...instanceContext(process, instanceId, current), + ...actedBy(actor), + }), + ]); + await schedule.stage(unit, process, instanceId); + }); + logger.info("process abandoned", { process: letter.handler, letterId: letter.id }); + }, stillParked: async (letter) => { const instance = await instanceOfLetter(letter); return instance?.status === "failed" diff --git a/packages/core/src/kernel/process/deliver-event.ts b/packages/core/src/kernel/process/deliver-event.ts index 85491a28..1170608f 100644 --- a/packages/core/src/kernel/process/deliver-event.ts +++ b/packages/core/src/kernel/process/deliver-event.ts @@ -65,28 +65,12 @@ export const createEventDelivery: CreateEventDeliveryFunction = ({ const leaseMs = (process: ProcessRuntime): number => process.handlerTimeoutMs * 2; // Whether `event` waits behind the instance's failure: not the event that failed, delivered - // again, and nothing once the failure's letter was discarded, since the instance is given up. - // A letter is written with the failure, so one that is gone was purged once resolved: an - // instance still failed on it was given up. - const parks = async ( - unit: UnitOfWork, - process: ProcessRuntime, - event: StoredEvent, - instance: ProcessInstance, - ): Promise => { - if ( - !actsOn(process, event) || - event.id === instance.failure?.eventId || - instance.handledEventIds.has(event.id) || - instance.parked.some((parked) => parked.eventId === event.id) - ) { - return false; - } - const letterId = instance.failure?.letterId; - if (letterId === undefined) return true; - const letter = await unit.deadLetterStore.get(letterId); - return letter !== null && letter.status !== "discarded"; - }; + // again. An instance given up is `abandoned` and parks nothing, whatever became of its letter. + const parks = (process: ProcessRuntime, event: StoredEvent, instance: ProcessInstance): boolean => + actsOn(process, event) && + event.id !== instance.failure?.eventId && + !instance.handledEventIds.has(event.id) && + !instance.parked.some((parked) => parked.eventId === event.id); // The instance with its start staged in `entries` when `event` starts it; as it is otherwise. const started = ( @@ -115,7 +99,7 @@ export const createEventDelivery: CreateEventDeliveryFunction = ({ if (!instance.exists) return false; let parked = false; if (instance.status === "failed") { - if (await parks(unit, process, event, instance)) { + if (parks(process, event, instance)) { entries.push(lifecycleEntries.parked(event)); parked = true; } @@ -264,7 +248,9 @@ export const createEventDelivery: CreateEventDeliveryFunction = ({ const instance = await units.live.load(process, instanceId); if (followsUp(process, instance, event)) return handle(process, event, instanceId); const ended = instance.exists - ? instance.status === "completed" || instance.status === "timed_out" + ? instance.status === "completed" || + instance.status === "timed_out" || + instance.status === "abandoned" : !startsOn(process, event); if (ended) return "done"; if ( diff --git a/packages/core/src/kernel/process/lifecycle.test.ts b/packages/core/src/kernel/process/lifecycle.test.ts index a74fac25..33801947 100644 --- a/packages/core/src/kernel/process/lifecycle.test.ts +++ b/packages/core/src/kernel/process/lifecycle.test.ts @@ -7,6 +7,7 @@ import { type FoldProcessArgs, foldProcess, instanceContext, + lifecycleEntries, PROCESS_EVENTS, type ProcessInstance, processAggregateType, @@ -107,6 +108,7 @@ describe("foldProcess", () => { failed: "ProcessFailed", eventParked: "ProcessEventParked", resumed: "ProcessResumed", + abandoned: "ProcessAbandoned", }); }); @@ -373,4 +375,32 @@ describe("deadlineContext", () => { depth: 0, }); }); + + it("ends a failed process with ProcessAbandoned, and only a failed one", () => { + const failed = [ + lifecycle(PROCESS_EVENTS.started, { state: { step: 0 } }, 1), + lifecycle(PROCESS_EVENTS.failed, { eventId: "e2", letterId: "l-1" }, 2), + ]; + expect( + fold({ + initialState: {}, + events: [...failed, lifecycle(PROCESS_EVENTS.abandoned, { letterId: "l-1" }, 3)], + }), + ).toMatchObject({ status: "abandoned", failure: { eventId: "e2", letterId: "l-1" } }); + expect( + fold({ + initialState: {}, + events: [ + lifecycle(PROCESS_EVENTS.started, { state: { step: 0 } }, 1), + lifecycle(PROCESS_EVENTS.abandoned, { letterId: "l-1" }, 2), + ], + }).status, + ).toBe("started"); + const context = { correlationId: "c", causationId: "x", depth: 0 }; + expect(lifecycleEntries.abandoned("l-1", context)).toEqual({ + type: "ProcessAbandoned", + payload: { letterId: "l-1" }, + context, + }); + }); }); diff --git a/packages/core/src/kernel/process/lifecycle.ts b/packages/core/src/kernel/process/lifecycle.ts index 23dde01d..a82b10ce 100644 --- a/packages/core/src/kernel/process/lifecycle.ts +++ b/packages/core/src/kernel/process/lifecycle.ts @@ -8,9 +8,10 @@ import type { ProcessRuntime } from "./build-processes.ts"; import { type Deadline, deadlineAt, reachedKey, TIMEOUT_DEADLINE } from "./deadlines.ts"; /** - * Status of a process instance, derived from its lifecycle events. + * Status of a process instance, derived from its lifecycle events. `abandoned` is a failed + * instance whose failure's dead letter was discarded: it ends there, as `completed` does. */ -export type ProcessStatus = "started" | "completed" | "failed" | "timed_out"; +export type ProcessStatus = "started" | "completed" | "failed" | "timed_out" | "abandoned"; export const PROCESS_EVENTS: { readonly started: "ProcessStarted"; @@ -21,6 +22,7 @@ export const PROCESS_EVENTS: { readonly failed: "ProcessFailed"; readonly eventParked: "ProcessEventParked"; readonly resumed: "ProcessResumed"; + readonly abandoned: "ProcessAbandoned"; } = { started: "ProcessStarted", handled: "ProcessHandled", @@ -30,6 +32,7 @@ export const PROCESS_EVENTS: { failed: "ProcessFailed", eventParked: "ProcessEventParked", resumed: "ProcessResumed", + abandoned: "ProcessAbandoned", }; /** @@ -175,8 +178,8 @@ const causeOf = (event: StoredEvent): DeadlineCause => { }; /** - * A failed instance stays failed until `ProcessResumed`; a `ProcessHandled` takes its event off - * the parked ones. + * A failed instance stays failed until `ProcessResumed`, or ends with `ProcessAbandoned`; a + * `ProcessHandled` takes its event off the parked ones. */ export const foldProcess: FoldProcessFunction = ({ initialState, deadlineFields, events }) => { const handled = new Set(); @@ -274,6 +277,9 @@ export const foldProcess: FoldProcessFunction = ({ initialState, deadlineFields, case PROCESS_EVENTS.resumed: if (status === "failed") status = "started"; break; + case PROCESS_EVENTS.abandoned: + if (status === "failed") status = "abandoned"; + break; default: break; } @@ -353,6 +359,7 @@ export interface LifecycleEntries { failed(failedOn: FailedOn, letter: NewDeadLetter, context: CausationContext): LifecycleEntry; parked(event: StoredEvent): LifecycleEntry; resumed(context: CausationContext): LifecycleEntry; + abandoned(letterId: string, context: CausationContext): LifecycleEntry; deadlineReached( due: Deadline, state: object, @@ -404,6 +411,11 @@ export const lifecycleEntries: LifecycleEntries = { context: causedBy(event), }), resumed: (context) => ({ type: PROCESS_EVENTS.resumed, payload: {}, context }), + abandoned: (letterId, context) => ({ + type: PROCESS_EVENTS.abandoned, + payload: { letterId }, + context, + }), deadlineReached: (due, state, context, id) => ({ type: PROCESS_EVENTS.deadlineReached, payload: { field: due.field, at: due.at, state }, diff --git a/packages/core/src/kernel/process/parked.fuzz.test.ts b/packages/core/src/kernel/process/parked.fuzz.test.ts index 202d4cf6..0ea35ac1 100644 --- a/packages/core/src/kernel/process/parked.fuzz.test.ts +++ b/packages/core/src/kernel/process/parked.fuzz.test.ts @@ -256,13 +256,7 @@ const run = async (seed: number): Promise => { aggregateType: "process:order.ledger", aggregateId: order, }); - const instance = fold({ initialState: {}, events }); - const blocking = instance.failure?.letterId; - const letter = - blocking === undefined ? null : await harness.storage.deadLetterStore.get(blocking); - return instance.status === "failed" && letter?.status === "discarded" - ? "abandoned" - : instance.status; + return fold({ initialState: {}, events }).status; }), ); const failed = await harness.storage.deadLetterStore.list({ status: "failed" }); @@ -328,10 +322,15 @@ const run = async (seed: number): Promise => { .filter((id) => id?.startsWith("pay-") === true || id === template.id); const instance = fold({ initialState: {}, events }); const blocking = instance.failure?.letterId; - const abandoned = - instance.status === "failed" && - blocking !== undefined && - (await harness.storage.deadLetterStore.get(blocking))?.status === "discarded"; + const abandoned = instance.status === "abandoned"; + if (abandoned) { + expect( + blocking === undefined + ? null + : (await harness.storage.deadLetterStore.get(blocking))?.status, + `${where}: ${order} was abandoned by discarding its failure's letter`, + ).toBe("discarded"); + } const orderEvents = ( await harness.storage.eventStore.load({ aggregateType: "order", aggregateId: order }) ).events; @@ -360,8 +359,13 @@ const run = async (seed: number): Promise => { (event.payload as { letterId?: string }).letterId === instance.failure?.letterId, ); expect( - events.slice(after + 1).filter((event) => event.type !== PROCESS_EVENTS.eventParked) - .length + 0, + events + .slice(after + 1) + .filter( + (event) => + event.type !== PROCESS_EVENTS.eventParked && + event.type !== PROCESS_EVENTS.abandoned, + ).length, `${where}: ${order} ran nothing once given up`, ).toBe(0); } else { diff --git a/packages/core/src/kernel/process/parked.test.ts b/packages/core/src/kernel/process/parked.test.ts index 3b70df32..d66c1868 100644 --- a/packages/core/src/kernel/process/parked.test.ts +++ b/packages/core/src/kernel/process/parked.test.ts @@ -368,7 +368,7 @@ describe("events of a failed process", () => { expect(await deadLetters.get("nope")).toBeNull(); }); - it("stay given up once the discarded failure's letter is purged", async () => { + it("stay given up once the discarded failure's letter is purged: the instance is abandoned", async () => { const context = await setUp(); const { harness, deadLetters, settle, types } = context; await failOnFirstPayment(context); @@ -391,13 +391,107 @@ describe("events of a failed process", () => { }); await settle(); expect((await types()).filter((type) => type === PROCESS_EVENTS.eventParked)).toEqual([]); - expect((await types()).at(-1)).toBe(PROCESS_EVENTS.failed); + expect((await types()).slice(-2)).toEqual([PROCESS_EVENTS.failed, PROCESS_EVENTS.abandoned]); expect(await harness.storage.checkpointStore.get(PROCESSES_SUBSCRIBER)).toBe( await harness.storage.eventStore.lastPosition(), ); expect(await harness.storage.deadLetterStore.list()).toEqual([]); }); + it("keep parking behind a failure whose letter is gone without a discard: only a discard abandons", async () => { + const context = await setUp(); + const { harness, deadLetters, settle, types } = context; + await failOnFirstPayment(context); + const [letter] = await deadLetters.list(); + await harness.storage.deadLetterStore.updateStatus( + letter?.id ?? "", + "retried", + harness.clock.now(), + ); + await harness.storage.deadLetterStore.purge({ resolvedBy: harness.clock.now(), limit: 10 }); + const order = await harness.storage.eventStore.load({ + aggregateType: "order", + aggregateId: "o-1", + }); + const last = order.events.at(-1); + if (last === undefined) throw new Error("no order"); + await harness.storage.eventStore.append({ + aggregateType: "order", + aggregateId: "o-1", + expectedVersion: order.events.length, + events: [{ ...last, id: "after-purge", version: order.events.length + 1 }], + }); + await settle(); + expect((await types()).slice(-2)).toEqual([PROCESS_EVENTS.failed, PROCESS_EVENTS.eventParked]); + }); + + it("abandon the instance as whoever discards, once, and refuse a discard a retry settled", async () => { + const context = await setUp(); + const { harness, settle } = context; + await failOnFirstPayment(context); + const actor = createDeadLetters({ + recordAccess: unrecordedAccess, + storage: harness.storage, + retries: harness.retries, + processes: harness.processes, + clock: harness.clock, + logger: harness.logger, + errorText: "omit", + actor: "operator-7", + }); + const [letter] = await actor.list(); + const id = letter?.id ?? ""; + await actor.discard(id); + const abandoned = (await harness.storage.eventStore.load(stream)).events.at(-1); + expect(abandoned).toMatchObject({ + type: PROCESS_EVENTS.abandoned, + payload: { letterId: id }, + metadata: { actor: "operator-7" }, + }); + await expect(actor.discard(id)).rejects.toMatchObject({ code: "DEAD_LETTER_SETTLED" }); + await settle(); + expect( + (await harness.storage.eventStore.load(stream)).events.filter( + (event) => event.type === PROCESS_EVENTS.abandoned, + ), + ).toHaveLength(1); + }); + + it("abandon nothing for a letter its instance is not failed on", async () => { + const context = await setUp(); + const { harness, deadLetters, types } = context; + await failOnFirstPayment(context); + const [letter] = await harness.storage.deadLetterStore.list(); + if (letter === undefined) throw new Error("no letter"); + const { status: _status, resolvedAt: _resolvedAt, ...stale } = letter; + await harness.storage.deadLetterStore.add({ ...stale, id: "stale" }); + const before = await types(); + expect((await deadLetters.discard("stale")).status).toBe("discarded"); + expect(await types()).toEqual(before); + expect((await deadLetters.list())[0]).toMatchObject({ id: letter.id, status: "failed" }); + }); + + it("refuse a discard a retry settled while it ran, abandoning nothing", async () => { + const context = await setUp(); + const { harness, deadLetters, types } = context; + await failOnFirstPayment(context); + const [letter] = await deadLetters.list(); + const before = await types(); + const { get } = harness.storage.deadLetterStore; + let reads = 0; + harness.storage.deadLetterStore.get = async (id) => { + reads += 1; + const stored = await get(id); + return reads === 1 || stored === null ? stored : { ...stored, status: "retried" }; + }; + await expect(deadLetters.discard(letter?.id ?? "")).rejects.toMatchObject({ + code: "DEAD_LETTER_SETTLED", + }); + harness.storage.deadLetterStore.get = get; + expect(await types()).toEqual(before); + expect((await harness.storage.deadLetterStore.get(letter?.id ?? ""))?.status).toBe("failed"); + }); + it("wait behind a failed deadline, which runs first when retried", async () => { const context = await setUp(); const { harness, deadLetters, settle, types } = context; diff --git a/packages/core/src/kernel/process/runner.ts b/packages/core/src/kernel/process/runner.ts index 16b1abcc..9ea95ffd 100644 --- a/packages/core/src/kernel/process/runner.ts +++ b/packages/core/src/kernel/process/runner.ts @@ -117,5 +117,6 @@ export const createProcessRunner: CreateProcessRunnerFunction = ({ retryDeadline: deadLetters.retryDeadline, parkedBehind: deadLetters.parkedBehind, stillParked: deadLetters.stillParked, + abandon: deadLetters.abandon, }; }; From fb2660ea65c81ceab0cd4d7a3eb2bf64c03a438c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:26:44 +0100 Subject: [PATCH 03/19] feat: keep each subscriber's last failure sealed, and read the runtime's status and schema versions --- packages/cloudflare/src/bounda-object.ts | 27 +++ packages/cloudflare/src/client.ts | 11 ++ packages/cloudflare/test/client.test.ts | 6 + packages/cloudflare/test/recording-stub.ts | 14 +- .../cloudflare/test/storage-contracts.test.ts | 2 + packages/core/src/adapter/adapter.ts | 37 ++++ packages/core/src/adapter/index.ts | 7 + packages/core/src/adapter/sql/index.ts | 10 +- packages/core/src/adapter/sql/migrations.ts | 64 +++++-- .../adapter/sql/subscriber-failure-store.ts | 88 ++++++++++ packages/core/src/adapter/sqlite/adapter.ts | 50 ++++++ .../core/src/adapter/sqlite/migrations.ts | 18 +- packages/core/src/adapter/sqlite/schema.ts | 24 +++ .../core/src/adapter/sqlite/sqlite.test.ts | 3 + .../storage/subscriber-failure-store.ts | 63 +++++++ packages/core/src/adapter/testing/index.ts | 5 + .../testing/storage-schema.contract.ts | 5 +- .../subscriber-failure-store.contract.ts | 134 ++++++++++++++ packages/core/src/index.ts | 14 +- packages/core/src/kernel/access-log.test.ts | 8 +- packages/core/src/kernel/app.ts | 45 +++++ .../core/src/kernel/dispatch/delivery.test.ts | 4 + packages/core/src/kernel/dispatch/delivery.ts | 13 +- .../dispatch/dispatcher-backoff.test.ts | 23 ++- .../dispatch/dispatcher-catch-up.test.ts | 17 ++ .../core/src/kernel/dispatch/dispatcher.ts | 28 ++- .../core/src/kernel/dispatch/last-failures.ts | 79 +++++++++ .../kernel/dispatch/subscriber-failures.ts | 115 ++++++++++++ .../src/kernel/personal-data/export.test.ts | 1 + .../core/src/kernel/personal-data/export.ts | 28 +++ packages/core/src/kernel/status.ts | 164 ++++++++++++++++++ packages/core/src/memory/index.ts | 5 + packages/core/src/memory/memory.test.ts | 2 + .../src/memory/subscriber-failure-store.ts | 43 +++++ packages/core/src/modules/audit.ts | 26 ++- packages/core/src/modules/index.ts | 2 + packages/core/test-types/inference.test-d.ts | 107 ++++++++++++ packages/postgresql/src/index.ts | 37 ++++ packages/postgresql/src/migrations.ts | 40 ++++- packages/postgresql/src/postgresql.test.ts | 2 + packages/postgresql/src/schema.ts | 26 +++ .../src/cloudflare/clients.test.ts | 6 + .../react-router/src/cloudflare/clients.ts | 3 + .../src/cloudflare/test-support.ts | 3 + packages/sqlite/src/sqlite.test.ts | 5 + 45 files changed, 1382 insertions(+), 32 deletions(-) create mode 100644 packages/core/src/adapter/sql/subscriber-failure-store.ts create mode 100644 packages/core/src/adapter/storage/subscriber-failure-store.ts create mode 100644 packages/core/src/adapter/testing/subscriber-failure-store.contract.ts create mode 100644 packages/core/src/kernel/dispatch/last-failures.ts create mode 100644 packages/core/src/kernel/dispatch/subscriber-failures.ts create mode 100644 packages/core/src/kernel/status.ts create mode 100644 packages/core/src/memory/subscriber-failure-store.ts diff --git a/packages/cloudflare/src/bounda-object.ts b/packages/cloudflare/src/bounda-object.ts index 17d4d702..066941f6 100644 --- a/packages/cloudflare/src/bounda-object.ts +++ b/packages/cloudflare/src/bounda-object.ts @@ -1,6 +1,7 @@ import { DurableObject } from "cloudflare:workers"; import { type AppLag, + type AppStatus, type BoundaApp, type BoundaClient, type Clock, @@ -22,6 +23,7 @@ import { type Registry, type StoredDispatch, type StoredEvent, + type SubscriberFailure, systemClock, } from "@bounda-dev/core"; import type { Config } from "@bounda-dev/core/config"; @@ -82,6 +84,16 @@ export interface BoundaObjectMethods { ): Promise>; query(name: string, payload?: unknown, actor?: string): Promise>; lag(): Promise>; + status(): Promise>; + /** + * Each subscriber's last failure with its text opened, recorded in the access log as + * `BoundaClient.subscriberFailures` records it. + */ + listSubscriberFailures(actor?: string): Promise>; + getSubscriberFailure( + subscriber: string, + actor?: string, + ): Promise>; readEvents(args?: ReadEventsArgs): Promise>; listDeadLetters( args?: ListDeadLettersArgs, @@ -318,6 +330,21 @@ export const createBoundaObject: CreateBoundaObjectFunction = this.#ready().getLag()); } + status(): Promise> { + return settle(() => this.#ready().status()); + } + + listSubscriberFailures(actor?: string): Promise> { + return settle(() => this.#as(actor).subscriberFailures.list()); + } + + getSubscriberFailure( + subscriber: string, + actor?: string, + ): Promise> { + return settle(() => this.#as(actor).subscriberFailures.get(subscriber)); + } + readEvents(args?: ReadEventsArgs): Promise> { return settle(() => this.#ready().events.read(args)); } diff --git a/packages/cloudflare/src/client.ts b/packages/cloudflare/src/client.ts index 76cbc7fe..4af30892 100644 --- a/packages/cloudflare/src/client.ts +++ b/packages/cloudflare/src/client.ts @@ -1,6 +1,7 @@ import { type AppLag, type AppRegistry, + type AppStatus, type BoundaClient, type CommandsFacade, type Consistency, @@ -17,6 +18,7 @@ import { type Registry, type StoredDispatch, type StoredEvent, + type SubscriberFailure, } from "@bounda-dev/core"; import { unwrap } from "./outcome.ts"; @@ -35,6 +37,9 @@ export interface BoundaStub { ): PromiseLike; query(name: string, payload?: unknown, actor?: string): PromiseLike; lag(): PromiseLike; + status(): PromiseLike; + listSubscriberFailures(actor?: string): PromiseLike; + getSubscriberFailure(subscriber: string, actor?: string): PromiseLike; readEvents(args?: ReadEventsArgs): PromiseLike; listDeadLetters(args?: ListDeadLettersArgs, actor?: string): PromiseLike; retryDeadLetter(id: string, actor?: string): PromiseLike; @@ -109,6 +114,12 @@ export const connect: ConnectFunction = ( ), queries: byName>((name, payload) => unwrap(stub.query(name, payload, actor))), getLag: () => unwrap(stub.lag()), + status: () => unwrap(stub.status()), + subscriberFailures: { + list: () => unwrap(stub.listSubscriberFailures(actor)), + get: (subscriber) => + unwrap(stub.getSubscriberFailure(subscriber, actor)), + }, events: { read: (args) => unwrap(stub.readEvents(args)) }, deadLetters: { list: (args) => unwrap(stub.listDeadLetters(args, actor)), diff --git a/packages/cloudflare/test/client.test.ts b/packages/cloudflare/test/client.test.ts index 53cf151b..221491d6 100644 --- a/packages/cloudflare/test/client.test.ts +++ b/packages/cloudflare/test/client.test.ts @@ -71,6 +71,8 @@ describe("connect", () => { await user.deadLetters.list({ status: "failed" }); await user.deadLetters.retry("l-1"); await user.as("operator-1").deadLetters.discard("l-1"); + await user.subscriberFailures.list(); + await store.subscriberFailures.get("projection:orders"); await store.queries.getOrder({ orderId: "o-1" }); expect(recorded).toMatchObject({ commands: [["payOrder", { orderId: "o-1" }, undefined, "eventual", "user-1"]], @@ -89,6 +91,10 @@ describe("connect", () => { ["retry", "l-1", "user-1"], ["discard", "l-1", "operator-1"], ], + failures: [ + ["list", "user-1"], + ["get", "projection:orders", undefined], + ], }); }); diff --git a/packages/cloudflare/test/recording-stub.ts b/packages/cloudflare/test/recording-stub.ts index 94912a4d..804c416f 100644 --- a/packages/cloudflare/test/recording-stub.ts +++ b/packages/cloudflare/test/recording-stub.ts @@ -2,7 +2,7 @@ import type { BoundaStub } from "../src/client.ts"; /** * A stand-in for a Bounda object's stub that records the arguments of every command, query and - * operation on a person or a dead letter, and answers each call with an empty success. + * operation on a person, a dead letter or a subscriber's failure, and answers each call with an empty success. */ export const recordingStub = (): { readonly stub: BoundaStub; @@ -11,12 +11,14 @@ export const recordingStub = (): { readonly forgets: unknown[][]; readonly exports: unknown[][]; readonly deadLetters: unknown[][]; + readonly failures: unknown[][]; } => { const commands: unknown[][] = []; const queries: unknown[][] = []; const forgets: unknown[][] = []; const exports: unknown[][] = []; const deadLetters: unknown[][] = []; + const failures: unknown[][] = []; const answer = async () => ({ ok: true, value: null }); return { commands, @@ -24,6 +26,7 @@ export const recordingStub = (): { forgets, exports, deadLetters, + failures, stub: { command: async (...args) => { commands.push(args); @@ -38,6 +41,15 @@ export const recordingStub = (): { return answer(); }, lag: answer, + status: answer, + listSubscriberFailures: async (...args) => { + failures.push(["list", ...args]); + return answer(); + }, + getSubscriberFailure: async (...args) => { + failures.push(["get", ...args]); + return answer(); + }, readEvents: answer, listDeadLetters: async (...args) => { deadLetters.push(["list", ...args]); diff --git a/packages/cloudflare/test/storage-contracts.test.ts b/packages/cloudflare/test/storage-contracts.test.ts index 7575534c..dc98e86f 100644 --- a/packages/cloudflare/test/storage-contracts.test.ts +++ b/packages/cloudflare/test/storage-contracts.test.ts @@ -20,6 +20,7 @@ import { snapshotStoreContract, storageSchemaContract, storageTransactionContract, + subscriberFailureStoreContract, tableContract, viewContract, } from "@bounda-dev/core/adapter/testing"; @@ -64,6 +65,7 @@ describe("the Durable Object adapter", () => { personalKeyStoreContract({ create: async () => (await freshStorage()).personalKeys }); personStoreContract({ create: async () => (await freshStorage()).persons }); inspectionContract({ create: freshAdapter }); + subscriberFailureStoreContract({ create: async () => (await freshStorage()).subscriberFailures }); forgetLedgerContract({ create: async () => { const adapter = await freshAdapter(); diff --git a/packages/core/src/adapter/adapter.ts b/packages/core/src/adapter/adapter.ts index 00059c02..bd92479e 100644 --- a/packages/core/src/adapter/adapter.ts +++ b/packages/core/src/adapter/adapter.ts @@ -13,6 +13,7 @@ import type { PersonStore } from "./storage/person-store.ts"; import type { PersonalKeyStore } from "./storage/personal-key-store.ts"; import type { Scheduler } from "./storage/scheduler.ts"; import type { SnapshotStore } from "./storage/snapshot-store.ts"; +import type { SubscriberFailureStore } from "./storage/subscriber-failure-store.ts"; import type { ReadClient, Table } from "./storage/table.ts"; /** @@ -57,6 +58,10 @@ export interface Storage { * command of an aggregate with snapshots. */ readonly snapshotStore: SnapshotStore; + /** + * The last failure of each subscriber, written by the dispatcher outside any transaction. + */ + readonly subscriberFailures: SubscriberFailureStore; /** * Only for a backend that can push new events; without it the dispatcher only polls. */ @@ -211,6 +216,38 @@ export interface Adapter * searches. It reads the whole database, so it is for tests. */ inspect(): Promise; + /** + * The version of each scope of Bounda's tables this adapter knows, as its database holds it, + * `0` for one never migrated there, beside the latest this version of Bounda migrates it to. + * Reads only: nothing is created or migrated. `[]` for an adapter without tables. + */ + schemaVersions(): Promise; + /** + * Brings the tables of `scopes` in this adapter's database to their latest version, as opening + * them does, and resolves to the versions after, as `schemaVersions` gives them: what + * `bounda migrate` runs before a deploy. Throws `StorageSchemaTooNewError` for a scope a newer + * version migrated, writing nothing. + */ + migrate(scopes: readonly SchemaScopeName[]): Promise; +} + +/** + * A set of Bounda's tables versioned together: `storage`, the write side; `checkpoints`, the + * positions of the subscribers, which read models in a database of their own keep there too; and + * `forget-ledger`, a forget ledger's table, wherever it lives. + */ +export type SchemaScopeName = "storage" | "checkpoints" | "forget-ledger"; + +export interface SchemaVersion { + readonly scope: string; + /** + * How many of the scope's migrations the database applied. + */ + readonly version: number; + /** + * How many this version of Bounda has. + */ + readonly latest: number; } export interface IsAdapterFunction { diff --git a/packages/core/src/adapter/index.ts b/packages/core/src/adapter/index.ts index 10d68117..199f2540 100644 --- a/packages/core/src/adapter/index.ts +++ b/packages/core/src/adapter/index.ts @@ -8,6 +8,8 @@ export type { ReadModelTransactArgs, ReadModelTransacted, ReadModelTransaction, + SchemaScopeName, + SchemaVersion, Storage, StorageTransaction, } from "./adapter.ts"; @@ -104,6 +106,11 @@ export type { Scheduler, } from "./storage/scheduler.ts"; export type { Snapshot, SnapshotStore } from "./storage/snapshot-store.ts"; +export type { + NewSubscriberFailure, + StoredSubscriberFailure, + SubscriberFailureStore, +} from "./storage/subscriber-failure-store.ts"; export type { FieldFilter, FindManyArgs, diff --git a/packages/core/src/adapter/sql/index.ts b/packages/core/src/adapter/sql/index.ts index 222d51c0..3c1ac128 100644 --- a/packages/core/src/adapter/sql/index.ts +++ b/packages/core/src/adapter/sql/index.ts @@ -38,10 +38,17 @@ export { inspectSqlTable, quoteName, storeOfTable } from "./inspection.ts"; export type { MigrateSchemasArgs, MigrateSchemasFunction, + ReadSchemaVersionsArgs, + ReadSchemaVersionsFunction, SchemaMigration, SchemaScope, } from "./migrations.ts"; -export { migrateSchemas, SCHEMA_SCOPES, schemaMigrationsTableFor } from "./migrations.ts"; +export { + migrateSchemas, + readSchemaVersions, + SCHEMA_SCOPES, + schemaMigrationsTableFor, +} from "./migrations.ts"; export type { CreateSqlPersonStoreArgs, CreateSqlPersonStoreFunction } from "./person-store.ts"; export { createSqlPersonStore } from "./person-store.ts"; export type { @@ -62,3 +69,4 @@ export { export { byExecuteAt, earliestDue } from "./scheduling.ts"; export type { SqlExecutor } from "./sql-table.ts"; export { createSqlReadClient, createSqlTable } from "./sql-table.ts"; +export { createSqlSubscriberFailureStore } from "./subscriber-failure-store.ts"; diff --git a/packages/core/src/adapter/sql/migrations.ts b/packages/core/src/adapter/sql/migrations.ts index a940cb53..8b2246fd 100644 --- a/packages/core/src/adapter/sql/migrations.ts +++ b/packages/core/src/adapter/sql/migrations.ts @@ -1,4 +1,5 @@ import { StorageSchemaTooNewError } from "../../contracts/errors.ts"; +import type { SchemaVersion } from "../adapter.ts"; import type { SqlTransaction } from "./database.ts"; import type { SqlDialect } from "./dialect.ts"; import { quoteIdentifier, storageTableNameFor } from "./identifiers.ts"; @@ -90,6 +91,52 @@ export interface MigrateSchemasFunction { * together apply each migration once, and one that fails leaves the ones before it applied. * Throws `StorageSchemaTooNewError`, before writing anything, when a scope is past what it knows. */ +const versionsIn = async ( + executor: SqlExecutor, + table: string, + isMissingTable: (error: unknown) => boolean, +): Promise> => { + try { + const rows = await executor.all( + `SELECT "scope", max("version") AS "version" FROM ${table} GROUP BY "scope"`, + [], + ); + return new Map(rows.map((row) => [String(row.scope), Number(row.version)])); + } catch (error) { + if (isMissingTable(error)) return new Map(); + throw error; + } +}; + +export interface ReadSchemaVersionsArgs { + readonly db: SqlExecutor; + // Quoted. + readonly table: string; + readonly scopes: readonly SchemaScope[]; + readonly isMissingTable: (error: unknown) => boolean; +} + +export interface ReadSchemaVersionsFunction { + (args: ReadSchemaVersionsArgs): Promise; +} + +/** + * The version each scope is at, `0` for one never migrated, beside its latest: one read, no lock. + */ +export const readSchemaVersions: ReadSchemaVersionsFunction = async ({ + db, + table, + scopes, + isMissingTable, +}) => { + const found = await versionsIn(db, table, isMissingTable); + return scopes.map(({ scope, migrations }) => ({ + scope, + version: found.get(scope) ?? 0, + latest: migrations.length, + })); +}; + export const migrateSchemas: MigrateSchemasFunction = async ({ db, dialect, @@ -100,18 +147,6 @@ export const migrateSchemas: MigrateSchemasFunction = async ({ now = () => new Date(), }) => { const p = dialect.placeholder; - const versionsIn = async (executor: SqlExecutor): Promise> => { - try { - const rows = await executor.all( - `SELECT "scope", max("version") AS "version" FROM ${table} GROUP BY "scope"`, - [], - ); - return new Map(rows.map((row) => [String(row.scope), Number(row.version)])); - } catch (error) { - if (isMissingTable(error)) return new Map(); - throw error; - } - }; const versionOf = (versions: ReadonlyMap, schema: SchemaScope): number => { const version = versions.get(schema.scope) ?? 0; const supported = schema.migrations.length; @@ -120,13 +155,14 @@ export const migrateSchemas: MigrateSchemasFunction = async ({ } return version; }; - const found = await versionsIn(db); + const found = await versionsIn(db, table, isMissingTable); const behind = scopes.map((schema) => ({ schema, at: versionOf(found, schema) })); for (const { schema, at } of behind) { for (const migration of schema.migrations.slice(at)) { await locked(async (tx) => { await tx.run(schemaMigrationsTableStatement(table), []); - if (migration.version <= versionOf(await versionsIn(tx), schema)) return; + if (migration.version <= versionOf(await versionsIn(tx, table, isMissingTable), schema)) + return; await migration.migrate(tx); await tx.run( `INSERT INTO ${table} ("scope", "version", "applied_at") VALUES (${p(1)}, ${p(2)}, ${p(3)})`, diff --git a/packages/core/src/adapter/sql/subscriber-failure-store.ts b/packages/core/src/adapter/sql/subscriber-failure-store.ts new file mode 100644 index 00000000..c8738142 --- /dev/null +++ b/packages/core/src/adapter/sql/subscriber-failure-store.ts @@ -0,0 +1,88 @@ +import type { + StoredSubscriberFailure, + SubscriberFailureStore, +} from "../storage/subscriber-failure-store.ts"; +import { decodeErrorText, encodeErrorText } from "./dead-letters.ts"; +import type { SqlDialect } from "./dialect.ts"; +import { codePointOrder } from "./order.ts"; +import type { SqlExecutor } from "./sql-table.ts"; + +export interface CreateSqlSubscriberFailureStoreArgs { + readonly db: SqlExecutor; + /** + * Already quoted. + */ + readonly table: string; + readonly dialect: Pick; +} + +export interface CreateSqlSubscriberFailureStoreFunction { + (args: CreateSqlSubscriberFailureStoreArgs): SubscriberFailureStore; +} + +const COLUMNS = `"subscriber", "position", "event_id", "event_type", "person", "error_name", "error_code", "error_message", "error_stack", "attempts", "first_failed_at", "last_failed_at", "resolved_at"`; + +const failureOf = (row: Record): StoredSubscriberFailure => ({ + subscriber: String(row.subscriber), + position: Number(row.position), + eventId: String(row.event_id), + eventType: String(row.event_type), + person: row.person === null ? null : String(row.person), + errorName: String(row.error_name), + ...(row.error_code === null ? {} : { errorCode: String(row.error_code) }), + errorMessage: decodeErrorText(row.error_message), + ...(row.error_stack === null ? {} : { errorStack: decodeErrorText(row.error_stack) }), + attempts: Number(row.attempts), + firstFailedAt: String(row.first_failed_at), + lastFailedAt: String(row.last_failed_at), + resolvedAt: row.resolved_at === null ? null : String(row.resolved_at), +}); + +/** + * The subscriber failure store of SQLite and PostgreSQL alike, on the SQL both speak. A record is + * one statement, so two instances failing on the same event at once count both attempts. The + * error's text is kept as JSON, as a dead letter's is, so text in the clear and a sealed envelope + * never read alike. + */ +export const createSqlSubscriberFailureStore: CreateSqlSubscriberFailureStoreFunction = ({ + db, + table, + dialect, +}) => { + const p = dialect.placeholder; + const again = `${table}."resolved_at" IS NULL AND ${table}."event_id" = EXCLUDED."event_id"`; + return { + record: (failure) => + db.run( + `INSERT INTO ${table} (${COLUMNS}) VALUES (${p(1)}, ${p(2)}, ${p(3)}, ${p(4)}, ${p(5)}, ${p(6)}, ${p(7)}, ${p(8)}, ${p(9)}, 1, ${p(10)}, ${p(11)}, NULL) ON CONFLICT ("subscriber") DO UPDATE SET "position" = EXCLUDED."position", "event_id" = EXCLUDED."event_id", "event_type" = EXCLUDED."event_type", "person" = EXCLUDED."person", "error_name" = EXCLUDED."error_name", "error_code" = EXCLUDED."error_code", "error_message" = EXCLUDED."error_message", "error_stack" = EXCLUDED."error_stack", "attempts" = CASE WHEN ${again} THEN ${table}."attempts" + 1 ELSE 1 END, "first_failed_at" = CASE WHEN ${again} THEN ${table}."first_failed_at" ELSE EXCLUDED."first_failed_at" END, "last_failed_at" = EXCLUDED."last_failed_at", "resolved_at" = NULL`, + [ + failure.subscriber, + failure.position, + failure.eventId, + failure.eventType, + failure.person, + failure.errorName, + failure.errorCode ?? null, + encodeErrorText(failure.errorMessage), + failure.errorStack === undefined ? null : encodeErrorText(failure.errorStack), + failure.failedAt, + failure.failedAt, + ], + ), + resolve: (subscriber, at) => + db.run( + `UPDATE ${table} SET "resolved_at" = ${p(1)} WHERE "subscriber" = ${p(2)} AND "resolved_at" IS NULL`, + [at.toISOString(), subscriber], + ), + get: async (subscriber) => { + const [row] = await db.all(`SELECT ${COLUMNS} FROM ${table} WHERE "subscriber" = ${p(1)}`, [ + subscriber, + ]); + return row === undefined ? null : failureOf(row); + }, + list: async () => + (await db.all(`SELECT ${COLUMNS} FROM ${table}`, [])) + .map(failureOf) + .sort((a, b) => codePointOrder(a.subscriber, b.subscriber)), + }; +}; diff --git a/packages/core/src/adapter/sqlite/adapter.ts b/packages/core/src/adapter/sqlite/adapter.ts index e732b68d..8f6014b0 100644 --- a/packages/core/src/adapter/sqlite/adapter.ts +++ b/packages/core/src/adapter/sqlite/adapter.ts @@ -2,6 +2,7 @@ import type { Adapter, CreateReadModelArgs, CreateReadModelRebuildArgs, + SchemaScopeName, StorageTransaction, } from "../adapter.ts"; import { createSqlCheckpointStore } from "../sql/checkpoint-store.ts"; @@ -10,13 +11,20 @@ import { sqliteDialect } from "../sql/dialect.ts"; import { createSqlForgetLedger } from "../sql/forget-ledger.ts"; import { quoteIdentifier, storageTableNameFor } from "../sql/identifiers.ts"; import { inspectSqlTable, quoteName } from "../sql/inspection.ts"; +import { + readSchemaVersions, + type SchemaScope, + schemaMigrationsTableFor, +} from "../sql/migrations.ts"; import { createSqlPersonStore } from "../sql/person-store.ts"; import { createSqlPersonalKeyStore } from "../sql/personal-key-store.ts"; +import { createSqlSubscriberFailureStore } from "../sql/subscriber-failure-store.ts"; import { createSqliteDeadLetterStore } from "./dead-letter-store.ts"; import { createSqliteEventStore } from "./event-store.ts"; import { createSqliteIdempotencyStore } from "./idempotency-store.ts"; import { createSqliteInboxLedger } from "./inbox-ledger.ts"; import { + isMissingSqliteTable, migrateSqlite, sqliteCheckpointsScope, sqliteForgetLedgerScope, @@ -93,10 +101,47 @@ export const createSqliteAdapter: CreateSqliteAdapterFunction = < throw error; } }; + const ledgerTable = quoteIdentifier( + storageTableNameFor({ prefix: tablePrefix, table: "forgetLedger" }), + ); + const scopes: Readonly> = { + storage: sqliteStorageScope(storageTablesFor(tablePrefix)), + checkpoints: sqliteCheckpointsScope(storageTablesFor(tablePrefix).checkpoints), + "forget-ledger": sqliteForgetLedgerScope(ledgerTable), + }; + const versionsOn = (db: SqlDatabase) => + readSchemaVersions({ + db, + table: schemaMigrationsTableFor(tablePrefix), + scopes: Object.values(scopes), + isMissingTable: isMissingSqliteTable, + }); + // A connection for one call, handed back whatever the call does. + const briefly = async ( + work: (connection: SqliteConnection) => Promise, + ): Promise => { + const connection = acquire(); + try { + return await work(connection); + } finally { + await release(); + } + }; return { kind: "bounda-adapter", name, options, + schemaVersions: () => briefly(({ db }) => versionsOn(db)), + migrate: (names) => + briefly(async ({ db, isBusy }) => { + await migrateSqlite({ + db, + isBusy, + prefix: tablePrefix, + scopes: names.map((scope) => scopes[scope]), + }); + return versionsOn(db); + }), createStorage: () => using(async ({ db, isBusy }) => { const tables = storageTablesFor(tablePrefix); @@ -161,6 +206,11 @@ export const createSqliteAdapter: CreateSqliteAdapterFunction = < table: tables.checkpoints, dialect: sqliteDialect, }), + subscriberFailures: createSqlSubscriberFailureStore({ + db, + table: tables.subscriberFailures, + dialect: sqliteDialect, + }), transact: (work) => db.write((tx) => work(storesOver(boundTo(tx)))), close: release, }; diff --git a/packages/core/src/adapter/sqlite/migrations.ts b/packages/core/src/adapter/sqlite/migrations.ts index 6e748c4f..5f82f938 100644 --- a/packages/core/src/adapter/sqlite/migrations.ts +++ b/packages/core/src/adapter/sqlite/migrations.ts @@ -16,6 +16,7 @@ import { personRowsBackfillStatements, type StorageTables, storageSchemaStatements, + subscriberFailuresTableStatement, } from "./schema.ts"; import { busyRetryDelay, pause } from "./wal.ts"; @@ -118,7 +119,13 @@ export interface SqliteStorageScopeFunction { export const sqliteStorageScope: SqliteStorageScopeFunction = (tables) => ({ scope: SCHEMA_SCOPES.storage, - migrations: [{ version: 1, migrate: (tx) => adoptStorageTables(tx, tables) }], + migrations: [ + { version: 1, migrate: (tx) => adoptStorageTables(tx, tables) }, + { + version: 2, + migrate: (tx) => tx.run(subscriberFailuresTableStatement(tables.subscriberFailures), []), + }, + ], }); export interface SqliteCheckpointsScopeFunction { @@ -148,6 +155,13 @@ export const sqliteForgetLedgerScope: SqliteForgetLedgerScopeFunction = (table) ], }); +export interface IsMissingSqliteTableFunction { + (error: unknown): boolean; +} + +export const isMissingSqliteTable: IsMissingSqliteTableFunction = (error) => + error instanceof Error && /no such table/i.test(error.message); + export interface MigrateSqliteArgs { readonly db: SqlDatabase; /** @@ -190,5 +204,5 @@ export const migrateSqlite: MigrateSqliteFunction = ({ } } }, - isMissingTable: (error) => error instanceof Error && /no such table/i.test(error.message), + isMissingTable: isMissingSqliteTable, }); diff --git a/packages/core/src/adapter/sqlite/schema.ts b/packages/core/src/adapter/sqlite/schema.ts index 3654a361..601abbbc 100644 --- a/packages/core/src/adapter/sqlite/schema.ts +++ b/packages/core/src/adapter/sqlite/schema.ts @@ -12,6 +12,7 @@ export interface StorageTables { readonly persons: string; readonly snapshots: string; readonly personRows: string; + readonly subscriberFailures: string; } export interface StorageTablesForFunction { @@ -29,6 +30,7 @@ export const storageTablesFor: StorageTablesForFunction = (prefix) => ({ persons: quoteIdentifier(storageTableNameFor({ prefix, table: "persons" })), snapshots: quoteIdentifier(storageTableNameFor({ prefix, table: "snapshots" })), personRows: quoteIdentifier(storageTableNameFor({ prefix, table: "personRows" })), + subscriberFailures: quoteIdentifier(storageTableNameFor({ prefix, table: "subscriberFailures" })), }); /** @@ -239,3 +241,25 @@ export const personRowsBackfillStatements: PersonRowsBackfillStatementsFunction ([table, indexed]) => `INSERT OR IGNORE INTO ${tables.personRows} ("person", "store", "row") SELECT json_extract("named"."value", '$.person'), '${indexed.store}', ${rowOf('"stored"', indexed)} FROM ${table} AS "stored", json_each("stored"."persons") AS "named"`, ); + +export interface SubscriberFailuresTableStatementFunction { + (table: string): string; +} + +// Migration 2 of the storage tables. One row per subscriber, without an index: there are few. +export const subscriberFailuresTableStatement: SubscriberFailuresTableStatementFunction = (table) => + `CREATE TABLE IF NOT EXISTS ${table} ( + "subscriber" TEXT PRIMARY KEY, + "position" INTEGER NOT NULL, + "event_id" TEXT NOT NULL, + "event_type" TEXT NOT NULL, + "person" TEXT, + "error_name" TEXT NOT NULL, + "error_code" TEXT, + "error_message" TEXT NOT NULL, + "error_stack" TEXT, + "attempts" INTEGER NOT NULL, + "first_failed_at" TEXT NOT NULL, + "last_failed_at" TEXT NOT NULL, + "resolved_at" TEXT + )`; diff --git a/packages/core/src/adapter/sqlite/sqlite.test.ts b/packages/core/src/adapter/sqlite/sqlite.test.ts index a1b9dc0d..ad1402b8 100644 --- a/packages/core/src/adapter/sqlite/sqlite.test.ts +++ b/packages/core/src/adapter/sqlite/sqlite.test.ts @@ -34,6 +34,7 @@ import { snapshotStoreContract, storageSchemaContract, storageTransactionContract, + subscriberFailureStoreContract, tableContract, viewContract, } from "../testing/index.ts"; @@ -133,6 +134,7 @@ describe("the SQLite stores on node:sqlite", () => { personalKeyStoreContract({ create: async () => (await storage()).personalKeys }); personStoreContract({ create: async () => (await storage()).persons }); inspectionContract({ create: async () => nodeSqlite().adapter }); + subscriberFailureStoreContract({ create: async () => (await storage()).subscriberFailures }); forgetLedgerContract({ create: async () => { const { adapter } = nodeSqlite(); @@ -528,6 +530,7 @@ describe("createSqliteAdapter", () => { "app_rm_order_summary", "app_scheduled_commands", "app_snapshots", + "app_subscriber_failures", ]); }); diff --git a/packages/core/src/adapter/storage/subscriber-failure-store.ts b/packages/core/src/adapter/storage/subscriber-failure-store.ts new file mode 100644 index 00000000..a305b761 --- /dev/null +++ b/packages/core/src/adapter/storage/subscriber-failure-store.ts @@ -0,0 +1,63 @@ +import type { SealedValue } from "./personal-key-store.ts"; + +/** + * The last failure of one subscriber, kept as its row: what the logs and spans leave out of an + * error, kept where the person it may quote can be forgotten. + */ +export interface StoredSubscriberFailure { + readonly subscriber: string; + /** + * The global position of the event the subscriber failed on. + */ + readonly position: number; + readonly eventId: string; + readonly eventType: string; + /** + * Whom the event names, under whose key the error's text is sealed; `null` for an event that + * names nobody, whose text is kept in the clear. + */ + readonly person: string | null; + readonly errorName: string; + readonly errorCode?: string; + readonly errorMessage: string | SealedValue; + readonly errorStack?: string | SealedValue; + /** + * How many deliveries in a row failed on this event, across instances. + */ + readonly attempts: number; + readonly firstFailedAt: string; + readonly lastFailedAt: string; + /** + * When an instance found the subscriber past it, `null` while none has. + */ + readonly resolvedAt: string | null; +} + +/** + * A failure as a delivery records it, at `failedAt`. + */ +export type NewSubscriberFailure = Omit< + StoredSubscriberFailure, + "attempts" | "firstFailedAt" | "lastFailedAt" | "resolvedAt" +> & { readonly failedAt: string }; + +/** + * One row per subscriber, which only its last failure replaces. + */ +export interface SubscriberFailureStore { + /** + * Records the subscriber's last failure: one on the event its unresolved row names counts one + * attempt more and keeps when they began, any other starts again from one attempt. + */ + record(failure: NewSubscriberFailure): Promise; + /** + * Marks the subscriber's last failure resolved at `at`, unless it is resolved already or none + * is kept. + */ + resolve(subscriber: string, at: Date): Promise; + get(subscriber: string): Promise; + /** + * Every subscriber's last failure, by subscriber. + */ + list(): Promise; +} diff --git a/packages/core/src/adapter/testing/index.ts b/packages/core/src/adapter/testing/index.ts index 46e77d42..0448a73b 100644 --- a/packages/core/src/adapter/testing/index.ts +++ b/packages/core/src/adapter/testing/index.ts @@ -78,6 +78,11 @@ export type { StorageTransactionContractFunction, } from "./storage-transaction.contract.ts"; export { storageTransactionContract } from "./storage-transaction.contract.ts"; +export type { + SubscriberFailureStoreContractArgs, + SubscriberFailureStoreContractFunction, +} from "./subscriber-failure-store.contract.ts"; +export { subscriberFailureStoreContract } from "./subscriber-failure-store.contract.ts"; export type { ContractRow, TableContractArgs, diff --git a/packages/core/src/adapter/testing/storage-schema.contract.ts b/packages/core/src/adapter/testing/storage-schema.contract.ts index 3b36b98f..c635cd82 100644 --- a/packages/core/src/adapter/testing/storage-schema.contract.ts +++ b/packages/core/src/adapter/testing/storage-schema.contract.ts @@ -30,10 +30,11 @@ export interface StorageSchemaContractFunction { (args: StorageSchemaContractArgs): void; } -// What a new database holds: every scope the storage migrates, at its latest version. +// What a new database holds: every migration of every scope the storage migrates, one row each. const LATEST = [ { scope: "checkpoints", version: 1 }, { scope: "storage", version: 1 }, + { scope: "storage", version: 2 }, ]; const versionsOf = async ({ execute, prefix }: StorageSchemaTarget) => @@ -169,7 +170,7 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ code: "STORAGE_SCHEMA_TOO_NEW", scope: "storage", version: 999, - supported: 1, + supported: 2, }); expect(await shapeOf(target, dialect)).toEqual(shape); }); diff --git a/packages/core/src/adapter/testing/subscriber-failure-store.contract.ts b/packages/core/src/adapter/testing/subscriber-failure-store.contract.ts new file mode 100644 index 00000000..c9fbc9fa --- /dev/null +++ b/packages/core/src/adapter/testing/subscriber-failure-store.contract.ts @@ -0,0 +1,134 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import type { + NewSubscriberFailure, + SubscriberFailureStore, +} from "../storage/subscriber-failure-store.ts"; + +export interface SubscriberFailureStoreContractArgs { + readonly create: () => Promise; +} + +export interface SubscriberFailureStoreContractFunction { + (args: SubscriberFailureStoreContractArgs): void; +} + +const at = (minute: number): string => new Date(Date.UTC(2026, 2, 1, 0, minute)).toISOString(); + +const failure = (overrides: Partial = {}): NewSubscriberFailure => ({ + subscriber: "projection:orders", + position: 7, + eventId: "e-7", + eventType: "OrderPlaced", + person: "c-1", + errorName: "ValidationError", + errorCode: "VALIDATION_FAILED", + errorMessage: { $personal: { person: "c-1", key: "k", iv: "i", data: "d" } }, + errorStack: "Error: x\n at f", + failedAt: at(0), + ...overrides, +}); + +/** + * The behaviour every subscriber failure store must exhibit. Call it inside a `describe` of the + * adapter's test file with a factory that returns a fresh, empty store. + */ +export const subscriberFailureStoreContract: SubscriberFailureStoreContractFunction = ({ + create, +}) => { + describe("subscriber failure store contract", () => { + let store: SubscriberFailureStore; + + beforeEach(async () => { + store = await create(); + }); + + it("keeps nothing for a subscriber that never failed, and resolves nothing for it", async () => { + expect(await store.get("projection:orders")).toBeNull(); + await store.resolve("projection:orders", new Date(at(1))); + expect(await store.list()).toEqual([]); + }); + + it("keeps a failure as it was recorded, sealed text, stack and code included", async () => { + await store.record(failure()); + const { failedAt: _failedAt, ...recorded } = failure(); + expect(await store.get("projection:orders")).toEqual({ + ...recorded, + attempts: 1, + firstFailedAt: at(0), + lastFailedAt: at(0), + resolvedAt: null, + }); + }); + + it("keeps text in the clear apart from a sealed envelope, and an error without code or stack", async () => { + const { + errorCode: _code, + errorStack: _stack, + ...bare + } = failure({ + person: null, + errorMessage: '{"$personal":"not sealed"}', + }); + await store.record(bare); + expect(await store.get("projection:orders")).toEqual( + expect.objectContaining({ person: null, errorMessage: '{"$personal":"not sealed"}' }), + ); + const kept = await store.get("projection:orders"); + expect(kept).not.toHaveProperty("errorCode"); + expect(kept).not.toHaveProperty("errorStack"); + }); + + it("counts the failures in a row on one event, and starts again on another", async () => { + await store.record(failure()); + await store.record(failure({ failedAt: at(1) })); + await store.record(failure({ failedAt: at(2), errorName: "TypeError" })); + expect(await store.get("projection:orders")).toMatchObject({ + attempts: 3, + firstFailedAt: at(0), + lastFailedAt: at(2), + errorName: "TypeError", + }); + await store.record(failure({ position: 9, eventId: "e-9", failedAt: at(3) })); + expect(await store.get("projection:orders")).toMatchObject({ + position: 9, + eventId: "e-9", + attempts: 1, + firstFailedAt: at(3), + lastFailedAt: at(3), + }); + }); + + it("resolves the last failure once, and counts afresh a failure on the same event after it", async () => { + await store.record(failure()); + await store.resolve("projection:orders", new Date(at(5))); + await store.resolve("projection:orders", new Date(at(6))); + expect(await store.get("projection:orders")).toMatchObject({ resolvedAt: at(5) }); + await store.record(failure({ failedAt: at(7) })); + expect(await store.get("projection:orders")).toMatchObject({ + attempts: 1, + firstFailedAt: at(7), + resolvedAt: null, + }); + }); + + it("keeps one row per subscriber, listed by subscriber", async () => { + await store.record(failure({ subscriber: "policies" })); + await store.record(failure({ subscriber: "Processes" })); + await store.record(failure()); + expect((await store.list()).map(({ subscriber }) => subscriber)).toEqual([ + "Processes", + "policies", + "projection:orders", + ]); + }); + + it("counts both of two instances failing on the same event at once", async () => { + await store.record(failure()); + await Promise.all([ + store.record(failure({ failedAt: at(1) })), + store.record(failure({ failedAt: at(1) })), + ]); + expect(await store.get("projection:orders")).toMatchObject({ attempts: 3 }); + }); + }); +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5e0a6bc8..6686d195 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -35,12 +35,17 @@ export type { } from "./kernel/app.ts"; export { createApp } from "./kernel/app.ts"; export type { DeadLetters } from "./kernel/dead-letters/dead-letters.ts"; +export type { DeadLetterRetentionStatus } from "./kernel/dead-letters/retention.ts"; export type { DispatcherLag, SubscriberFailing, SubscriberKind, SubscriberLag, } from "./kernel/dispatch/dispatcher.ts"; +export type { + SubscriberFailure, + SubscriberFailures, +} from "./kernel/dispatch/subscriber-failures.ts"; export type { GlobalStream, ReadEventsArgs } from "./kernel/events/global-stream.ts"; export type { ExportedData, @@ -48,9 +53,15 @@ export type { ExportedEvent, ExportedRow, ExportedScheduledCommand, + ExportedSubscriberFailure, PersonalDataExport, } from "./kernel/personal-data/export.ts"; -export type { ForgetAgainEntry } from "./kernel/personal-data/reconciliation.ts"; +export type { + ForgetAgainEntry, + ForgetLedgerReconciliationStatus, +} from "./kernel/personal-data/reconciliation.ts"; +export type { RetentionStatus } from "./kernel/personal-data/retention.ts"; +export type { KeyRotationStatus } from "./kernel/personal-data/rotation.ts"; export type { ProcessStatus } from "./kernel/process/lifecycle.ts"; export type { RebuildReadModelArgs, @@ -72,5 +83,6 @@ export type { export { errorFields } from "./kernel/shared/error-fields.ts"; export type { IdempotencyKeyForFunction } from "./kernel/shared/idempotency-key.ts"; export { idempotencyKeyFor } from "./kernel/shared/idempotency-key.ts"; +export type { AppStatus, SchemaStatus, SubscriberFailureStatus } from "./kernel/status.ts"; export * from "./modules/index.ts"; export type { AppEnv, AppRegistry, EnvSection, Register } from "./register/index.ts"; diff --git a/packages/core/src/kernel/access-log.test.ts b/packages/core/src/kernel/access-log.test.ts index 83091731..484eb007 100644 --- a/packages/core/src/kernel/access-log.test.ts +++ b/packages/core/src/kernel/access-log.test.ts @@ -210,7 +210,13 @@ describe("the access log", () => { personCount: 1, at: AT, outcome: "ok", - detail: { events: 1, scheduledCommands: 0, deadLetters: 1, readModelRows: 1 }, + detail: { + events: 1, + scheduledCommands: 0, + deadLetters: 1, + subscriberFailures: 0, + readModelRows: 1, + }, }, ]); expect(records[0]).toMatchObject({ diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 6805a8e7..925ee5a8 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -22,7 +22,12 @@ import { createRestrictedRetries } from "./dead-letters/restricted.ts"; import { createDeadLetterRetention } from "./dead-letters/retention.ts"; import { createDeadLetterRetry } from "./dead-letters/retry.ts"; import { createDispatcher, type DispatcherLag } from "./dispatch/dispatcher.ts"; +import { createLastFailures } from "./dispatch/last-failures.ts"; import { alignReactiveCheckpoints } from "./dispatch/reactive-checkpoints.ts"; +import { + createSubscriberFailures, + type SubscriberFailures, +} from "./dispatch/subscriber-failures.ts"; import { appStorage } from "./events/app-event-store.ts"; import { createGlobalStream, type GlobalStream } from "./events/global-stream.ts"; import { createAccessLog, recorded } from "./personal-data/access.ts"; @@ -70,6 +75,7 @@ import { guardedLogger } from "./shared/guarded-logger.ts"; import { ignoredRetries, type PendingRetries } from "./shared/pending-retries.ts"; import { createRestoreWatch } from "./shared/restores.ts"; import { createSnapshotSubscriber, takesSnapshots } from "./snapshot/subscriber.ts"; +import { type AppStatus, readAppStatus } from "./status.ts"; import { ATTRIBUTES, METRICS, meter, tracedOperation } from "./telemetry.ts"; /** @@ -95,6 +101,18 @@ export interface BoundaClient { */ readonly queries: QueriesFacade; getLag(): Promise; + /** + * Where the runtime's background work stands, as the database says: the versions of its tables, + * every subscriber's lag and last failure without its text, the backfill of personal data, the + * forget ledger's reconciliation, the key rotation, the retention and the purge of dead letters. + * What `bounda status` prints. Records nothing in the access log: none of it is personal data. + */ + status(): Promise; + /** + * Each subscriber's last failure with its text opened; each call that finds one is recorded in + * the access log, as `exportPersonalData` is. + */ + readonly subscriberFailures: SubscriberFailures; readonly events: GlobalStream; /** * Retrying and discarding a letter is acted by the client's actor, as `as` says. Each call but @@ -678,6 +696,12 @@ export const assembleApp: AssembleAppFunction = async ({ catchUp: config.runtime.dispatcher.catchUp, idleIntervalMs: config.runtime.dispatcher.idleIntervalMs, ...(storage.notifier === undefined ? {} : { notifier: storage.notifier }), + lastFailures: createLastFailures({ + store: written.subscriberFailures, + aggregates, + keys: storage.personalKeys, + logger, + }), clock, logger, errorText, @@ -764,6 +788,18 @@ export const assembleApp: AssembleAppFunction = async ({ reconciliation === undefined ? null : { reconciled: reconciliation.reconciled() }, personalDataBackfill: backfill === undefined ? null : await backfill.status(), }); + const status = (): Promise => + readAppStatus({ + config, + lag: getLag, + failures: written.subscriberFailures, + restores, + backfill, + reconciliation, + rotation, + retention, + letterRetention, + }); // Before the app serves anything: a backup restored brings back persons forgotten since, and // the read models that show them hear of it by the time the start resolves. A ledger that // does not answer in time is left to finish behind the start, as one that fails is retried, @@ -805,6 +841,7 @@ export const assembleApp: AssembleAppFunction = async ({ events: exported.events.length, scheduledCommands: exported.scheduledCommands.length, deadLetters: exported.deadLetters.length, + subscriberFailures: exported.subscriberFailures.length, readModelRows: Object.values(exported.readModels).reduce( (sum, rows) => sum + rows.length, 0, @@ -835,6 +872,14 @@ export const assembleApp: AssembleAppFunction = async ({ }) as CommandsFacade, queries: queryRunner.facade(actor) as QueriesFacade, getLag, + status: () => tracedOperation({ errorText, operation: "status", actor, run: status }), + subscriberFailures: createSubscriberFailures({ + store: written.subscriberFailures, + keys: storage.personalKeys, + actor, + recordAccess, + errorText, + }), events, deadLetters: createDeadLetters({ storage, diff --git a/packages/core/src/kernel/dispatch/delivery.test.ts b/packages/core/src/kernel/dispatch/delivery.test.ts index 2914c261..c17a06e5 100644 --- a/packages/core/src/kernel/dispatch/delivery.test.ts +++ b/packages/core/src/kernel/dispatch/delivery.test.ts @@ -212,6 +212,8 @@ describe("createCheckpointedSubscriber", () => { expect(await subscriber.deliver({ read, wait: true })).toEqual({ outcome: "failed", failure: { position: 3, eventId: "order-1-3", eventType: "OrderPlaced", errorName: "Error" }, + event: expect.objectContaining({ id: "order-1-3", position: 3 }), + error: expect.objectContaining({ message: "poison" }), }); expect(seen).toEqual([ [1, 2, 3, 4], @@ -348,6 +350,8 @@ describe("createCheckpointedSubscriber", () => { eventType: "OrderPlaced", errorName: "string", }, + event: expect.objectContaining({ id: "order-1-1", position: 1 }), + error: "not an error", }); expect(transactions.waits).toEqual([true]); }); diff --git a/packages/core/src/kernel/dispatch/delivery.ts b/packages/core/src/kernel/dispatch/delivery.ts index 03dded2c..2926aea4 100644 --- a/packages/core/src/kernel/dispatch/delivery.ts +++ b/packages/core/src/kernel/dispatch/delivery.ts @@ -53,9 +53,18 @@ const withoutStack = ({ ...fields }: ErrorFields): Omit => fields; +/** + * A failed delivery also carries the event it failed on and what it threw, for the record the + * dispatcher keeps of it. + */ export type Delivery = | { readonly outcome: Exclude } - | { readonly outcome: "failed"; readonly failure: DeliveryFailure }; + | { + readonly outcome: "failed"; + readonly failure: DeliveryFailure; + readonly event: StoredEvent; + readonly error: unknown; + }; /** * Thrown by `process` when the first `done` events went through and the next one threw `cause`. @@ -223,6 +232,8 @@ export const createCheckpointedSubscriber: CreateCheckpointedSubscriberFunction eventType: failed.type, ...withoutStack(errorFields({ error: cause, errorText })), }, + event: failed, + error: cause, }; } }; diff --git a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts index bf91b425..07ea8c3a 100644 --- a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts +++ b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts @@ -1,19 +1,38 @@ import { describe, expect, it } from "vitest"; -import { pendingEvent } from "../../adapter/testing/fixtures.ts"; +import { pendingEvent, testMetadata } from "../../adapter/testing/fixtures.ts"; import { createFixedClock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; import { silentLogger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; import { advanceUntilWaiting, eventually } from "../test-support.ts"; import type { CheckpointedSubscriber, Delivery, DeliveryFailure } from "./delivery.ts"; import { createDispatcher, type Subscriber } from "./dispatcher.ts"; +const failedEvent: StoredEvent = { + id: "e-1", + aggregateType: "order", + aggregateId: "o-1", + version: 1, + position: 1, + type: "OrderPlaced", + payload: {}, + timestamp: "2026-01-01T00:00:00.000Z", + metadata: testMetadata, + person: null, +}; + const failure: DeliveryFailure = { position: 1, eventId: "order-1-1", eventType: "OrderPlaced", errorName: "Error", }; -const failed: Delivery = { outcome: "failed", failure }; +const failed: Delivery = { + outcome: "failed", + failure, + event: failedEvent, + error: new Error("boom"), +}; interface Scripted extends CheckpointedSubscriber { readonly deliveries: number; diff --git a/packages/core/src/kernel/dispatch/dispatcher-catch-up.test.ts b/packages/core/src/kernel/dispatch/dispatcher-catch-up.test.ts index 061ebff3..7109955a 100644 --- a/packages/core/src/kernel/dispatch/dispatcher-catch-up.test.ts +++ b/packages/core/src/kernel/dispatch/dispatcher-catch-up.test.ts @@ -1,10 +1,25 @@ import { describe, expect, it } from "vitest"; +import { testMetadata } from "../../adapter/testing/fixtures.ts"; import { createFixedClock, type FixedClock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; import { memory } from "../../memory/index.ts"; import { createRecordingLogger, eventually } from "../test-support.ts"; import type { CheckpointedSubscriber, Delivery } from "./delivery.ts"; import { createDispatcher, type DispatcherCatchUp, type Subscriber } from "./dispatcher.ts"; +const failedEvent: StoredEvent = { + id: "e-1", + aggregateType: "order", + aggregateId: "o-1", + version: 1, + position: 1, + type: "OrderPlaced", + payload: {}, + timestamp: "2026-01-01T00:00:00.000Z", + metadata: testMetadata, + person: null, +}; + interface Fake extends CheckpointedSubscriber { at: number; readonly deliveries: boolean[]; @@ -42,6 +57,8 @@ const busy = async (): Promise => ({ outcome: "busy" }); const failed = async (): Promise => ({ outcome: "failed", failure: { position: 1, eventId: "e-1", eventType: "OrderPlaced", errorName: "Error" }, + event: failedEvent, + error: new Error("boom"), }); const setUp = async ( diff --git a/packages/core/src/kernel/dispatch/dispatcher.ts b/packages/core/src/kernel/dispatch/dispatcher.ts index ee3a1ab9..efc1c50f 100644 --- a/packages/core/src/kernel/dispatch/dispatcher.ts +++ b/packages/core/src/kernel/dispatch/dispatcher.ts @@ -21,6 +21,7 @@ import { type Subscriber, type SubscriberKind, } from "./delivery.ts"; +import type { LastFailures } from "./last-failures.ts"; const untilNothingMoves = async (pass: () => Promise): Promise => { for (;;) { @@ -133,6 +134,10 @@ export interface CreateDispatcherArgs { */ readonly idleIntervalMs?: number; readonly notifier?: EventNotifier; + /** + * Where each subscriber's last failure is kept, its text sealed; without it, nowhere. + */ + readonly lastFailures?: LastFailures; readonly clock: Clock; readonly logger: Logger; readonly errorText: ErrorText; @@ -171,6 +176,7 @@ export const createDispatcher: CreateDispatcherFunction = ({ catchUp = { timeoutMs: DEFAULT_CATCH_UP_TIMEOUT_MS, pollIntervalMs: DEFAULT_CATCH_UP_POLL_MS }, idleIntervalMs = pollIntervalMs, notifier, + lastFailures, clock, logger, errorText, @@ -194,7 +200,12 @@ export const createDispatcher: CreateDispatcherFunction = ({ const delayFor = (attempts: number): number => Math.min(backoff.maxDelayMs, backoff.baseDelayMs * 2 ** (attempts - 1)); - const record = (subscriber: string, delivery: Delivery): void => { + // The subscribers whose last failure kept in the store this process has resolved, or found + // past, since it started: a failure another instance or an earlier start met is resolved by the + // first delivery that moves the subscriber here. + const checked = new Set(); + + const record = async (subscriber: string, delivery: Delivery): Promise => { const current = failing.get(subscriber); if (delivery.outcome === "failed") { const attempts = (current?.attempts ?? 0) + 1; @@ -204,8 +215,19 @@ export const createDispatcher: CreateDispatcherFunction = ({ since: current?.since ?? now(), retryAt: now() + delayFor(attempts), }); + checked.delete(subscriber); + await lastFailures?.record({ + subscriber, + event: delivery.event, + error: delivery.error, + at: clock.now(), + }); return; } + if (delivery.outcome === "advanced" && !checked.has(subscriber)) { + checked.add(subscriber); + await lastFailures?.resolve(subscriber, clock.now()); + } if (current === undefined || delivery.outcome === "busy" || delivery.outcome === "held") return; failing.delete(subscriber); if (delivery.outcome !== "advanced") return; @@ -238,7 +260,7 @@ export const createDispatcher: CreateDispatcherFunction = ({ if ((await subscriber.position()) >= position) return true; if (now() >= deadline || backingOff(subscriber.name)) return false; const delivery = await subscriber.deliver({ read, wait: false }); - record(subscriber.name, delivery); + await record(subscriber.name, delivery); if (delivery.outcome === "failed") return false; if (!moving.has(delivery.outcome)) await sleep(catchUp.pollIntervalMs); } @@ -257,7 +279,7 @@ export const createDispatcher: CreateDispatcherFunction = ({ if (only !== undefined && subscriber.kind !== only) continue; if (patient && backingOff(subscriber.name)) continue; const delivery = await subscriber.deliver({ read, wait }); - record(subscriber.name, delivery); + await record(subscriber.name, delivery); advanced = moving.has(delivery.outcome) || advanced; } return advanced; diff --git a/packages/core/src/kernel/dispatch/last-failures.ts b/packages/core/src/kernel/dispatch/last-failures.ts new file mode 100644 index 00000000..48223081 --- /dev/null +++ b/packages/core/src/kernel/dispatch/last-failures.ts @@ -0,0 +1,79 @@ +import type { SubscriberFailureStore } from "../../adapter/storage/subscriber-failure-store.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; +import type { Logger } from "../../contracts/logger.ts"; +import type { AggregatesRuntime } from "../aggregate/runtime.ts"; +import { sealError } from "../personal-data/errors.ts"; +import { openedUnder } from "../personal-data/event-store.ts"; +import { personOfEvent } from "../personal-data/persons.ts"; +import type { PersonalKeys } from "../personal-data/wrapped-keys.ts"; + +export interface RecordLastFailureArgs { + readonly subscriber: string; + /** + * The event the subscriber failed on, as it was handed out. + */ + readonly event: StoredEvent; + readonly error: unknown; + readonly at: Date; +} + +/** + * What the dispatcher keeps of the failures it meets: each subscriber's last one, its text sealed + * under the person the event names, which no log or span carries without + * `runtime.telemetry.errorText: "include"`. + */ +export interface LastFailures { + /** + * Never rejects: a store that fails is logged, and the delivery goes on as it would. + */ + record(args: RecordLastFailureArgs): Promise; + /** + * Never rejects, as `record`. + */ + resolve(subscriber: string, at: Date): Promise; +} + +export interface CreateLastFailuresArgs { + readonly store: SubscriberFailureStore; + readonly aggregates: AggregatesRuntime; + readonly keys: PersonalKeys; + readonly logger: Logger; +} + +export interface CreateLastFailuresFunction { + (args: CreateLastFailuresArgs): LastFailures; +} + +export const createLastFailures: CreateLastFailuresFunction = ({ + store, + aggregates, + keys, + logger, +}) => ({ + record: async ({ subscriber, event, error, at }) => { + try { + const person = personOfEvent({ aggregates, event }); + await store.record({ + subscriber, + position: event.position, + eventId: event.id, + eventType: event.type, + person, + ...(await sealError({ error, person, keys, read: openedUnder(event) })), + failedAt: at.toISOString(), + }); + } catch (recordError) { + logger.error("subscriber failure could not be recorded", { + subscriber, + error: recordError, + }); + } + }, + resolve: async (subscriber, at) => { + try { + await store.resolve(subscriber, at); + } catch (error) { + logger.error("subscriber failure could not be marked resolved", { subscriber, error }); + } + }, +}); diff --git a/packages/core/src/kernel/dispatch/subscriber-failures.ts b/packages/core/src/kernel/dispatch/subscriber-failures.ts new file mode 100644 index 00000000..cbe018e5 --- /dev/null +++ b/packages/core/src/kernel/dispatch/subscriber-failures.ts @@ -0,0 +1,115 @@ +import type { + StoredSubscriberFailure, + SubscriberFailureStore, +} from "../../adapter/storage/subscriber-failure-store.ts"; +import type { ErrorText } from "../../config/types.ts"; +import { type RecordAccess, recorded } from "../personal-data/access.ts"; +import { sealedKeys } from "../personal-data/commands.ts"; +import { openTextWith } from "../personal-data/errors.ts"; +import type { PersonalKeys } from "../personal-data/wrapped-keys.ts"; +import { tracedOperation } from "../telemetry.ts"; + +/** + * A subscriber's last failure, with the error's text opened: `null` once the person the event + * named, under whose key it was sealed, is forgotten. + */ +export interface SubscriberFailure + extends Omit { + readonly errorMessage: string | null; + readonly errorStack?: string | null; +} + +/** + * The last failure of each subscriber, kept with its text whatever + * `runtime.telemetry.errorText` says, since a projection's error otherwise leaves no text + * anywhere. Each call that finds a failure hands its text out opened, so it is recorded in the + * access log once it settles, naming the person the event named, and rejects with what the log's + * hook throws. + */ +export interface SubscriberFailures { + /** + * Every subscriber's last failure, by subscriber, resolved ones included. + */ + list(): Promise; + /** + * One subscriber's last failure, `null` when it never failed. + */ + get(subscriber: string): Promise; +} + +export interface CreateSubscriberFailuresArgs { + readonly store: SubscriberFailureStore; + readonly keys: PersonalKeys; + readonly actor: string | undefined; + readonly recordAccess: RecordAccess; + readonly errorText: ErrorText; +} + +export interface CreateSubscriberFailuresFunction { + (args: CreateSubscriberFailuresArgs): SubscriberFailures; +} + +const personsOf = (failures: readonly StoredSubscriberFailure[]): string[] => + failures.flatMap(({ person }) => (person === null ? [] : [person])); + +export const createSubscriberFailures: CreateSubscriberFailuresFunction = ({ + store, + keys, + actor, + recordAccess, + errorText, +}) => { + const opened = async ( + { errorMessage, errorStack, ...failure }: StoredSubscriberFailure, + open: (value: StoredSubscriberFailure["errorMessage"]) => Promise, + ): Promise => ({ + ...failure, + errorMessage: await open(errorMessage), + ...(errorStack === undefined ? {} : { errorStack: await open(errorStack) }), + }); + + const list = async (): Promise => { + const failures = await store.list(); + // Each person's key read once for the whole list. + const open = openTextWith(sealedKeys(keys)); + return recorded( + (outcome) => + recordAccess({ + operation: "subscriberFailures.list", + detail: { failures: failures.length }, + actor, + persons: personsOf(failures), + outcome, + }), + () => Promise.all(failures.map((failure) => opened(failure, open))), + ); + }; + + const get = async (subscriber: string): Promise => { + const failure = await store.get(subscriber); + if (failure === null) return null; + return recorded( + (outcome) => + recordAccess({ + operation: "subscriberFailures.get", + detail: { subscriber }, + actor, + persons: personsOf([failure]), + outcome, + }), + () => opened(failure, openTextWith(sealedKeys(keys))), + ); + }; + + return { + list: () => + tracedOperation({ operation: "subscriberFailures.list", actor, errorText, run: list }), + get: (subscriber) => + tracedOperation({ + operation: "subscriberFailures.get", + actor, + errorText, + run: () => get(subscriber), + }), + }; +}; diff --git a/packages/core/src/kernel/personal-data/export.test.ts b/packages/core/src/kernel/personal-data/export.test.ts index f9395182..64a993c4 100644 --- a/packages/core/src/kernel/personal-data/export.test.ts +++ b/packages/core/src/kernel/personal-data/export.test.ts @@ -305,6 +305,7 @@ describe("exportPersonalData", () => { ], scheduledCommands: [], deadLetters: [], + subscriberFailures: [], readModels: { customerDirectory: [], orderNotes: [], orderCustomers: [] }, }); expect(await exportOf("c-1", 100)).toEqual(exported); diff --git a/packages/core/src/kernel/personal-data/export.ts b/packages/core/src/kernel/personal-data/export.ts index db5f27f2..caaa493c 100644 --- a/packages/core/src/kernel/personal-data/export.ts +++ b/packages/core/src/kernel/personal-data/export.ts @@ -70,6 +70,18 @@ export interface ExportedDeadLetter { readonly data: ExportedData; } +/** + * The last failure of a subscriber on an event that names the person, whose error text is sealed + * under their key: opened, `null` once they are forgotten. + */ +export interface ExportedSubscriberFailure { + readonly subscriber: string; + readonly eventId: string; + readonly eventType: string; + readonly errorMessage: string | null; + readonly errorStack?: string | null; +} + /** * A row of a read model whose field marked `.person()` names the person: its primary key, which * tells it apart, and its fields marked `.personal()`, a field the row does not have left out. @@ -94,6 +106,7 @@ export interface PersonalDataExport { readonly events: readonly ExportedEvent[]; readonly scheduledCommands: readonly ExportedScheduledCommand[]; readonly deadLetters: readonly ExportedDeadLetter[]; + readonly subscriberFailures: readonly ExportedSubscriberFailure[]; /** * By read model, every one with a field marked `.person()`, rows or none. */ @@ -263,12 +276,27 @@ export const exportPersonalData: ExportPersonalDataFunction = async ({ const deadLetters = await Promise.all( (await storage.deadLetterStore.listByPerson(personId)).map(exportedLetter), ); + // One row per subscriber: few enough to read whole. + const subscriberFailures = await Promise.all( + (await storage.subscriberFailures.list()) + .filter((failure) => failure.person === personId) + .map(async (failure) => ({ + subscriber: failure.subscriber, + eventId: failure.eventId, + eventType: failure.eventType, + errorMessage: await openText(failure.errorMessage), + ...(failure.errorStack === undefined + ? {} + : { errorStack: await openText(failure.errorStack) }), + })), + ); return { personId, exportedAt, events, scheduledCommands, deadLetters, + subscriberFailures, readModels: Object.fromEntries(await rows), }; }; diff --git a/packages/core/src/kernel/status.ts b/packages/core/src/kernel/status.ts new file mode 100644 index 00000000..3109c430 --- /dev/null +++ b/packages/core/src/kernel/status.ts @@ -0,0 +1,164 @@ +import type { Adapter, SchemaScopeName, SchemaVersion } from "../adapter/adapter.ts"; +import { isAdapter } from "../adapter/adapter.ts"; +import type { + StoredSubscriberFailure, + SubscriberFailureStore, +} from "../adapter/storage/subscriber-failure-store.ts"; +import type { ResolvedConfig } from "../config/types.ts"; +import type { AppLag } from "./app.ts"; +import type { DeadLetterRetention, DeadLetterRetentionStatus } from "./dead-letters/retention.ts"; +import type { PersonalDataBackfill } from "./personal-data/backfill.ts"; +import type { + ForgetLedgerReconciliation, + ForgetLedgerReconciliationStatus, +} from "./personal-data/reconciliation.ts"; +import type { RetentionStatus, RetentionSweep } from "./personal-data/retention.ts"; +import type { KeyRotation, KeyRotationStatus } from "./personal-data/rotation.ts"; +import type { RestoreWatch } from "./shared/restores.ts"; + +/** + * A scope of Bounda's tables where it lives: `storage`, a read model's database of its own as + * `readModels.`, or `forgetLedger`. + */ +export interface SchemaStatus extends SchemaVersion { + readonly database: string; +} + +/** + * A subscriber's last failure without its text, which `subscriberFailures` opens. `current` while + * the subscriber is still stuck on it. + */ +export interface SubscriberFailureStatus + extends Omit { + readonly current: boolean; +} + +/** + * Where the runtime's background work stands, as the database says: what `bounda status` prints. + * Nothing in it is personal data, so reading it records nothing in the access log. + */ +export interface AppStatus { + /** + * Every scope of Bounda's tables the app uses, at the version its database holds beside the + * latest this version of Bounda migrates it to. + */ + readonly schema: readonly SchemaStatus[]; + /** + * How many times a running instance found the database gone back to an earlier copy. + */ + readonly restoresFound: number; + readonly lag: AppLag; + readonly subscriberFailures: readonly SubscriberFailureStatus[]; + /** + * `null` when no schema marks personal data. + */ + readonly personalDataBackfill: AppLag["personalDataBackfill"]; + /** + * `null` without `runtime.personalData.forgetLedger`. + */ + readonly forgetLedger: ForgetLedgerReconciliationStatus | null; + /** + * `null` without `runtime.personalData.keyWrapper`. + */ + readonly keyRotation: KeyRotationStatus | null; + /** + * `null` without `runtime.personalData.retention`. + */ + readonly retention: RetentionStatus | null; + /** + * `null` without `runtime.deadLetters.retention`. + */ + readonly deadLetterRetention: DeadLetterRetentionStatus | null; +} + +export interface ReadAppStatusArgs { + readonly config: ResolvedConfig; + readonly lag: () => Promise; + readonly failures: SubscriberFailureStore; + readonly restores: RestoreWatch; + readonly backfill?: PersonalDataBackfill | undefined; + readonly reconciliation?: ForgetLedgerReconciliation | undefined; + readonly rotation?: KeyRotation | undefined; + readonly retention?: RetentionSweep | undefined; + readonly letterRetention?: DeadLetterRetention | undefined; +} + +export interface ReadAppStatusFunction { + (args: ReadAppStatusArgs): Promise; +} + +interface Database { + readonly database: string; + readonly adapter: Adapter; + readonly scopes: readonly SchemaScopeName[]; +} + +// Each database the configuration names, once, with the scopes it keeps there. +const databasesOf = (config: ResolvedConfig): readonly Database[] => { + const databases: Database[] = []; + if (isAdapter(config.storage)) { + databases.push({ + database: "storage", + adapter: config.storage, + scopes: ["storage", "checkpoints"], + }); + } + for (const [name, adapter] of Object.entries(config.readModels)) { + if (!isAdapter(adapter) || databases.some((known) => known.adapter === adapter)) continue; + databases.push({ database: `readModels.${name}`, adapter, scopes: ["checkpoints"] }); + } + const ledger = config.runtime.personalData.forgetLedger?.store; + if (isAdapter(ledger)) { + databases.push({ database: "forgetLedger", adapter: ledger, scopes: ["forget-ledger"] }); + } + return databases; +}; + +const schemaOf = async ({ database, adapter, scopes }: Database): Promise => + (await adapter.schemaVersions()) + .filter(({ scope }) => (scopes as readonly string[]).includes(scope)) + .map((version) => ({ database, ...version })); + +export const readAppStatus: ReadAppStatusFunction = async ({ + config, + lag, + failures, + restores, + backfill, + reconciliation, + rotation, + retention, + letterRetention, +}) => { + const [schema, restoresFound, current, stored, forgetLedger, keyRotation, retained, purged] = + await Promise.all([ + Promise.all(databasesOf(config).map(schemaOf)).then((all) => all.flat()), + restores.found(), + lag(), + failures.list(), + reconciliation?.status() ?? null, + rotation?.status() ?? null, + retention?.status() ?? null, + letterRetention?.status() ?? null, + ]); + const positions = new Map(current.subscribers.map((each) => [each.subscriber, each.position])); + return { + schema, + restoresFound, + lag: current, + // A subscriber the app no longer runs is stuck on nothing. + subscriberFailures: stored.map(({ errorMessage: _message, errorStack: _stack, ...failure }) => { + const position = positions.get(failure.subscriber); + return { + ...failure, + current: + failure.resolvedAt === null && position !== undefined && position < failure.position, + }; + }), + personalDataBackfill: backfill === undefined ? null : await backfill.status(), + forgetLedger, + keyRotation, + retention: retained, + deadLetterRetention: purged, + }; +}; diff --git a/packages/core/src/memory/index.ts b/packages/core/src/memory/index.ts index 64130dad..2a379d55 100644 --- a/packages/core/src/memory/index.ts +++ b/packages/core/src/memory/index.ts @@ -26,6 +26,7 @@ import { createKeptPersonStore } from "./person-store.ts"; import { createKeptPersonalKeyStore } from "./personal-key-store.ts"; import { createKeptScheduler } from "./scheduler.ts"; import { createMemorySnapshotStore } from "./snapshot-store.ts"; +import { createMemorySubscriberFailureStore } from "./subscriber-failure-store.ts"; import { createMemoryReadClient, createMemoryTable, type MemoryTable } from "./table.ts"; import { createCheckpointJournal, @@ -122,6 +123,9 @@ export const memory: MemoryFunction = (options = {}) => { kind: "bounda-adapter", name: "memory", options, + // Nothing to version: the stores live in this process alone. + schemaVersions: async () => [], + migrate: async () => [], createStorage: async () => { const notifier = createMemoryEventNotifier(); const inboxLedger = createKeptInboxLedger(); @@ -185,6 +189,7 @@ export const memory: MemoryFunction = (options = {}) => { notifier, checkpointStore, snapshotStore, + subscriberFailures: createMemorySubscriberFailureStore(), transact: createMemoryStorageTransaction({ ...stores, entries: { diff --git a/packages/core/src/memory/memory.test.ts b/packages/core/src/memory/memory.test.ts index 0d0910a6..9175c23f 100644 --- a/packages/core/src/memory/memory.test.ts +++ b/packages/core/src/memory/memory.test.ts @@ -18,6 +18,7 @@ import { schedulerContract, snapshotStoreContract, storageTransactionContract, + subscriberFailureStoreContract, tableContract, viewContract, } from "../adapter/testing/index.ts"; @@ -41,6 +42,7 @@ describe("memory adapter", () => { personalKeyStoreContract({ create: async () => (await storage()).personalKeys }); personStoreContract({ create: async () => (await storage()).persons }); inspectionContract({ create: async () => memory() }); + subscriberFailureStoreContract({ create: async () => (await storage()).subscriberFailures }); forgetLedgerContract({ create: async () => { const adapter = memory(); diff --git a/packages/core/src/memory/subscriber-failure-store.ts b/packages/core/src/memory/subscriber-failure-store.ts new file mode 100644 index 00000000..e76ad248 --- /dev/null +++ b/packages/core/src/memory/subscriber-failure-store.ts @@ -0,0 +1,43 @@ +import type { + StoredSubscriberFailure, + SubscriberFailureStore, +} from "../adapter/storage/subscriber-failure-store.ts"; +import { jsonCopy } from "./json-copy.ts"; + +export interface CreateMemorySubscriberFailureStoreFunction { + (): SubscriberFailureStore; +} + +export const createMemorySubscriberFailureStore: CreateMemorySubscriberFailureStoreFunction = + () => { + const failures = new Map(); + return { + record: async ({ failedAt, ...failure }) => { + const kept = failures.get(failure.subscriber); + const again = kept?.resolvedAt === null && kept.eventId === failure.eventId; + failures.set( + failure.subscriber, + jsonCopy({ + ...failure, + attempts: again ? kept.attempts + 1 : 1, + firstFailedAt: again ? kept.firstFailedAt : failedAt, + lastFailedAt: failedAt, + resolvedAt: null, + }), + ); + }, + resolve: async (subscriber, at) => { + const kept = failures.get(subscriber); + if (kept === undefined || kept.resolvedAt !== null) return; + failures.set(subscriber, { ...kept, resolvedAt: at.toISOString() }); + }, + get: async (subscriber) => { + const kept = failures.get(subscriber); + return kept === undefined ? null : jsonCopy(kept); + }, + list: async () => + [...failures.values()] + .sort((a, b) => (a.subscriber < b.subscriber ? -1 : 1)) + .map((kept) => jsonCopy(kept)), + }; + }; diff --git a/packages/core/src/modules/audit.ts b/packages/core/src/modules/audit.ts index 67946cae..355f9445 100644 --- a/packages/core/src/modules/audit.ts +++ b/packages/core/src/modules/audit.ts @@ -63,6 +63,7 @@ export interface ExportAccess extends AccessRecord { readonly events: number; readonly scheduledCommands: number; readonly deadLetters: number; + readonly subscriberFailures: number; readonly readModelRows: number; } | null; } @@ -88,6 +89,27 @@ export interface DeadLetterAccess extends AccessRecord { }; } +/** + * `subscriberFailures.list`, with how many failures it read. `persons` are those the failures' + * events name, under whose key their error text is sealed. + */ +export interface SubscriberFailuresListAccess extends AccessRecord { + readonly operation: "subscriberFailures.list"; + readonly detail: { + readonly failures: number; + }; +} + +/** + * `subscriberFailures.get` of one subscriber's last failure. + */ +export interface SubscriberFailureAccess extends AccessRecord { + readonly operation: "subscriberFailures.get"; + readonly detail: { + readonly subscriber: string; + }; +} + /** * One record of the access log: an operation that handed out personal data in the clear, who * called it and whose data it was. @@ -96,7 +118,9 @@ export type PersonalDataAccess = | QueryAccess | ExportAccess | DeadLettersListAccess - | DeadLetterAccess; + | DeadLetterAccess + | SubscriberFailuresListAccess + | SubscriberFailureAccess; /** * What `onPersonalDataAccess` receives: the record, and the host's environment and store name as a diff --git a/packages/core/src/modules/index.ts b/packages/core/src/modules/index.ts index 88e60476..a3750551 100644 --- a/packages/core/src/modules/index.ts +++ b/packages/core/src/modules/index.ts @@ -15,6 +15,8 @@ export type { OnPersonalDataAccessArgs, PersonalDataAccess, QueryAccess, + SubscriberFailureAccess, + SubscriberFailuresListAccess, } from "./audit.ts"; export { MAX_ACCESS_PERSONS } from "./audit.ts"; export type { diff --git a/packages/core/test-types/inference.test-d.ts b/packages/core/test-types/inference.test-d.ts index 04d9e55e..cb9ede16 100644 --- a/packages/core/test-types/inference.test-d.ts +++ b/packages/core/test-types/inference.test-d.ts @@ -2,6 +2,7 @@ import type { AccessOutcome, AfterOptions, AppRegistry, + AppStatus, BoundaApp, BoundaClient, CancelledCommand, @@ -21,6 +22,7 @@ import type { DeadLetter, DeadLetterAccess, DeadLetterKind, + DeadLetterRetentionStatus, DecidedDispatch, DispatchOptions, DispatchResult, @@ -35,16 +37,19 @@ import type { ExportedEvent, ExportedRow, ExportedScheduledCommand, + ExportedSubscriberFailure, Field, FieldFilter, FieldsArgs, FieldTypes, FieldTypesOf, FieldTypesOfRow, + ForgetLedgerReconciliationStatus, ForgetsPersonalData, HasPersonalFields, ImplementationModule, Instant, + KeyRotationStatus, Logger, OnPersonalDataAccess, OnPersonalDataAccessArgs, @@ -63,6 +68,7 @@ import type { ProcessingRestrictedPayload, ProcessingRestrictionLiftedEvent, ProcessingRestrictionLiftedPayload, + ProcessStatus, QueriesFacade, RangeFilter, ReactionDispatchResult, @@ -73,10 +79,15 @@ import type { RejectFunction, RejectionCodeOf, RemovedDeadLetter, + RetentionStatus, ScheduledDispatch, + SchemaStatus, SetFilter, StoredDispatch, StoredEvent, + SubscriberFailure, + SubscriberFailureStatus, + SubscriberFailures, Table, Where, } from "@bounda-dev/core"; @@ -87,6 +98,12 @@ import { MAX_ACCESS_PERSONS, MAX_ACTOR_LENGTH, } from "@bounda-dev/core"; +import type { + Adapter, + SchemaScopeName, + SchemaVersion, + StoredSubscriberFailure, +} from "@bounda-dev/core/adapter"; import { describe, expectTypeOf, it } from "vitest"; import type { registry } from "./fixtures/order-app/.bounda/registry.ts"; import type { @@ -653,6 +670,8 @@ describe("personal data", () => { | "deadLetters.get" | "deadLetters.retry" | "deadLetters.discard" + | "subscriberFailures.list" + | "subscriberFailures.get" >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -670,12 +689,19 @@ describe("personal data", () => { expectTypeOf< Extract["detail"] >().toEqualTypeOf<{ readonly letters: number }>(); + expectTypeOf< + Extract["detail"] + >().toEqualTypeOf<{ readonly failures: number }>(); + expectTypeOf< + Extract["detail"] + >().toEqualTypeOf<{ readonly subscriber: string }>(); expectTypeOf< Extract["detail"] >().toEqualTypeOf<{ readonly events: number; readonly scheduledCommands: number; readonly deadLetters: number; + readonly subscriberFailures: number; readonly readModelRows: number; } | null>(); expectTypeOf().toEqualTypeOf(); @@ -761,8 +787,16 @@ describe("personal data", () => { readonly events: readonly ExportedEvent[]; readonly scheduledCommands: readonly ExportedScheduledCommand[]; readonly deadLetters: readonly ExportedDeadLetter[]; + readonly subscriberFailures: readonly ExportedSubscriberFailure[]; readonly readModels: Readonly>; }>(); + expectTypeOf().toEqualTypeOf<{ + readonly subscriber: string; + readonly eventId: string; + readonly eventType: string; + readonly errorMessage: string | null; + readonly errorStack?: string | null; + }>(); expectTypeOf().toEqualTypeOf<{ readonly key: unknown; readonly data: ExportedData; @@ -1269,3 +1303,76 @@ describe("facades", () => { >().toEqualTypeOf(); }); }); + +describe("operations", () => { + it("types the status every client reads, without text, and the scopes of the tables", () => { + expectTypeOf().toEqualTypeOf<() => Promise>(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf<{ + readonly database: string; + readonly scope: string; + readonly version: number; + readonly latest: number; + }>(); + expectTypeOf().toEqualTypeOf<{ + readonly scope: string; + readonly version: number; + readonly latest: number; + }>(); + expectTypeOf().toEqualTypeOf<"storage" | "checkpoints" | "forget-ledger">(); + expectTypeOf().toEqualTypeOf< + () => Promise + >(); + expectTypeOf().toEqualTypeOf< + (scopes: readonly SchemaScopeName[]) => Promise + >(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf< + AppStatus["deadLetterRetention"] + >().toEqualTypeOf(); + expectTypeOf< + AppStatus["forgetLedger"] + >().toEqualTypeOf(); + expectTypeOf< + AppStatus["personalDataBackfill"] + >().toEqualTypeOf(); + expectTypeOf< + "errorMessage" | "errorStack" extends keyof SubscriberFailureStatus ? true : false + >().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf<{ + readonly kid: string; + readonly previous: readonly string[]; + readonly pending: number; + readonly completeAt: string | null; + }>(); + expectTypeOf().toEqualTypeOf<{ + readonly retentionMs: number; + readonly due: number; + readonly nextDueAt: string | null; + }>(); + }); + + it("opens a subscriber's last failure only through a recorded call", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + () => Promise + >(); + expectTypeOf().toEqualTypeOf< + (subscriber: string) => Promise + >(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + }); + + it("names a process given up as abandoned", () => { + expectTypeOf().toEqualTypeOf< + "started" | "completed" | "failed" | "timed_out" | "abandoned" + >(); + }); +}); diff --git a/packages/postgresql/src/index.ts b/packages/postgresql/src/index.ts index 845b6333..65a5a846 100644 --- a/packages/postgresql/src/index.ts +++ b/packages/postgresql/src/index.ts @@ -2,6 +2,7 @@ import type { Adapter, CreateReadModelArgs, CreateReadModelRebuildArgs, + SchemaScopeName, StorageTransaction, } from "@bounda-dev/core/adapter"; import { @@ -9,9 +10,11 @@ import { createSqlForgetLedger, createSqlPersonalKeyStore, createSqlPersonStore, + createSqlSubscriberFailureStore, inspectSqlTable, postgresqlDialect, quoteName, + type SchemaScope, type SqlTransaction, } from "@bounda-dev/core/adapter/sql"; import postgres, { type Sql } from "postgres"; @@ -26,6 +29,7 @@ import { postgresqlCheckpointsScope, postgresqlForgetLedgerScope, postgresqlStorageScope, + readPostgresqlSchemaVersions, } from "./migrations.ts"; import { type PostgresqlOptions, resolvePostgresqlOptions } from "./options.ts"; import { openPostgresqlReadModel, rebuildPostgresqlReadModel } from "./read-model.ts"; @@ -98,10 +102,38 @@ export const postgresql: PostgresqlFunction = (options) => { } }; + const scopes: Readonly> = { + storage: postgresqlStorageScope({ schema, tables }), + checkpoints: postgresqlCheckpointsScope(tables.checkpoints), + "forget-ledger": postgresqlForgetLedgerScope(tables.forgetLedger), + }; + // A use of the pool for one call, handed back whatever the call does. + const briefly = async (work: (connection: Connection) => Promise): Promise => { + const connection = open(); + try { + return await work(connection); + } finally { + await release(); + } + }; + const versionsOn = (db: PostgresqlDatabase) => + readPostgresqlSchemaVersions({ db, prefix: tablePrefix, scopes: Object.values(scopes) }); + return { kind: "bounda-adapter", name: "postgresql", options, + schemaVersions: () => briefly(({ db }) => versionsOn(db)), + migrate: (names) => + briefly(async ({ db }) => { + await migratePostgresql({ + db, + schema, + prefix: tablePrefix, + scopes: names.map((scope) => scopes[scope]), + }); + return versionsOn(db); + }), createStorage: () => using(async ({ db, sql }) => { await migratePostgresql({ @@ -162,6 +194,11 @@ export const postgresql: PostgresqlFunction = (options) => { table: tables.checkpoints, dialect: postgresqlDialect, }), + subscriberFailures: createSqlSubscriberFailureStore({ + db, + table: tables.subscriberFailures, + dialect: postgresqlDialect, + }), // The append lock comes first, before any row the work may lock: two transactions that // took the lock and a row in opposite orders would deadlock. transact: (work) => diff --git a/packages/postgresql/src/migrations.ts b/packages/postgresql/src/migrations.ts index 0959fd57..1335c5e9 100644 --- a/packages/postgresql/src/migrations.ts +++ b/packages/postgresql/src/migrations.ts @@ -1,9 +1,11 @@ import { ConfigurationError } from "@bounda-dev/core"; +import type { SchemaVersion } from "@bounda-dev/core/adapter"; import { forgetLedgerStatements, migrateSchemas, postgresqlDialect, quoteIdentifier, + readSchemaVersions, resolvedAtOfEarlierLetters, SCHEMA_SCOPES, type SchemaScope, @@ -17,6 +19,7 @@ import { inSchema, type StorageTables, storageSchemaStatements, + subscriberFailuresTableStatement, } from "./schema.ts"; type StorageTable = Exclude< @@ -143,7 +146,13 @@ export interface PostgresqlStorageScopeFunction { export const postgresqlStorageScope: PostgresqlStorageScopeFunction = ({ schema, tables }) => ({ scope: SCHEMA_SCOPES.storage, - migrations: [{ version: 1, migrate: (tx) => adoptStorageTables(tx, schema, tables) }], + migrations: [ + { version: 1, migrate: (tx) => adoptStorageTables(tx, schema, tables) }, + { + version: 2, + migrate: (tx) => tx.run(subscriberFailuresTableStatement(tables.subscriberFailures), []), + }, + ], }); export interface PostgresqlCheckpointsScopeFunction { @@ -187,6 +196,32 @@ export interface MigratePostgresqlFunction { (args: MigratePostgresqlArgs): Promise; } +const isUndefinedTable = (error: unknown): boolean => + error instanceof Error && Reflect.get(error, "code") === UNDEFINED_TABLE; + +export interface ReadPostgresqlSchemaVersionsArgs { + readonly db: PostgresqlDatabase; + readonly prefix: string; + readonly scopes: readonly SchemaScope[]; +} + +export interface ReadPostgresqlSchemaVersionsFunction { + (args: ReadPostgresqlSchemaVersionsArgs): Promise; +} + +// In the schema the connection's `search_path` names, where the adapter keeps its tables. +export const readPostgresqlSchemaVersions: ReadPostgresqlSchemaVersionsFunction = ({ + db, + prefix, + scopes, +}) => + readSchemaVersions({ + db, + table: schemaMigrationsTableFor(prefix), + scopes, + isMissingTable: isUndefinedTable, + }); + /** * `migrateSchemas` in a schema, under the schema's advisory lock, which the read models' changes * to their tables take too. @@ -198,6 +233,5 @@ export const migratePostgresql: MigratePostgresqlFunction = ({ db, schema, prefi table: schemaMigrationsTableFor(prefix), scopes, locked: (work) => inSchema({ db, schema, work }), - isMissingTable: (error) => - error instanceof Error && Reflect.get(error, "code") === UNDEFINED_TABLE, + isMissingTable: isUndefinedTable, }); diff --git a/packages/postgresql/src/postgresql.test.ts b/packages/postgresql/src/postgresql.test.ts index f5eca94a..83c958d8 100644 --- a/packages/postgresql/src/postgresql.test.ts +++ b/packages/postgresql/src/postgresql.test.ts @@ -32,6 +32,7 @@ import { snapshotStoreContract, storageSchemaContract, storageTransactionContract, + subscriberFailureStoreContract, tableContract, viewContract, } from "@bounda-dev/core/adapter/testing"; @@ -130,6 +131,7 @@ describe.skipIf(container === null)("postgresql adapter", () => { return fresh({ tablePrefix: "bounda_", schema: `inspected_${run}_${prefixes}` }); }, }); + subscriberFailureStoreContract({ create: async () => (await openStorage()).subscriberFailures }); forgetLedgerContract({ create: async () => { await closeOpened(); diff --git a/packages/postgresql/src/schema.ts b/packages/postgresql/src/schema.ts index 12095d18..009af396 100644 --- a/packages/postgresql/src/schema.ts +++ b/packages/postgresql/src/schema.ts @@ -16,6 +16,7 @@ export interface StorageTables { readonly personalKeys: string; readonly persons: string; readonly snapshots: string; + readonly subscriberFailures: string; // In a database of its own, apart from the other storage tables. readonly forgetLedger: string; readonly appendLockKey: string; @@ -61,6 +62,9 @@ export const storageTablesFor: StorageTablesForFunction = ({ prefix, schema }) = personalKeys: quoteIdentifier(storageTableNameFor({ prefix, table: "personalKeys" })), persons: quoteIdentifier(storageTableNameFor({ prefix, table: "persons" })), snapshots: quoteIdentifier(storageTableNameFor({ prefix, table: "snapshots" })), + subscriberFailures: quoteIdentifier( + storageTableNameFor({ prefix, table: "subscriberFailures" }), + ), forgetLedger: quoteIdentifier(forgetLedger), appendLockKey: `bounda:${schema}.${events}`, checkpointsLockKey: `bounda:${schema}.${checkpoints}`, @@ -81,6 +85,28 @@ export const checkpointTableStatement: CheckpointTableStatementFunction = (table "position" bigint NOT NULL )`; +export interface SubscriberFailuresTableStatementFunction { + (table: string): string; +} + +// Migration 2 of the storage tables. One row per subscriber, without an index: there are few. +export const subscriberFailuresTableStatement: SubscriberFailuresTableStatementFunction = (table) => + `CREATE TABLE IF NOT EXISTS ${table} ( + "subscriber" text PRIMARY KEY, + "position" bigint NOT NULL, + "event_id" text NOT NULL, + "event_type" text NOT NULL, + "person" text, + "error_name" text NOT NULL, + "error_code" text, + "error_message" text NOT NULL, + "error_stack" text, + "attempts" integer NOT NULL, + "first_failed_at" text NOT NULL, + "last_failed_at" text NOT NULL, + "resolved_at" text + )`; + export interface StorageSchemaStatementsFunction { (tables: StorageTables): readonly string[]; } diff --git a/packages/react-router/src/cloudflare/clients.test.ts b/packages/react-router/src/cloudflare/clients.test.ts index 94f3c141..db307ac9 100644 --- a/packages/react-router/src/cloudflare/clients.test.ts +++ b/packages/react-router/src/cloudflare/clients.test.ts @@ -67,6 +67,9 @@ describe("cloudflareClients", () => { })(requestArgs()); await client.getLag(); + await client.status(); + await client.subscriberFailures.list(); + await client.subscriberFailures.get("projection:orders"); await client.events.read({ after: 3, limit: 10 }); await client.deadLetters.list({ limit: 5 }); await client.deadLetters.retry("d-1"); @@ -78,6 +81,9 @@ describe("cloudflareClients", () => { await client.unrestrict("c-2"); expect(store.calls).toEqual([ ["acme", "lag"], + ["acme", "status"], + ["acme", "listSubscriberFailures", undefined], + ["acme", "getSubscriberFailure", "projection:orders", undefined], ["acme", "readEvents", { after: 3, limit: 10 }], ["acme", "listDeadLetters", { limit: 5 }, undefined], ["acme", "retryDeadLetter", "d-1", undefined], diff --git a/packages/react-router/src/cloudflare/clients.ts b/packages/react-router/src/cloudflare/clients.ts index 66dea885..ac5fabe2 100644 --- a/packages/react-router/src/cloudflare/clients.ts +++ b/packages/react-router/src/cloudflare/clients.ts @@ -40,6 +40,9 @@ const lazyStub = (stubOf: () => Promise): BoundaStub => ({ command: async (...args) => (await stubOf()).command(...args), query: async (...args) => (await stubOf()).query(...args), lag: async () => (await stubOf()).lag(), + status: async () => (await stubOf()).status(), + listSubscriberFailures: async (...args) => (await stubOf()).listSubscriberFailures(...args), + getSubscriberFailure: async (...args) => (await stubOf()).getSubscriberFailure(...args), readEvents: async (args) => (await stubOf()).readEvents(args), listDeadLetters: async (...args) => (await stubOf()).listDeadLetters(...args), retryDeadLetter: async (...args) => (await stubOf()).retryDeadLetter(...args), diff --git a/packages/react-router/src/cloudflare/test-support.ts b/packages/react-router/src/cloudflare/test-support.ts index 16b765f8..5bf35b3b 100644 --- a/packages/react-router/src/cloudflare/test-support.ts +++ b/packages/react-router/src/cloudflare/test-support.ts @@ -34,6 +34,9 @@ export const storeNamespace = (): FakeStoreNamespace => { command: record("command"), query: record("query"), lag: record("lag"), + status: record("status"), + listSubscriberFailures: record("listSubscriberFailures"), + getSubscriberFailure: record("getSubscriberFailure"), readEvents: record("readEvents"), listDeadLetters: record("listDeadLetters"), retryDeadLetter: record("retryDeadLetter"), diff --git a/packages/sqlite/src/sqlite.test.ts b/packages/sqlite/src/sqlite.test.ts index a31720ad..3e2227ba 100644 --- a/packages/sqlite/src/sqlite.test.ts +++ b/packages/sqlite/src/sqlite.test.ts @@ -22,12 +22,14 @@ import { inspectionContract, pendingEvent, personalKeyStoreContract, + personStoreContract, readModelRebuildContract, readModelTransactionContract, schedulerContract, snapshotStoreContract, storageSchemaContract, storageTransactionContract, + subscriberFailureStoreContract, tableContract, viewContract, } from "@bounda-dev/core/adapter/testing"; @@ -49,6 +51,8 @@ describe("sqlite adapter in memory", () => { snapshotStoreContract({ create: () => openStorage() }); storageTransactionContract({ create: () => openStorage() }); personalKeyStoreContract({ create: async () => (await openStorage()).personalKeys }); + personStoreContract({ create: async () => (await openStorage()).persons }); + subscriberFailureStoreContract({ create: async () => (await openStorage()).subscriberFailures }); tableContract({ create: async () => ( @@ -397,6 +401,7 @@ describe("sqlite adapter on a file", () => { "app_rm_order_summary", "app_scheduled_commands", "app_snapshots", + "app_subscriber_failures", ]); await readModel.close(); await second.close(); From 455903c20451639ba03641a35e7a82c397addf61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:33:15 +0100 Subject: [PATCH 04/19] feat(cli): bounda status and bounda migrate --- packages/cli/src/run.test.ts | 163 ++++++++ packages/cli/src/run.ts | 95 +++++ packages/cli/src/status-output.test.ts | 127 +++++++ packages/cli/src/status-output.ts | 113 ++++++ .../testing/storage-schema.contract.ts | 44 +++ packages/core/src/index.ts | 2 + .../dispatch/dispatcher-backoff.test.ts | 16 + .../src/kernel/dispatch/last-failures.test.ts | 64 ++++ .../dispatch/subscriber-failures-app.test.ts | 351 ++++++++++++++++++ packages/core/src/kernel/migrate.test.ts | 49 +++ packages/core/src/kernel/migrate.ts | 28 ++ .../personal-data/backfill-walk.test.ts | 42 +++ .../personal-data/reconciliation.test.ts | 71 ++++ .../src/kernel/shared/restores-app.test.ts | 31 ++ packages/core/src/kernel/status.ts | 23 +- 15 files changed, 1214 insertions(+), 5 deletions(-) create mode 100644 packages/cli/src/status-output.test.ts create mode 100644 packages/cli/src/status-output.ts create mode 100644 packages/core/src/kernel/dispatch/last-failures.test.ts create mode 100644 packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts create mode 100644 packages/core/src/kernel/migrate.test.ts create mode 100644 packages/core/src/kernel/migrate.ts diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 5ba996b7..8f3c51f7 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -761,3 +761,166 @@ describe("bounda forgotten and bounda forget", () => { ); }); }); + +describe("bounda status", () => { + const fixture = resolve(import.meta.dirname, "fixtures/rebuild-project"); + const failing = ["--registry", ".bounda/failing-registry.ts"]; + + const fail = async (): Promise => { + const project = await loadProject({ + root: fixture, + registryPath: ".bounda/failing-registry.ts", + logger: silentLogger, + }); + const app = await createApp({ ...project, logger: silentLogger }); + await app.commands.increment({ counterId: `c-${Date.now()}` }); + await app.runUntilIdle({ maxPasses: 2 }); + await app.stop(); + }; + + it("prints every subscriber's lag and last failure without its text, and the background jobs", async () => { + await fail(); + const result = await cli(["status", ...failing], fixture); + expect(result.code).toBe(EXIT_OK); + expect(besidesLogs(result.stderr)).toBe(""); + expect(result.stdout).not.toContain("ana@example.com"); + const lines = result.stdout.split("\n"); + expect(lines.slice(0, 3)).toEqual(["tables", " no tables to version", expect.any(String)]); + expect(lines[2]).toMatch(/^restores found while running: \d+$/); + expect(result.stdout).toMatch(/^ {2}projection:failingTotals {2}at \d+, \d+ behind$/m); + expect(result.stdout).toMatch( + /^ {4}failing: Error on Incremented at \d+, \d+ attempts? since \S+$/m, + ); + // The fixture's other registry marks personal data, and the store is shared by the tests. + expect(lines.at(-6)).toMatch( + /^personal data backfill: (no personal data marked|sealed through position \d+, caught up)$/, + ); + expect(lines.slice(-5, -1)).toEqual([ + "forget ledger: not configured", + "key rotation: not configured", + "retention: not configured", + "dead letter retention: not configured", + ]); + expect(logged(result.stderr, "info", "personal data accessed")).toEqual([]); + }); + + it("prints the text of each failure with --error-text, through a call the access log records", async () => { + await fail(); + const result = await cli( + ["status", ...failing, "--error-text", "--actor", "operator-1"], + fixture, + ); + expect(result.code).toBe(EXIT_OK); + expect(result.stdout).toMatch(/^ {4}cannot total ana@example\.com$/m); + expect(logged(result.stderr, "info", "personal data accessed")).toEqual([ + expect.objectContaining({ + operation: "subscriberFailures.list", + actor: "operator-1", + outcome: "ok", + }), + ]); + }); + + it("prints the status as JSON with --json, with the texts only when asked for", async () => { + await fail(); + const quiet = await cli(["status", ...failing, "--json"], fixture); + expect(quiet.code).toBe(EXIT_OK); + const status = JSON.parse(quiet.stdout); + expect(status).toMatchObject({ schema: [], forgetLedger: null, keyRotation: null }); + expect(status.subscriberFailures).toContainEqual( + expect.objectContaining({ subscriber: "projection:failingTotals", errorName: "Error" }), + ); + expect(quiet.stdout).not.toContain("ana@example.com"); + const told = await cli(["status", ...failing, "--json", "--error-text"], fixture); + expect(JSON.parse(told.stdout).subscriberFailures).toContainEqual( + expect.objectContaining({ errorMessage: "cannot total ana@example.com" }), + ); + }); + + it("documents its options in its help", async () => { + const help = await cli(["status", "--help"], fixture); + expect(help.code).toBe(EXIT_OK); + expect(help.stdout.replace(/\s+/g, " ")).toContain( + "print the version of the tables, every subscriber's lag and last failure, and how the backfill, forget ledger, key rotation and retention stand", + ); + for (const option of ["--json", "--error-text", "--actor ", "--registry "]) { + expect(help.stdout).toContain(option); + } + }); +}); + +describe("bounda migrate", () => { + // A configuration whose storage records the scopes it is asked to migrate: what the CLI hands + // over. Migrating real tables is core's, tested there on every database. + const migrating = async (): Promise => { + const root = await mkdtemp(join(tmpdir(), "bounda-cli-migrate-")); + temporary.push(root); + await writeFile( + join(root, "bounda.config.ts"), + `const versions = (scopes) => [ + { scope: "storage", version: scopes.includes("storage") ? 2 : 0, latest: 2 }, + { scope: "checkpoints", version: scopes.includes("checkpoints") ? 1 : 0, latest: 1 }, +]; +const adapter = { + kind: "bounda-adapter", + name: "recording", + options: {}, + createStorage: async () => { throw new Error("not opened by migrate"); }, + createReadModel: async () => { throw new Error("not opened by migrate"); }, + rebuildReadModel: async () => { throw new Error("not opened by migrate"); }, + createForgetLedger: async () => { throw new Error("not opened by migrate"); }, + schemaVersions: async () => versions([]), + migrate: async (scopes) => { + globalThis.migratedScopes = scopes; + if (process.env.REFUSE_MIGRATION === "1") throw new Error("too new for ana@example.com"); + return versions(scopes); + }, +}; +export default { storage: adapter }; +`, + ); + return root; + }; + + it("migrates the scopes of every database the configuration names, needing no registry", async () => { + const root = await migrating(); + const result = await cli(["migrate"], root); + expect(result.code).toBe(EXIT_OK); + expect(Reflect.get(globalThis, "migratedScopes")).toEqual(["storage", "checkpoints"]); + expect(result.stdout).toBe( + "migrated\n storage storage 2 of 2\n storage checkpoints 1 of 1\n", + ); + }); + + it("names the error a migration fails on without its text, and exits 2", async () => { + const root = await migrating(); + process.env.REFUSE_MIGRATION = "1"; + try { + const result = await cli(["migrate", "--root", root], repoRoot); + expect(result.code).toBe(EXIT_FAILURE); + expect(result.stderr).not.toContain("ana@example.com"); + expect(result.stderr).toContain("error: Error; its message may quote personal data"); + } finally { + delete process.env.REFUSE_MIGRATION; + } + }); + + it("exits 2 with the reason when the configuration is not there", async () => { + const root = await mkdtemp(join(tmpdir(), "bounda-cli-migrate-")); + temporary.push(root); + const result = await cli(["migrate"], root); + expect(result.code).toBe(EXIT_FAILURE); + expect(result.stderr).toMatch(/^error: Cannot find the configuration/); + }); + + it("documents its options in its help", async () => { + const help = await cli(["migrate", "--help"], repoRoot); + expect(help.code).toBe(EXIT_OK); + expect(help.stdout.replace(/\s+/g, " ")).toContain( + "bring Bounda's tables to this release's version in every database of the configuration, before a deploy", + ); + for (const option of ["--root ", "--config "]) { + expect(help.stdout).toContain(option); + } + }); +}); diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index aee44f57..acee39f3 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -9,7 +9,10 @@ import { type ListDeadLettersArgs, type Logger, MAX_ACTOR_LENGTH, + migrateTables, + type Registry, rebuildReadModel, + type SubscriberFailure, } from "@bounda-dev/core"; import type { Config, ErrorText } from "@bounda-dev/core/config"; import { boot, createConsoleLogger, loadProject } from "@bounda-dev/core/node"; @@ -20,6 +23,7 @@ import { generateProject } from "./generate/generate.ts"; import { ConventionError } from "./generate/problems.ts"; import { watchFromFirstRun } from "./generate/watch.ts"; import { formatConventionError, formatReport, formatWarnings } from "./reporter.ts"; +import { formatSchema, formatStatus } from "./status-output.ts"; export interface Output { write(text: string): void; @@ -356,6 +360,75 @@ const runForget = async ( }); }; +interface StatusOptions extends ActingOptions { + readonly json: boolean; +} + +// The text of each failure is opened only when asked for, through the call the access log records. +const runStatus = ( + options: StatusOptions, + cwd: string, + stdout: Output, + stderr: Output, +): Promise => + withApp(options, cwd, stderr, async (app) => { + const status = await app.status(); + const opened = options.errorText + ? await actingAs(app, options).subscriberFailures.list() + : undefined; + if (options.json) { + line( + stdout, + JSON.stringify( + opened === undefined ? status : { ...status, subscriberFailures: opened }, + null, + 2, + ), + ); + } else { + line( + stdout, + formatStatus({ + status, + ...(opened === undefined + ? {} + : { opened: new Map(opened.map((f) => [f.subscriber, f])) }), + }), + ); + } + return EXIT_OK; + }); + +// Only the configuration is read: the tables are migrated before any app of this release starts. +const NO_REGISTRY: Registry = { aggregates: {}, readModels: {} }; + +const runMigrate = async ( + options: ProjectOptions, + cwd: string, + stdout: Output, + stderr: Output, +): Promise => { + const logger = logTo(stderr); + let config: Config; + try { + ({ config } = await loadProject({ + root: rootOf(options, cwd), + configPath: options.config, + registry: NO_REGISTRY, + logger, + })); + } catch (error) { + line(stderr, `error: ${error instanceof Error ? error.message : String(error)}`); + return EXIT_FAILURE; + } + try { + line(stdout, `migrated\n${formatSchema(await migrateTables({ config }))}`); + return EXIT_OK; + } catch (error) { + return failed(stderr, error, errorTextOf(options, config)); + } +}; + // Records, so a kind or status added to core fails to compile here until the CLI accepts it. const DEAD_LETTER_KINDS = Object.keys({ policy: true, @@ -514,6 +587,28 @@ export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal exitCode = await runForget(options, cwd, stdout, stderr); }); + actingOptions( + program + .command("status") + .description( + "print the version of the tables, every subscriber's lag and last failure, and how the backfill, forget ledger, key rotation and retention stand", + ) + .option("--json", "print the status as JSON", false), + ).action(async (options: StatusOptions) => { + exitCode = await runStatus(options, cwd, stdout, stderr); + }); + + program + .command("migrate") + .description( + "bring Bounda's tables to this release's version in every database of the configuration, before a deploy", + ) + .option("--root ", "project root (default: current directory)") + .option("--config ", "configuration module under the root", "bounda.config.ts") + .action(async (options: ProjectOptions) => { + exitCode = await runMigrate({ ...options, errorText: false }, cwd, stdout, stderr); + }); + try { await program.parseAsync([...argv], { from: "user" }); return exitCode; diff --git a/packages/cli/src/status-output.test.ts b/packages/cli/src/status-output.test.ts new file mode 100644 index 00000000..e54e44d8 --- /dev/null +++ b/packages/cli/src/status-output.test.ts @@ -0,0 +1,127 @@ +import type { AppStatus, SubscriberFailure } from "@bounda-dev/core"; +import { describe, expect, it } from "vitest"; +import { formatSchema, formatStatus } from "./status-output.ts"; + +const failure = { + subscriber: "projection:orders", + position: 7, + eventId: "e-7", + eventType: "OrderPlaced", + person: "c-1", + errorName: "ValidationError", + errorCode: "VALIDATION_FAILED", + attempts: 3, + firstFailedAt: "2026-05-01T00:00:00.000Z", + lastFailedAt: "2026-05-01T00:01:00.000Z", + resolvedAt: null, +}; + +const status: AppStatus = { + schema: [ + { database: "storage", scope: "storage", version: 1, latest: 2 }, + { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, + ], + restoresFound: 2, + lag: { + lastPosition: 9, + maxLag: 3, + subscribers: [ + { subscriber: "projection:orders", position: 6, lag: 3 }, + { subscriber: "policies", position: 9, lag: 0 }, + { subscriber: "processes", position: 9, lag: 0 }, + ], + waitingDeadlines: 0, + forgetLedger: { reconciled: true }, + personalDataBackfill: { position: 9, caughtUp: false }, + }, + subscriberFailures: [ + { ...failure, current: true }, + { + ...failure, + subscriber: "policies", + errorCode: undefined, + errorName: "Error", + attempts: 1, + resolvedAt: "2026-05-02T00:00:00.000Z", + current: false, + }, + ], + personalDataBackfill: { position: 9, caughtUp: false }, + forgetLedger: { + reconciled: false, + appliedThrough: "2026-04-30T00:00:00.000Z", + lastRunAt: null, + held: { until: "2026-05-01T00:05:00.000Z", attempts: 2 }, + }, + keyRotation: { kid: "k-2", previous: ["k-1"], pending: 1, completeAt: null }, + retention: { retentionMs: 30 * 86_400_000, due: 1, nextDueAt: "2026-05-03T00:00:00.000Z" }, + deadLetterRetention: { retentionMs: 90_000, nextDueAt: null }, +}; + +describe("formatStatus", () => { + it("prints every section, the failures without their text, and what is pending", () => { + expect(formatStatus({ status }).split("\n")).toEqual([ + "tables", + " storage storage 1 of 2 (migrations pending: bounda migrate)", + " storage checkpoints 1 of 1", + "restores found while running: 2", + "subscribers (head at 9)", + " projection:orders at 6, 3 behind", + " failing: ValidationError (VALIDATION_FAILED) on OrderPlaced at 7, 3 attempts since 2026-05-01T00:00:00.000Z", + " policies at 9, 0 behind", + " last failed, resolved 2026-05-02T00:00:00.000Z: Error on OrderPlaced at 7, 1 attempt since 2026-05-01T00:00:00.000Z", + " processes at 9, 0 behind", + "personal data backfill: sealed through position 9, not caught up yet", + "forget ledger: not reconciled since this instance started, applied through 2026-04-30T00:00:00.000Z, last run never, held until 2026-05-01T00:05:00.000Z after 2 failures", + "key rotation: under k-2 (previous k-1), 1 key left to rewrap", + "retention: 30d, 1 person past it, next due 2026-05-03T00:00:00.000Z", + "dead letter retention: 90s, next purge when a letter is resolved", + ]); + }); + + it("prints the text of a failure when opened, and says when it went with its person", () => { + const opened = (errorMessage: string | null): ReadonlyMap => + new Map([["projection:orders", { ...failure, errorMessage }]]); + expect(formatStatus({ status, opened: opened("cannot place for ana") })).toContain( + " cannot place for ana\n", + ); + expect(formatStatus({ status, opened: opened(null) })).toContain( + " its text went with the person it was sealed under\n", + ); + }); + + it("says what is not configured, and a rotation that wrapped every key", () => { + const bare: AppStatus = { + ...status, + personalDataBackfill: null, + forgetLedger: { + reconciled: true, + appliedThrough: null, + lastRunAt: "2026-05-01T00:00:00.000Z", + held: null, + }, + keyRotation: { kid: "k-1", previous: [], pending: 0, completeAt: "2026-05-01T00:00:00.000Z" }, + retention: null, + deadLetterRetention: null, + }; + const lines = formatStatus({ status: bare }).split("\n"); + expect(lines.slice(-5)).toEqual([ + "personal data backfill: no personal data marked", + "forget ledger: reconciled, applied through nothing yet, last run 2026-05-01T00:00:00.000Z", + "key rotation: under k-1, every key wrapped (checked 2026-05-01T00:00:00.000Z)", + "retention: not configured", + "dead letter retention: not configured", + ]); + expect( + formatStatus({ + status: { ...bare, keyRotation: { ...bare.keyRotation, completeAt: null } as never }, + }), + ).toContain("key rotation: under k-1, every key wrapped\n"); + }); +}); + +describe("formatSchema", () => { + it("says so when no database has tables to version", () => { + expect(formatSchema([])).toBe(" no tables to version"); + }); +}); diff --git a/packages/cli/src/status-output.ts b/packages/cli/src/status-output.ts new file mode 100644 index 00000000..16f6ef53 --- /dev/null +++ b/packages/cli/src/status-output.ts @@ -0,0 +1,113 @@ +import type { AppStatus, SchemaStatus, SubscriberFailure } from "@bounda-dev/core"; + +export interface FormatStatusArgs { + readonly status: AppStatus; + /** + * Each subscriber's last failure with its text, by subscriber, when `--error-text` asked for it. + */ + readonly opened?: ReadonlyMap; +} + +export interface FormatStatusFunction { + (args: FormatStatusArgs): string; +} + +export interface FormatSchemaFunction { + (schema: readonly SchemaStatus[]): string; +} + +const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? "" : "s"}`; + +const days = (ms: number): string => + ms % 86_400_000 === 0 ? `${ms / 86_400_000}d` : `${Math.round(ms / 1_000)}s`; + +const identity = (errorName: string, errorCode: string | undefined): string => + `${errorName}${errorCode === undefined ? "" : ` (${errorCode})`}`; + +/** + * The version of every scope of Bounda's tables, by database, as `bounda status` and + * `bounda migrate` print it. + */ +export const formatSchema: FormatSchemaFunction = (schema) => + schema.length === 0 + ? " no tables to version" + : schema + .map( + ({ database, scope, version, latest }) => + ` ${database} ${scope} ${version} of ${latest}${version < latest ? " (migrations pending: bounda migrate)" : ""}`, + ) + .join("\n"); + +const subscribersOf = ({ status, opened }: FormatStatusArgs): string[] => + status.lag.subscribers.flatMap(({ subscriber, position, lag }) => { + const failure = status.subscriberFailures.find((each) => each.subscriber === subscriber); + const text = opened?.get(subscriber)?.errorMessage; + return [ + ` ${subscriber} at ${position}, ${lag} behind`, + ...(failure === undefined + ? [] + : [ + ` ${failure.current ? "failing" : `last failed, resolved ${failure.resolvedAt ?? "since"}`}: ${identity(failure.errorName, failure.errorCode)} on ${failure.eventType} at ${failure.position}, ${plural(failure.attempts, "attempt")} since ${failure.firstFailedAt}`, + ...(text === undefined + ? [] + : [ + ` ${text === null ? "its text went with the person it was sealed under" : text}`, + ]), + ]), + ]; + }); + +const backfillOf = ({ personalDataBackfill: backfill }: AppStatus): string => + backfill === null + ? "no personal data marked" + : `sealed through position ${backfill.position}, ${backfill.caughtUp ? "caught up" : "not caught up yet"}`; + +const ledgerOf = ({ forgetLedger: ledger }: AppStatus): string => { + if (ledger === null) return "not configured"; + return [ + ledger.reconciled ? "reconciled" : "not reconciled since this instance started", + `applied through ${ledger.appliedThrough ?? "nothing yet"}`, + `last run ${ledger.lastRunAt ?? "never"}`, + ...(ledger.held === null + ? [] + : [`held until ${ledger.held.until} after ${plural(ledger.held.attempts, "failure")}`]), + ].join(", "); +}; + +const rotationOf = ({ keyRotation: rotation }: AppStatus): string => { + if (rotation === null) return "not configured"; + const left = + rotation.pending === 0 + ? `every key wrapped${rotation.completeAt === null ? "" : ` (checked ${rotation.completeAt})`}` + : `${plural(rotation.pending, "key")} left to rewrap`; + return `under ${rotation.kid}${rotation.previous.length === 0 ? "" : ` (previous ${rotation.previous.join(", ")})`}, ${left}`; +}; + +const retentionOf = ({ retention }: AppStatus): string => + retention === null + ? "not configured" + : `${days(retention.retentionMs)}, ${plural(retention.due, "person")} past it, next due ${retention.nextDueAt ?? "never"}`; + +const purgeOf = ({ deadLetterRetention: purge }: AppStatus): string => + purge === null + ? "not configured" + : `${days(purge.retentionMs)}, next purge ${purge.nextDueAt ?? "when a letter is resolved"}`; + +/** + * Where the runtime's background work stands, as `bounda status` prints it. + */ +export const formatStatus: FormatStatusFunction = (args) => { + const { status } = args; + return [ + "tables", + formatSchema(status.schema), + `restores found while running: ${status.restoresFound}`, + `subscribers (head at ${status.lag.lastPosition})`, + ...subscribersOf(args), + `personal data backfill: ${backfillOf(status)}`, + `forget ledger: ${ledgerOf(status)}`, + `key rotation: ${rotationOf(status)}`, + `retention: ${retentionOf(status)}`, + `dead letter retention: ${purgeOf(status)}`, + ].join("\n"); +}; diff --git a/packages/core/src/adapter/testing/storage-schema.contract.ts b/packages/core/src/adapter/testing/storage-schema.contract.ts index c635cd82..c29b938f 100644 --- a/packages/core/src/adapter/testing/storage-schema.contract.ts +++ b/packages/core/src/adapter/testing/storage-schema.contract.ts @@ -143,6 +143,50 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ expect(await shapeOf(target, dialect)).toEqual(shape); }); + it("reads the version of every scope without migrating, and migrates only the scopes asked", async () => { + const target = await create(); + const versions = await target.adapter.schemaVersions(); + expect(versions).toEqual([ + { scope: "storage", version: 0, latest: 2 }, + { scope: "checkpoints", version: 0, latest: 1 }, + { scope: "forget-ledger", version: 0, latest: 1 }, + ]); + expect(await shapeOf(target, dialect)).toEqual([]); + expect(await target.adapter.migrate(["checkpoints"])).toEqual([ + { scope: "storage", version: 0, latest: 2 }, + { scope: "checkpoints", version: 1, latest: 1 }, + { scope: "forget-ledger", version: 0, latest: 1 }, + ]); + expect(await target.adapter.migrate(["storage", "forget-ledger"])).toEqual([ + { scope: "storage", version: 2, latest: 2 }, + { scope: "checkpoints", version: 1, latest: 1 }, + { scope: "forget-ledger", version: 1, latest: 1 }, + ]); + const shape = await shapeOf(target, dialect); + await opened(target.adapter, async () => undefined); + expect(await shapeOf(target, dialect)).toEqual(shape); + }); + + it("refuses to migrate a scope a newer version migrated, writing nothing", async () => { + const target = await create(); + const table = schemaMigrationsTableFor(target.prefix); + await target.execute(schemaMigrationsTableStatement(table)); + await target.execute( + `INSERT INTO ${table} ("scope", "version", "applied_at") VALUES ('checkpoints', 999, '2099-01-01T00:00:00.000Z')`, + ); + const shape = await shapeOf(target, dialect); + await expect(target.adapter.migrate(["storage", "checkpoints"])).rejects.toMatchObject({ + code: "STORAGE_SCHEMA_TOO_NEW", + scope: "checkpoints", + }); + expect(await shapeOf(target, dialect)).toEqual(shape); + expect(await target.adapter.schemaVersions()).toContainEqual({ + scope: "checkpoints", + version: 999, + latest: 1, + }); + }); + it.skipIf(!concurrent)("migrates once when instances start together", async () => { const target = await create(); const storages = await Promise.all([1, 2, 3].map(() => target.adapter.createStorage())); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6686d195..d73feb0b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -47,6 +47,8 @@ export type { SubscriberFailures, } from "./kernel/dispatch/subscriber-failures.ts"; export type { GlobalStream, ReadEventsArgs } from "./kernel/events/global-stream.ts"; +export type { MigrateTablesArgs, MigrateTablesFunction } from "./kernel/migrate.ts"; +export { migrateTables } from "./kernel/migrate.ts"; export type { ExportedData, ExportedDeadLetter, diff --git a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts index 07ea8c3a..4c118408 100644 --- a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts +++ b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts @@ -98,6 +98,22 @@ describe("the dispatcher's backoff", () => { expect(attemptsAt).toEqual([100, 1_100, 3_100, 7_100, 11_100]); }); + it("forgets every failure it met after a restore, so each is tried again at once", async () => { + const poisoned = scripted("projection:poisoned", [failed, failed, failed]); + const { clock, dispatcher } = await setUp([poisoned]); + await dispatcher.processOnce(); + await dispatcher.processOnce(); + expect(await failingOf(dispatcher, "projection:poisoned")).toMatchObject({ attempts: 2 }); + dispatcher.restored(); + expect(await failingOf(dispatcher, "projection:poisoned")).toBeUndefined(); + await dispatcher.catchUp("projection"); + expect(poisoned.deliveries).toBe(3); + expect(await failingOf(dispatcher, "projection:poisoned")).toMatchObject({ + attempts: 1, + since: clock.now().toISOString(), + }); + }); + it("says what a subscriber is stuck on, since when and when it is tried again, until it recovers", async () => { const poisoned = scripted("projection:poisoned", [failed, failed, { outcome: "advanced" }]); const { clock, dispatcher } = await setUp([poisoned]); diff --git a/packages/core/src/kernel/dispatch/last-failures.test.ts b/packages/core/src/kernel/dispatch/last-failures.test.ts new file mode 100644 index 00000000..cfa237b4 --- /dev/null +++ b/packages/core/src/kernel/dispatch/last-failures.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { testMetadata } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; +import { memory } from "../../memory/index.ts"; +import { buildAggregates } from "../aggregate/build-aggregates.ts"; +import { createPersonalKeyWrapping } from "../personal-data/wrapped-keys.ts"; +import { createRecordingLogger } from "../test-support.ts"; +import { createLastFailures } from "./last-failures.ts"; + +const event: StoredEvent = { + id: "e-1", + aggregateType: "order", + aggregateId: "o-1", + version: 1, + position: 4, + type: "OrderPlaced", + payload: {}, + timestamp: "2026-01-01T00:00:00.000Z", + metadata: testMetadata, + person: null, +}; + +const setUp = async () => { + const storage = await memory().createStorage(); + const { logger, entries } = createRecordingLogger("omit"); + const keys = createPersonalKeyWrapping({ + wrapper: null, + clock: createFixedClock(), + }).over(storage.personalKeys); + const failing = { + ...storage.subscriberFailures, + record: () => Promise.reject(new Error("store is down")), + resolve: () => Promise.reject(new Error("store is down")), + }; + const lastFailures = createLastFailures({ + store: failing, + aggregates: buildAggregates({ registry: { aggregates: {}, readModels: {} }, ports: {} }), + keys, + logger, + }); + return { lastFailures, entries }; +}; + +describe("createLastFailures", () => { + it("logs a failure it could not record or resolve, and never rejects", async () => { + const { lastFailures, entries } = await setUp(); + const at = new Date("2026-01-01T00:00:00.000Z"); + await lastFailures.record({ subscriber: "projection:x", event, error: new Error("boom"), at }); + await lastFailures.resolve("projection:x", at); + expect(entries).toEqual([ + { + level: "error", + message: "subscriber failure could not be recorded", + fields: expect.objectContaining({ subscriber: "projection:x", errorName: "Error" }), + }, + { + level: "error", + message: "subscriber failure could not be marked resolved", + fields: expect.objectContaining({ subscriber: "projection:x", errorName: "Error" }), + }, + ]); + }); +}); diff --git a/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts new file mode 100644 index 00000000..d2a3abfc --- /dev/null +++ b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts @@ -0,0 +1,351 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { Adapter } from "../../adapter/adapter.ts"; +import { createNodeSqliteAdapter } from "../../adapter/sqlite/node-sqlite.ts"; +import type { Table } from "../../adapter/storage/table.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; +import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import type { LogFields, Logger } from "../../contracts/logger.ts"; +import { memory } from "../../memory/index.ts"; +import type { PersonalDataAccess } from "../../modules/audit.ts"; +import type { PersonalDataForgottenEvent } from "../../modules/bounda-events.ts"; +import type { EventPayloadArgs, PayloadArgs } from "../../modules/payload.ts"; +import type { Registry } from "../../modules/registry.ts"; +import type { FieldsArgs } from "../../modules/view.ts"; +import { type BoundaApp, createApp } from "../app.ts"; + +// The projection quotes the personal field it read, as a careless one does, while `failing.on`. +const failing = { on: false }; + +interface CustomerRow { + readonly customerId: string; + readonly email: string; +} + +const registry = { + aggregates: { + customer: { + state: { initialState: {} }, + events: { + customerRegistered: { + payload: ({ z, personal }: EventPayloadArgs) => z.object({ email: personal(z.string()) }), + evolve: () => ({}), + }, + customerNoted: { + payload: ({ z }: EventPayloadArgs) => z.object({ note: z.string() }), + evolve: () => ({}), + }, + }, + commands: { + registerCustomer: { + module: { + payload: ({ z }: PayloadArgs) => + z.object({ customerId: z.string(), email: z.string() }), + handler: ({ + command, + events, + }: { + command: { payload: { email: string } }; + events: Record unknown>; + }) => [events.customerRegistered?.({ email: command.payload.email })], + }, + }, + noteCustomer: { + module: { + payload: ({ z }: PayloadArgs) => z.object({ customerId: z.string(), note: z.string() }), + handler: ({ + command, + events, + }: { + command: { payload: { note: string } }; + events: Record unknown>; + }) => [events.customerNoted?.({ note: command.payload.note })], + }, + }, + }, + policies: {}, + processes: {}, + }, + }, + readModels: { + customers: { + view: { + fields: ({ f }: FieldsArgs) => ({ + customerId: f.string().primaryKey(), + email: f.string(), + }), + }, + projections: { + customer: { + customerRegistered: { + project: async ({ + event, + table, + }: { + event: StoredEvent<"CustomerRegistered", { email: string | null }>; + table: Table; + }) => { + if (failing.on) throw new Error(`cannot project ${event.payload.email ?? "nobody"}`); + await table.upsert({ + customerId: event.aggregateId, + email: event.payload.email ?? "", + }); + }, + }, + customerNoted: { + project: async ({ + event, + }: { + event: StoredEvent<"CustomerNoted", { note: string }>; + }) => { + if (failing.on) throw new Error(`cannot note ${event.payload.note}`); + }, + }, + }, + bounda: { + personalDataForgotten: { + project: async ({ + event, + table, + }: { + event: PersonalDataForgottenEvent; + table: Table; + }) => { + await table.delete({ customerId: event.payload.personId }); + }, + }, + }, + }, + queries: {}, + }, + }, +} as unknown as Registry; + +const SUBSCRIBER = "projection:customers"; + +interface Logged { + readonly level: string; + readonly message: string; + readonly fields?: LogFields | undefined; +} + +const recordingLogger = (): Logger & { readonly logged: Logged[] } => { + const logged: Logged[] = []; + const at = (level: string) => (message: string, fields?: LogFields) => { + logged.push({ level, message, fields }); + }; + return { logged, debug: at("debug"), info: at("info"), warn: at("warn"), error: at("error") }; +}; + +const stops: (() => Promise)[] = []; + +afterEach(async () => { + failing.on = false; + for (const stop of stops.splice(0)) await stop(); +}); + +let started = 0; + +const start = async (adapter: Adapter) => { + started += 1; + const logger = recordingLogger(); + const app = await createApp({ + registry, + config: { storage: adapter }, + ids: createSequentialIdGenerator({ prefix: `app-${started}` }), + clock: createFixedClock(new Date("2026-05-01T00:00:00.000Z")), + logger, + }); + stops.push(() => app.stop()); + return { app, logger }; +}; + +const register = (app: BoundaApp, customerId: string) => + app.commands.registerCustomer?.({ customerId, email: `${customerId}@example.com` }); + +const accesses = (logger: { readonly logged: Logged[] }): PersonalDataAccess[] => + logger.logged + .filter(({ message }) => message === "personal data accessed") + .map(({ fields }) => fields as unknown as PersonalDataAccess); + +describe.each([ + ["memory", () => memory()], + ["SQLite", () => createNodeSqliteAdapter().adapter], +])("the last failure of each subscriber, on %s", (_name, adapterOf) => { + it("keeps the text of a projection's failure sealed under the event's person, out of the logs", async () => { + const adapter = adapterOf(); + const { app, logger } = await start(adapter); + failing.on = true; + await register(app, "c-1"); + await app.runUntilIdle({ maxPasses: 3 }); + const stored = await (await adapter.createStorage()).subscriberFailures.get(SUBSCRIBER); + expect(stored).toMatchObject({ + eventType: "CustomerRegistered", + person: "c-1", + errorName: "Error", + errorMessage: { $personal: { person: "c-1" } }, + resolvedAt: null, + }); + expect(JSON.stringify(stored)).not.toContain("c-1@example.com"); + expect(JSON.stringify(logger.logged)).not.toContain("c-1@example.com"); + + const status = await app.status(); + expect(status.subscriberFailures).toEqual([ + expect.objectContaining({ subscriber: SUBSCRIBER, person: "c-1", current: true }), + ]); + expect(JSON.stringify(status)).not.toContain("c-1@example.com"); + expect(accesses(logger)).toEqual([]); + + const [opened] = await app.as("operator-1").subscriberFailures.list(); + expect(opened).toMatchObject({ + subscriber: SUBSCRIBER, + errorMessage: "cannot project c-1@example.com", + errorStack: expect.stringContaining("cannot project c-1@example.com"), + }); + expect(accesses(logger)).toEqual([ + expect.objectContaining({ + operation: "subscriberFailures.list", + actor: "operator-1", + persons: ["c-1"], + detail: { failures: 1 }, + outcome: "ok", + }), + ]); + }); + + it("counts the failures on one event, and resolves the failure once the subscriber gets past it", async () => { + const adapter = adapterOf(); + const { app, logger } = await start(adapter); + failing.on = true; + await register(app, "c-1"); + await app.runUntilIdle({ maxPasses: 3 }); + const [stuck] = (await app.status()).subscriberFailures; + expect(stuck?.attempts).toBeGreaterThan(1); + failing.on = false; + await app.runUntilIdle(); + expect((await app.status()).subscriberFailures).toEqual([ + expect.objectContaining({ current: false, resolvedAt: "2026-05-01T00:00:00.000Z" }), + ]); + expect(await app.subscriberFailures.get(SUBSCRIBER)).toMatchObject({ + errorMessage: "cannot project c-1@example.com", + }); + expect(accesses(logger).at(-1)).toMatchObject({ + operation: "subscriberFailures.get", + detail: { subscriber: SUBSCRIBER }, + persons: ["c-1"], + }); + expect(await app.subscriberFailures.get("projection:nothing")).toBeNull(); + expect(accesses(logger)).toHaveLength(1); + }); + + it("reads the text as null once the person is forgotten, and hands it to their export", async () => { + const adapter = adapterOf(); + const { app } = await start(adapter); + failing.on = true; + await register(app, "c-1"); + await app.runUntilIdle({ maxPasses: 3 }); + expect((await app.exportPersonalData("c-1")).subscriberFailures).toEqual([ + { + subscriber: SUBSCRIBER, + eventId: expect.any(String), + eventType: "CustomerRegistered", + errorMessage: "cannot project c-1@example.com", + errorStack: expect.stringContaining("cannot project"), + }, + ]); + expect((await app.exportPersonalData("c-2")).subscriberFailures).toEqual([]); + await app.forget("c-1"); + expect(await app.subscriberFailures.get(SUBSCRIBER)).toMatchObject({ + person: "c-1", + errorMessage: null, + errorStack: null, + }); + }); + + it("keeps in the clear the text of a failure on an event that names nobody", async () => { + const adapter = adapterOf(); + const { app } = await start(adapter); + failing.on = true; + await app.commands.noteCustomer?.({ customerId: "c-1", note: "late" }); + await app.runUntilIdle({ maxPasses: 3 }); + expect(await (await adapter.createStorage()).subscriberFailures.get(SUBSCRIBER)).toMatchObject({ + person: null, + errorMessage: "cannot note late", + }); + }); + + it("is resolved by an instance that gets the subscriber past it, though another met it", async () => { + const adapter = adapterOf(); + const first = await start(adapter); + failing.on = true; + await register(first.app, "c-1"); + await first.app.runUntilIdle({ maxPasses: 3 }); + await first.app.stop(); + failing.on = false; + const second = await start(adapter); + await second.app.runUntilIdle(); + expect(await (await adapter.createStorage()).subscriberFailures.get(SUBSCRIBER)).toMatchObject({ + resolvedAt: "2026-05-01T00:00:00.000Z", + }); + }); +}); + +describe("the status of the app", () => { + it("names the version of every scope where it lives, and nothing of a store without tables", async () => { + const { adapter } = createNodeSqliteAdapter(); + const { app } = await start(adapter); + expect((await app.status()).schema).toEqual([ + { database: "storage", scope: "storage", version: 2, latest: 2 }, + { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, + ]); + const inMemory = await start(memory()); + expect(await inMemory.app.status()).toMatchObject({ + schema: [], + restoresFound: 0, + subscriberFailures: [], + personalDataBackfill: { caughtUp: false }, + forgetLedger: null, + keyRotation: null, + retention: null, + deadLetterRetention: null, + }); + }); + + it("names the read models' database and the ledger's apart from the storage", async () => { + // Three databases: the same path is taken for the same database. + const directory = mkdtempSync(join(tmpdir(), "bounda-status-")); + const storage = createNodeSqliteAdapter(join(directory, "storage.db")).adapter; + const readModels = createNodeSqliteAdapter(join(directory, "read-models.db")).adapter; + const ledger = createNodeSqliteAdapter(join(directory, "ledger.db")).adapter; + const app = await createApp({ + registry, + config: { + storage, + readModels: { customers: readModels }, + runtime: { + personalData: { forgetLedger: { store: ledger }, retention: "30d" }, + deadLetters: { retention: "7d" }, + }, + }, + clock: createFixedClock(new Date("2026-05-01T00:00:00.000Z")), + }); + stops.push(() => app.stop()); + await app.forget("c-0"); + await app.runUntilIdle(); + const status = await app.status(); + expect(status.schema).toEqual([ + { database: "storage", scope: "storage", version: 2, latest: 2 }, + { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, + { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, + { database: "forgetLedger", scope: "forget-ledger", version: 1, latest: 1 }, + ]); + expect(status).toMatchObject({ + forgetLedger: { reconciled: true, held: null }, + retention: { due: 0, nextDueAt: null }, + deadLetterRetention: { nextDueAt: null }, + }); + }); +}); diff --git a/packages/core/src/kernel/migrate.test.ts b/packages/core/src/kernel/migrate.test.ts new file mode 100644 index 00000000..b6a34025 --- /dev/null +++ b/packages/core/src/kernel/migrate.test.ts @@ -0,0 +1,49 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createNodeSqliteAdapter } from "../adapter/sqlite/node-sqlite.ts"; +import { memory } from "../memory/index.ts"; +import { migrateTables } from "./migrate.ts"; + +const tablesIn = (db: ReturnType["db"]): string[] => + ( + db.prepare(`SELECT "name" FROM sqlite_master WHERE "type" = 'table' ORDER BY "name"`).all() as { + name: string; + }[] + ).map(({ name }) => name); + +describe("migrateTables", () => { + it("migrates every database the configuration names to this release, each with its own scopes", async () => { + const directory = mkdtempSync(join(tmpdir(), "bounda-migrate-")); + const storage = createNodeSqliteAdapter(join(directory, "storage.db")); + const readModels = createNodeSqliteAdapter(join(directory, "read-models.db")); + const ledger = createNodeSqliteAdapter(join(directory, "ledger.db")); + const config = { + storage: storage.adapter, + readModels: { customers: readModels.adapter, orders: storage.adapter }, + runtime: { personalData: { forgetLedger: { store: ledger.adapter } } }, + }; + expect(await migrateTables({ config })).toEqual([ + { database: "storage", scope: "storage", version: 2, latest: 2 }, + { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, + { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, + { database: "forgetLedger", scope: "forget-ledger", version: 1, latest: 1 }, + ]); + expect(tablesIn(readModels.db)).toEqual(["bounda_checkpoints", "bounda_migrations"]); + expect(tablesIn(ledger.db)).toEqual(["bounda_forget_ledger", "bounda_migrations"]); + expect(tablesIn(storage.db)).toContain("bounda_subscriber_failures"); + expect(tablesIn(storage.db)).not.toContain("bounda_forget_ledger"); + expect(await migrateTables({ config })).toHaveLength(4); + }); + + it("has nothing to migrate in a store without tables", async () => { + expect(await migrateTables({ config: { storage: memory() } })).toEqual([]); + }); + + it("refuses a configuration it cannot read before migrating anything", async () => { + await expect( + migrateTables({ config: { storage: memory(), runtime: { role: "nobody" } } as never }), + ).rejects.toMatchObject({ code: "INVALID_CONFIGURATION" }); + }); +}); diff --git a/packages/core/src/kernel/migrate.ts b/packages/core/src/kernel/migrate.ts new file mode 100644 index 00000000..1fa9fb5d --- /dev/null +++ b/packages/core/src/kernel/migrate.ts @@ -0,0 +1,28 @@ +import { resolveConfig } from "../config/schema.ts"; +import type { Config } from "../config/types.ts"; +import { databasesOf, type SchemaStatus, schemaOf } from "./status.ts"; + +export interface MigrateTablesArgs { + readonly config: Config; +} + +export interface MigrateTablesFunction { + (args: MigrateTablesArgs): Promise; +} + +/** + * Brings Bounda's tables to the latest version of this release in every database the + * configuration names, as an app starting there would, and resolves to the versions after, by + * database: what `bounda migrate` runs before a deploy, so the instances that start find the + * tables migrated. Each database migrates under its own lock, one after the other; one that + * fails stops the rest. Rejects with `StorageSchemaTooNewError` for tables a newer release + * migrated, writing nothing to that database. An adapter without tables, as `memory()`, has + * nothing to migrate. + */ +export const migrateTables: MigrateTablesFunction = async ({ config }) => { + const migrated: SchemaStatus[] = []; + for (const database of databasesOf(resolveConfig(config))) { + migrated.push(...schemaOf(database, await database.adapter.migrate(database.scopes))); + } + return migrated; +}; diff --git a/packages/core/src/kernel/personal-data/backfill-walk.test.ts b/packages/core/src/kernel/personal-data/backfill-walk.test.ts index f63a8c78..8c642a48 100644 --- a/packages/core/src/kernel/personal-data/backfill-walk.test.ts +++ b/packages/core/src/kernel/personal-data/backfill-walk.test.ts @@ -614,4 +614,46 @@ describe("the backfill's walk", () => { await later.backfill.sweep(new Date(START.getTime() + 5_000)); expect((await rows(storage))["m1:started"]).toBe(START.getTime()); }); + + it("starts over at once after a restore, clearing the copy's hold on the instance that found it first", async () => { + const storage = await memory().createStorage(); + const { logs, logger } = recording(); + const one = walker({ storage, logger }); + await one.backfill.sweep(START); + expect(await one.backfill.nextDueAt()).toBeNull(); + await storage.checkpointStore.set(`${BACKFILL_PREFIX}m1:attempts`, 3); + await storage.checkpointStore.set(`${BACKFILL_PREFIX}m1:held-until`, START.getTime() + HOLD_MS); + await storage.checkpointStore.remove(`${BACKFILL_PREFIX}m1:stores`); + await one.backfill.restored(false); + expect(await rows(storage)).toMatchObject({ "m1:attempts": 3 }); + expect(await one.backfill.nextDueAt()).toEqual(START); + await one.backfill.sweep(START); + expect(await one.backfill.status()).toMatchObject({ caughtUp: false }); + await one.backfill.restored(true); + const left = await rows(storage); + expect(left).not.toHaveProperty("m1:attempts"); + expect(left).not.toHaveProperty("m1:held-until"); + expect(await one.backfill.nextDueAt()).toEqual(START); + await one.backfill.sweep(START); + expect(await one.backfill.status()).toMatchObject({ caughtUp: true }); + expect(logs.filter(([level]) => level === "error")).toEqual([]); + }); + + it("logs a hold it could not clear after a restore", async () => { + const base = await memory().createStorage(); + const storage: Storage = { + ...base, + checkpointStore: { + ...base.checkpointStore, + get: () => Promise.reject(new Error("store is down")), + }, + }; + const { logs, logger } = recording(); + await walker({ storage, logger }).backfill.restored(true); + expect(logs).toContainEqual([ + "error", + "personal data backfill's hold could not be cleared", + expect.objectContaining({ error: expect.objectContaining({ message: "store is down" }) }), + ]); + }); }); diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index c37298a8..b0eec13f 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -607,3 +607,74 @@ describe("forgetAgain", () => { expect(transactions).toBe(1); }); }); + +describe("the reconciliation's status and restores", () => { + it("says where it stands from the database's rows, and that it read the ledger through", async () => { + const { storage, reconciliation } = await setUp({ entries: [entry("c-1", "e-1", at(-5))] }); + expect(await reconciliation.status()).toEqual({ + reconciled: false, + appliedThrough: null, + lastRunAt: null, + held: null, + }); + await reconciliation.reconcile(NOW); + expect(await reconciliation.status()).toEqual({ + reconciled: true, + appliedThrough: at(-5), + lastRunAt: NOW.toISOString(), + held: null, + }); + await storage.checkpointStore.set(FORGET_LEDGER_ATTEMPTS, 2); + await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 60_000); + expect((await reconciliation.status()).held).toEqual({ + until: new Date(NOW.getTime() + 60_000).toISOString(), + attempts: 2, + }); + }); + + it("is due at once after a restore, and clears the copy's hold on the instance that found it first", async () => { + const { storage, reconciliation } = await setUp({ entries: [] }); + await reconciliation.reconcile(NOW); + expect(reconciliation.nextDueAt()).toBeNull(); + await storage.checkpointStore.set(FORGET_LEDGER_ATTEMPTS, 4); + await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 3_600_000); + await reconciliation.restored(false); + expect(reconciliation.nextDueAt()).toEqual(NOW); + expect(reconciliation.reconciled()).toBe(false); + expect(await storage.checkpointStore.get(FORGET_LEDGER_ATTEMPTS)).toBe(4); + await reconciliation.restored(true); + expect(await storage.checkpointStore.get(FORGET_LEDGER_ATTEMPTS)).toBe(0); + expect(await storage.checkpointStore.get(FORGET_LEDGER_HELD_UNTIL)).toBe(0); + expect(await reconciliation.sweep(NOW)).toBe(0); + expect(reconciliation.reconciled()).toBe(true); + }); + + it("logs a hold it could not clear after a restore, and is due all the same", async () => { + const storage = await memory().createStorage(); + const logged: unknown[][] = []; + const reconciliation = createForgetLedgerReconciliation({ + ledger: recordingLedger([]).ledger, + storage: { + ...storage, + checkpointStore: { + ...storage.checkpointStore, + get: () => Promise.reject(new Error("down")), + }, + }, + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + logger: { ...silentLogger, error: (...args: unknown[]) => logged.push(args) }, + concurrencyRetries: 3, + reconcileEveryMs: EVERY, + keepForMs: null, + }); + await reconciliation.restored(true); + expect(reconciliation.nextDueAt()).toEqual(NOW); + expect(logged).toEqual([ + [ + "forget ledger's reconciliation hold could not be cleared", + { error: expect.objectContaining({ message: "down" }) }, + ], + ]); + }); +}); diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index 876085b5..e1de476c 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { DatabaseSync, SQLInputValue } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; import { createNodeSqliteAdapter } from "../../adapter/sqlite/node-sqlite.ts"; @@ -224,4 +227,32 @@ describe("a database restored while the app runs", () => { .map((event) => (event.payload as PersonalDataForgottenPayload).reason); expect(reasons).toEqual(["restored"]); }); + + it("tells every running instance, the first that found it once", async () => { + const file = join(mkdtempSync(join(tmpdir(), "bounda-restore-")), "store.db"); + const one = createNodeSqliteAdapter(file); + const other = createNodeSqliteAdapter(file); + const firstLogger = recordingLogger(); + const otherLogger = recordingLogger(); + const first = await start({ adapter: one.adapter, logger: firstLogger }); + const second = await start({ adapter: other.adapter, logger: otherLogger }); + await first.app.runUntilIdle(); + await second.app.runUntilIdle(); + const copy = copyOf(one.db); + first.clock.advance(GENERATION_INTERVAL_MS); + second.clock.advance(GENERATION_INTERVAL_MS); + await first.app.runUntilIdle(); + await second.app.runUntilIdle(); + restore(one.db, copy); + await first.app.runUntilIdle(); + await second.app.runUntilIdle(); + const restarts = (logger: { readonly logged: Logged[] }) => + logger.logged.filter( + ({ message }) => + message === "the database went back to an earlier copy; background jobs start over", + ); + expect(restarts(firstLogger).map(({ fields }) => fields)).toEqual([{ first: true }]); + expect(restarts(otherLogger).map(({ fields }) => fields)).toEqual([{ first: false }]); + expect((await second.app.status()).restoresFound).toBe(1); + }); }); diff --git a/packages/core/src/kernel/status.ts b/packages/core/src/kernel/status.ts index 3109c430..fe4b91c7 100644 --- a/packages/core/src/kernel/status.ts +++ b/packages/core/src/kernel/status.ts @@ -87,14 +87,18 @@ export interface ReadAppStatusFunction { (args: ReadAppStatusArgs): Promise; } -interface Database { +export interface Database { readonly database: string; readonly adapter: Adapter; readonly scopes: readonly SchemaScopeName[]; } +export interface DatabasesOfFunction { + (config: ResolvedConfig): readonly Database[]; +} + // Each database the configuration names, once, with the scopes it keeps there. -const databasesOf = (config: ResolvedConfig): readonly Database[] => { +export const databasesOf: DatabasesOfFunction = (config) => { const databases: Database[] = []; if (isAdapter(config.storage)) { databases.push({ @@ -114,8 +118,13 @@ const databasesOf = (config: ResolvedConfig): readonly Database[] => { return databases; }; -const schemaOf = async ({ database, adapter, scopes }: Database): Promise => - (await adapter.schemaVersions()) +export interface SchemaOfFunction { + (database: Database, versions: readonly SchemaVersion[]): SchemaStatus[]; +} + +// The versions of the scopes the database keeps, named after it. +export const schemaOf: SchemaOfFunction = ({ database, scopes }, versions) => + versions .filter(({ scope }) => (scopes as readonly string[]).includes(scope)) .map((version) => ({ database, ...version })); @@ -132,7 +141,11 @@ export const readAppStatus: ReadAppStatusFunction = async ({ }) => { const [schema, restoresFound, current, stored, forgetLedger, keyRotation, retained, purged] = await Promise.all([ - Promise.all(databasesOf(config).map(schemaOf)).then((all) => all.flat()), + Promise.all( + databasesOf(config).map(async (database) => + schemaOf(database, await database.adapter.schemaVersions()), + ), + ).then((all) => all.flat()), restores.found(), lag(), failures.list(), From 1a910e1f5998c0dcdac1c6053c99a09ab825752b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:35:48 +0100 Subject: [PATCH 05/19] docs: operations of the background jobs, the status and migrations, and abandoned processes --- .changeset/personal-data-operations.md | 16 ++++ docs/src/content/docs/adapters/cloudflare.md | 6 +- docs/src/content/docs/adapters/index.mdx | 3 +- .../docs/concepts/when-a-process-fails.md | 3 +- docs/src/content/docs/guides/dead-letters.md | 13 ++- docs/src/content/docs/guides/deployment.mdx | 38 ++++++-- docs/src/content/docs/guides/personal-data.md | 63 ++++++++++--- docs/src/content/docs/reference/cli.md | 94 ++++++++++++++++++- docs/src/content/docs/reference/errors.md | 2 +- .../src/content/docs/reference/limitations.md | 10 +- .../content/docs/reference/observability.md | 11 ++- 11 files changed, 224 insertions(+), 35 deletions(-) create mode 100644 .changeset/personal-data-operations.md diff --git a/.changeset/personal-data-operations.md b/.changeset/personal-data-operations.md new file mode 100644 index 00000000..12832d9e --- /dev/null +++ b/.changeset/personal-data-operations.md @@ -0,0 +1,16 @@ +--- +"@bounda-dev/core": minor +"@bounda-dev/cli": minor +"@bounda-dev/postgresql": minor +"@bounda-dev/cloudflare": minor +"@bounda-dev/react-router": minor +--- + +Operate the runtime's background work as one: share a lease, start over after a restore, and say where it stands. + +- The backfill, the key rotation, the retention and the purge of resolved dead letters each run in one instance at a time, under a lease kept in `bounda_checkpoints`; an instance that stalls past it stops at its next item. +- A database restored under the running app is found by every instance on its next pass: the forget ledger is reconciled at once, the jobs forget what they knew, and the first instance clears the holds the copy brought back. `PersonStore.releaseHolds`/`countIdle` and `PersonalKeyStore.releaseHolds`/`countWrappedUnder` are new (breaking for custom adapters). +- Discarding a process dead letter ends its instance with `ProcessAbandoned` in the same transaction; `ProcessStatus` gains `"abandoned"`, and the purge of resolved letters no longer decides whether an instance was given up. +- Each subscriber's last failure is kept in `bounda_subscriber_failures` (storage migration 2), its text sealed under the person the event names whatever `runtime.telemetry.errorText` says. `app.subscriberFailures.list()`/`.get()` open it and are recorded in the access log (`subscriberFailures.list`, `subscriberFailures.get`); the export includes it as `subscriberFailures`. `Storage.subscriberFailures` is new (breaking for custom adapters), and `Delivery` failures carry the event and the error. +- `app.status()`, also on `connect` clients, reports the tables' versions by database, the restores found, every subscriber's lag and last failure without its text, and the backfill, ledger, rotation, retention and purge. `Adapter.schemaVersions()` and `Adapter.migrate(scopes)` are new (breaking for custom adapters), and `migrateTables({ config })` migrates every database a configuration names. +- `bounda status` prints it, with `--json`, and `--error-text` the failures' text through the recorded call; `bounda migrate` migrates the tables before a deploy, reading only the configuration. diff --git a/docs/src/content/docs/adapters/cloudflare.md b/docs/src/content/docs/adapters/cloudflare.md index 5f6899f1..5ca1e83c 100644 --- a/docs/src/content/docs/adapters/cloudflare.md +++ b/docs/src/content/docs/adapters/cloudflare.md @@ -204,8 +204,10 @@ export default { `connect(stub)` gives a `BoundaClient`, the type `@bounda-dev/core` gives to what a request sees of an app wherever it runs: the same `commands` and `queries` as `app.commands` and -`app.queries`, typed from your modules, plus `getLag()`, `events`, `deadLetters`, -`rebuildReadModel`, `forget`, `exportPersonalData`, `restrict` and `unrestrict`. +`app.queries`, typed from your modules, plus `getLag()`, `status()`, `events`, `deadLetters`, +`subscriberFailures`, `rebuildReadModel`, `forget`, `exportPersonalData`, `restrict` and +`unrestrict`. `bounda status` runs where Node can open the store, so a Durable Object's is read +through `status()`, from an admin route of the Worker, say. Its commands resolve once the read models reflect them; `connect(stub, { consistency: "eventual" })` resolves them once their events are stored, for a request that reads nothing after it. A refusal comes back as an `Error` with the same `name`, `message`, `code` and, for validation, `issues`, whatever diff --git a/docs/src/content/docs/adapters/index.mdx b/docs/src/content/docs/adapters/index.mdx index 98b76a4b..f172bca3 100644 --- a/docs/src/content/docs/adapters/index.mdx +++ b/docs/src/content/docs/adapters/index.mdx @@ -86,7 +86,7 @@ it brings them up to the version the installed Bounda expects | Table | Holds | | --- | --- | | `bounda_events` | Every event, with its stream version, a global position and the person it names, if any, which [`exportPersonalData`](/guides/personal-data/#exporting-a-persons-data) reads by | -| `bounda_checkpoints` | How far each projection, the policy runner, the process runner, the retention and the [forget ledger](/guides/personal-data/#backups)'s copy have read, where a paused rebuild stands, the time of the last ledger entry the database applied, when the last reconciliation ran and the wait of one that failed, and when the [key rotation](/guides/personal-data/#rotating-the-master-key) was last found complete for a set of master keys | +| `bounda_checkpoints` | How far each projection, the policy runner, the process runner, the retention and the [forget ledger](/guides/personal-data/#backups)'s copy have read, where a paused rebuild stands, the time of the last ledger entry the database applied, when the last reconciliation ran and the wait of one that failed, when the [key rotation](/guides/personal-data/#rotating-the-master-key) was last found complete for a set of master keys, the leases of the [background jobs](/guides/deployment/#more-than-one-instance), and the counters that tell a [restore under the running app](/guides/personal-data/#restores-while-running) | | `bounda_inbox` | Which handler already completed for which event, so a retry does not run it again | | `bounda_scheduled_commands` | Scheduled commands, process deadlines and timeouts, and delayed policy runs | | `bounda_dead_letters` | Handler runs that gave up, with the error and the attempt count | @@ -94,6 +94,7 @@ it brings them up to the version the installed Bounda expects | `bounda_personal_keys` | One key per person, which their [personal fields](/guides/personal-data/) are encrypted with, wrapped under the [master key](/guides/personal-data/#a-master-key) whose id it keeps, and the key's fingerprint, and the wait of a rewrap that failed; `forget` deletes it | | `bounda_persons` | One row per person the runtime knows something about: since when their [processing is restricted](/guides/personal-data/#restricting-processing), `NULL` while it is not, and, with a [retention](/guides/personal-data/#retention), when they were last active and last forgotten, the position of the last event recorded for them, and the wait of a forget the retention could not do | | `bounda_snapshots` | The last snapshot of every stream of an aggregate that takes them, with the fingerprint it is valid for | +| `bounda_subscriber_failures` | The last failure of each subscriber, with its text encrypted under the person the event names, and whether the subscriber got past it ([observability](/reference/observability/#error-text)) | | `bounda_migrations` | The version of each set of tables Bounda versions, `storage`, `checkpoints` and `forget-ledger`, one row per migration with when it ran; a forget ledger or read models in a database of their own keep theirs there | Read models get one table each, named after the read model in snake_case behind `rm_`, so no diff --git a/docs/src/content/docs/concepts/when-a-process-fails.md b/docs/src/content/docs/concepts/when-a-process-fails.md index b0eb303c..8cca2b42 100644 --- a/docs/src/content/docs/concepts/when-a-process-fails.md +++ b/docs/src/content/docs/concepts/when-a-process-fails.md @@ -52,7 +52,8 @@ never changes, so the instance is a stable sequence, and parking can follow it. So when a step fails for good, the instance parks every later event in its own stream, in the order it arrived, and the process runner moves on to the other instances. Retrying the dead letter runs the failed handler again and drains what is parked, in order, before the instance resumes; -discarding it gives the instance up. [A failed process](/guides/dead-letters/#a-failed-process) has +discarding it gives the instance up with `ProcessAbandoned`, which ends it as `abandoned`, a status +of its own, so the purge of resolved letters never decides what happens to it. [A failed process](/guides/dead-letters/#a-failed-process) has the rules: what is parked, how deadlines interleave, and what a retry cut short does. ## Why not park them as dead letters diff --git a/docs/src/content/docs/guides/dead-letters.md b/docs/src/content/docs/guides/dead-letters.md index 9c129e47..73e4609a 100644 --- a/docs/src/content/docs/guides/dead-letters.md +++ b/docs/src/content/docs/guides/dead-letters.md @@ -76,8 +76,10 @@ export default defineConfig({ ``` A host without a polling worker, such as a Durable Object, is woken for it through `nextDueAt()`. -A process instance whose letter was discarded stays given up once the letter is gone: the events -that reach it are still dropped. +One instance purges at a time, under a lease the others leave alone until it lapses. A process +instance whose letter was discarded stays given up once the letter is gone: discarding wrote +`ProcessAbandoned` in its stream, so the events that reach it afterwards are dropped whatever +became of the letter. ## What a letter holds @@ -139,8 +141,11 @@ process completes it, and what is parked after it is dropped, as for any complet `bounda dead-letters list` says how many events wait behind a failure (`3 events are parked behind it`), and so does `parked` on the letters of `app.deadLetters`. Discarding the letter gives the -instance up: it stays failed, its parked events never run, though they stay in its history, and -the events that reach it afterwards are dropped, as for an instance that has ended. +instance up: in the same transaction as the discard, `ProcessAbandoned` (with the `letterId`, acted +by whoever discards) ends it with the status `abandoned`. Its parked events never run, though they +stay in its history, and the events that reach it afterwards are dropped, as for an instance that +has ended. Only a discard does that: an instance whose letter is gone for any other reason keeps +parking until its failure is retried. The failure is state as well: `ProcessFailed` names its dead letter (`letterId`), and the two are written in the same transaction, with the claim of the event that failed. A failure that could diff --git a/docs/src/content/docs/guides/deployment.mdx b/docs/src/content/docs/guides/deployment.mdx index 8bff9170..0cc5c21c 100644 --- a/docs/src/content/docs/guides/deployment.mdx +++ b/docs/src/content/docs/guides/deployment.mdx @@ -92,6 +92,12 @@ instances coordinate through the database. read model. - Appends take one lock per store, which bounds writes to what one connection can commit ([the ceiling](/concepts/how-it-runs/#the-ceiling-with-numbers)). +- The background jobs on personal data and dead letters, the backfill of what was written before + its fields were marked, the rotation of the master key, the retention and the purge of resolved + letters, each run in one instance at a time, under a lease in `bounda_checkpoints` that the others + leave alone until it lapses, two minutes after the holder last renewed it, and that an instance + stopping gives back. An instance that stalls past its lease stops at its next item, and the one + that took the lease over goes on from what the store holds. [At least once, and exactly once per batch](/concepts/delivery-guarantees/) says what each promise covers, and [How Bounda runs](/concepts/how-it-runs/) why, and what the ceiling of one @@ -173,7 +179,10 @@ which may quote what the projection read, is in `errorMessage` only with process saw: another instance that never tried that read model reports no failure while its lag still grows. Alert on the lag, which every instance reads from the database; read `failing`, and the `subscriber failed` log line that names the event, to find out -why. The event itself, payload included, is one read of the global stream away: +why. The last failure of each subscriber, by any instance, is also kept in the database, with its +text encrypted under the person the event names: `bounda status` shows it without the text, and +`app.subscriberFailures.get("projection:orderSummary")` opens it, a read the +[access log](/guides/personal-data/#who-opened-what) records. The event itself, payload included, is one read of the global stream away: `app.events.read({ after: 41, limit: 1 })` ([Reading the global stream](/guides/reading-the-stream/)). `waitingDeadlines`, next to `subscribers`, counts the process deadlines this instance's worker @@ -185,9 +194,10 @@ while a runner is behind, and each runs anyway after ten rounds. ## Schema -There are no migration files to write or run, and no migration step in your deploy. Bounda -versions its own tables ([what an adapter creates](/adapters/#what-an-adapter-creates)) and brings -them forward when the app starts, before it serves anything: +There are no migration files to write or run. Bounda versions its own tables +([what an adapter creates](/adapters/#what-an-adapter-creates)) and brings them forward when the +app starts, before it serves anything, or before the deploy with +[`bounda migrate`](/reference/cli/#bounda-migrate): - A database at the version the installed Bounda expects costs one read, and no lock. - One that is behind is migrated in place, one migration at a time, each in a transaction that @@ -207,7 +217,18 @@ them forward when the app starts, before it serves anything: Stop every instance of the previous version before the first one of the new version starts: instances already running never read the version again, and their code was written for the tables before the migration, so a rolling deploy across a release that migrates would have them write rows -the new code misreads. One of them that starts after the migration is refused. Each table prefix +the new code misreads. One of them that starts after the migration is refused. `bounda status` +says, by database, at which version each scope of the tables is and which version this release +migrates it to; `bounda migrate` runs the migrations of every database the configuration names, as +a step of its own, so a large migration happens while nothing waits on it, and the new instances +start on tables already migrated: + +```sh +bounda status # "storage storage 1 of 2 (migrations pending: bounda migrate)" +# stop the previous version +bounda migrate +# start the new one +``` Each table prefix keeps its own versions, and each Durable Object, so each tenant, migrates its tables the first time it starts on the new code. A [forget ledger](/guides/personal-data/#backups) in another database is migrated there the first time the app uses it. @@ -336,9 +357,10 @@ Back up the database as the engine recommends: a copy of the SQLite file taken o key, wrapped under the [master key](/guides/personal-data/#a-master-key): keep the master key for as long as the backups. Restoring one taken before someone was forgotten brings them back, unless the app has a [forget ledger](/guides/personal-data/#backups) in another database, which forgets -them again when the app starts on the restored copy, or within `reconcileEvery` when the copy is -restored under the running app; restart it after restoring to reconcile at once. Set the ledger's -`keepFor` longer than the oldest backup you keep. +them again when the app starts on the restored copy, or at once when the copy is restored under the +running app: every running instance finds that the database went back, and its background jobs +start over (see [restores while running](/guides/personal-data/#restores-while-running)). Set the +ledger's `keepFor` longer than the oldest backup you keep. ## Observability diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index 1456d05f..16e273e3 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -296,6 +296,10 @@ person, for them to take away, as the right of access and the right to portabili { id, kind: "policy", handler: "customer.welcomeOnCustomerRegistered", errorMessage: "no mailbox for ana@example.com", errorStack, data: {} }, ], + subscriberFailures: [ + { subscriber: "projection:customerDirectory", eventId, eventType: "CustomerRegistered", + errorMessage: "cannot index ana@example.com", errorStack }, + ], readModels: { customerDirectory: [{ key: "c-42", data: { email: "ana@example.com" } }], orderSummary: [{ key: "o-9", data: {} }], @@ -312,9 +316,11 @@ whose payload holds personal fields of theirs, with `commandId` the id the dispa for one the runtime scheduled itself, the scheduler's key, each with the same `data`. `deadLetters` are the handler runs that gave up and name the person: work about them, whose `errorMessage` and `errorStack` are encrypted with their key and come out opened, and the -scheduled commands whose payload holds their fields, with its `data`. After `forget`, the -person's events and letters are still theirs, and every field and error text reads as `null`, as -a handler reads it. `readModels` has an entry for every read model with a field marked +scheduled commands whose payload holds their fields, with its `data`. `subscriberFailures` are the +last failures of the subscribers on an event that names them, kept with their error text +encrypted with their key (see [observability](/reference/observability/#error-text)), opened. +After `forget`, the person's events and letters are still theirs, and every field and error text +reads as `null`, as a handler reads it. `readModels` has an entry for every read model with a field marked [`.person()`](#personal-data-in-read-models): the rows whose person field holds the id, each with its primary key as `key`, so the person can tell them apart, and its fields marked `.personal()` as `data`, read through the read model's own storage, which may be another @@ -397,9 +403,11 @@ person's data in the clear records it, once per call: | Operation | `persons` | `detail` | | --- | --- | --- | -| `exportPersonalData` | the person | `{ events, scheduledCommands, deadLetters, readModelRows }`, how many of each; `null` when it failed | +| `exportPersonalData` | the person | `{ events, scheduledCommands, deadLetters, subscriberFailures, readModelRows }`, how many of each; `null` when it failed | | `deadLetters.list` | those the letters name | `{ letters }`, how many | | `deadLetters.get`, `retry`, `discard` | those the letter names | `{ letter }`, its id | +| `subscriberFailures.list` | those the failures' events name | `{ failures }`, how many | +| `subscriberFailures.get` | the one the failure's event names | `{ subscriber }`, its name | | a query of a read model with fields marked `.personal()` | those the `person` field names in the rows the query read | `{ query, readModel }` | Each record is `{ operation, actor, persons, personCount, at, outcome, detail }`, and a query's @@ -573,11 +581,14 @@ forgotten, with its own checkpoint, in batches, like any subscriber; it shows in started over from the first event it records the same rows. Every pass of the scheduled-command worker then sweeps: an indexed read of the persons whose last activity is older than the retention, not restricted and not forgotten since, a bounded batch of them each forgotten in a -transaction of its own. The deadline is worked out at sweep time from the setting, so changing it +transaction of its own. One instance sweeps at a time, under a lease kept in `bounda_checkpoints` +that the others leave alone until it lapses, two minutes after the holder last renewed it; an +instance that stalled past its lease stops at the next person, and the one that took it over goes +on. The deadline is worked out at sweep time from the setting, so changing it applies at once, to everyone, and turning it on applies it to the history: whoever was already inactive for that long is forgotten in the first passes. A person named by an event the subscriber has not recorded yet is left for a later pass, read in the transaction that forgets them, so nobody is forgotten for an -activity the row does not show yet, nor twice when two workers sweep at once; an event stored in +activity the row does not show yet, nor twice when a lease changes hands mid-way; an event stored in a transaction running beside it may still be forgotten with them, as one a command stores while `forget` runs may be. A person the sweep could not forget, because their forget failed or their row is behind, waits on their own, half a minute at first and twice as long after each failure up @@ -729,7 +740,8 @@ keyWrapper: masterKey({ from: "BOUNDA_MASTER_KEY", previous: ["BOUNDA_MASTER_KEY ``` A key wrapped under a previous master key still opens; the worker rewraps them under the current -one in batches, on every pass, as the retention sweep runs. A key that fails to rewrap waits on its +one in batches, on every pass, as the retention sweep runs, one instance at a time under a lease +of its own for each set of master keys, as the retention's. A key that fails to rewrap waits on its row, as a person the retention could not forget does, while the others go on. Once none is left the worker marks the rotation complete for this set of master keys, in `bounda_checkpoints`, and logs it once: @@ -905,12 +917,12 @@ The check and the forgetting are one transaction, so instances starting together once. A Durable Object does it on every cold start; restoring its storage restarts it. While the app runs, it reads the ledger again every `reconcileEvery`, five minutes unless you set -it, so a database restored under a running app forgets again within that time. One instance -takes each round, whichever finds it due first, and the time of the last round is kept in the -database, so a restore takes it back too and cannot hold the next round off. The +it. One instance takes each round, whichever finds it due first, and the time of the last round is +kept in the database, so a restore takes it back too and cannot hold the next round off. The round runs on the worker's passes, in every instance whose role runs the worker; a Durable Object runs it on the passes it wakes for, never waking for it alone, since restoring its storage -restarts it anyway. Restarting the app after a restore reconciles at once. +restarts it anyway. A database restored under a running app is reconciled at once, on the next +pass that [finds the restore](#restores-while-running). A ledger that cannot be read when the app starts does not stop it: the app logs an error, starts, and its worker tries again after 30 seconds, then twice as long after each failure, up to an hour. @@ -981,6 +993,35 @@ it appends. `app.forgetAgain(entries, { actor })` is the same in code. Without either, a restored backup brings back the data of everyone forgotten after it was taken, until they are forgotten again. +### Restores while running + +Every instance that runs the worker moves a counter in `bounda_checkpoints` one forward, at most +every five seconds, and remembers what it read. A copy restored under the running app holds less +than that, so the next pass of each instance finds the database went back; the first to find it +counts the restore in another row, which every other instance then sees changed, whatever its own +reading shows. Each instance then: + +- reconciles the forget ledger at once, without waiting for `reconcileEvery` or for a hold the copy + brought back; +- forgets what it knew of its background jobs, the backfill caught up, the rotation complete, the + waits after a failure, the dispatcher's backoff, and so reads the copy again on that pass; +- and, on the instance that found it first, clears the holds the copy brought back, on persons, + keys, the backfill and the reconciliation: they count failures against data the copy no longer + holds as it was. + +The attempts of handlers, a policy's, a process's or a scheduled command's, stay as the copy has +them: those runs happened, and what they did through a port is not undone. It logs: + +``` +the database went back to an earlier copy; background jobs start over +``` + +and `bounda status` counts the restores found. A copy taken within five seconds of being restored +may hold the counter every instance read last, and goes unnoticed: what it undid is that recent. +A copy restored while the app was stopped is not told apart from a database left alone, since no +running instance remembers what it held; the start reconciles the ledger whatever its hold says, +and the holds the copy brought back end on their own, within an hour at most. + ### Old backups The ledger handles a backup once it is restored. Rotating the [master key](#rotating-the-master-key) diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index f4f4f9a5..bf37c33b 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -1,6 +1,6 @@ --- title: CLI -description: "The bounda command: generate the registry and the types, rebuild a read model, deal with dead letters." +description: "The bounda command: generate the registry and the types, rebuild a read model, deal with dead letters, read the runtime's status and migrate its tables." sidebar: order: 0 --- @@ -47,11 +47,13 @@ registry that cannot be loaded, a read model that is not in the registry, a list read, and errors whose message Bounda writes from names and ids alone, such as a dead letter that is not found or was already settled. To read what a reaction failed on, read its letter: `bounda dead-letters list` opens the text it keeps encrypted, an access the access log records. +What a subscriber, a projection say, failed on last is in `bounda status --error-text`, recorded +the same way. ## Who runs a command `--actor ` names who runs a command that acts or reads personal data: `rebuild`, -`dead-letters list`, `retry` and `discard`, and `forget`. The command acts as that id, as +`dead-letters list`, `retry` and `discard`, `status`, and `forget`. The command acts as that id, as [`client.as(actor)`](/concepts/ids-causation-and-correlation/#who-acts) does in code: the events it writes carry it in their metadata, its log lines name it, and so does every access log record it causes. @@ -242,6 +244,90 @@ first, the error is printed, as [error text](#error-text) says, and the exit cod process letter it also says when one of the parked events failed the process again. What a retry runs for each kind is in [Dead letters](/guides/dead-letters/). +## `bounda status` + +Prints where the runtime's background work stands, as the database says. It boots the project as +`bounda dead-letters` does. + +```bash +bounda status +bounda status --json +bounda status --error-text --actor ops-ana +``` + +``` +tables + storage storage 2 of 2 + storage checkpoints 1 of 1 + forgetLedger forget-ledger 1 of 1 +restores found while running: 0 +subscribers (head at 1204) + projection:customerDirectory at 1197, 7 behind + failing: Error on CustomerRegistered at 1198, 4 attempts since 2026-11-09T10:02:11.000Z + policies at 1204, 0 behind +personal data backfill: sealed through position 1204, caught up +forget ledger: reconciled, applied through 2026-11-08T22:15:00.000Z, last run 2026-11-09T10:00:00.000Z +key rotation: under 7Kx0Qw9z (previous Wn3eT1aa), 12 keys left to rewrap +retention: 730d, 0 persons past it, next due 2028-01-04T09:13:00.000Z +dead letter retention: 30d, next purge 2026-11-20T08:00:00.000Z +``` + +- `tables`: each scope of Bounda's tables, in each database the configuration names (`storage`, + `readModels.` for read models in a database of their own, `forgetLedger`), at the version + the database holds of the one this release migrates it to; one behind says + `(migrations pending: bounda migrate)`. A store without tables, as `memory()`, has none. +- `restores found while running`: how many times a running instance found the database gone back + to an earlier copy ([restores while running](/guides/personal-data/#restores-while-running)). +- each subscriber's checkpoint and lag, and its last failure, by any instance: `failing` while it is + still stuck on it, `last failed, resolved …` once it got past. Its class and code, the event, how + many deliveries in a row failed and since when; never its text. +- the [backfill](/guides/personal-data/#marking-a-field-later) of personal data, the forget + ledger's [reconciliation](/guides/personal-data/#what-the-runtime-does-with-it), the + [rotation](/guides/personal-data/#rotating-the-master-key) of the master key, the + [retention](/guides/personal-data/#inactivity) and the [purge of resolved dead + letters](/guides/dead-letters/#keeping-resolved-letters), each `not configured` when it is off. + +`--error-text` also prints, under each failure, the text it keeps, encrypted under the person the +event names: it is read through `app.subscriberFailures.list()`, which records it in the +[access log](/guides/personal-data/#who-opened-what) with the persons and `--actor`. A failure whose +person was forgotten says its text went with them. + +| Option | Meaning | +| --- | --- | +| `--json` | Print `app.status()` as JSON; with `--error-text`, `subscriberFailures` holds the failures with their text | +| `--root`, `--config`, `--registry`, `--error-text` | As for `bounda rebuild` | +| `--actor ` | Who reads the failures' text with `--error-text`, named in the access log | + +## `bounda migrate` + +Brings Bounda's tables to this release's version in every database the configuration names, as +the first instance to start would, and prints the versions after: + +```bash +bounda migrate +``` + +``` +migrated + storage storage 2 of 2 + storage checkpoints 1 of 1 + readModels.reports checkpoints 1 of 1 +``` + +It reads only the configuration, never the registry, and starts no app: run it as a step of the +deploy, after stopping the previous version and before starting the new one (see +[Schema](/guides/deployment/#schema)). Each database migrates under its own lock, as a start does, +so running it twice, or while an instance starts, migrates once. Tables a newer release migrated +are refused untouched with [`STORAGE_SCHEMA_TOO_NEW`](/reference/errors/#codes). + +| Option | Meaning | +| --- | --- | +| `--root ` | The project root, the current directory by default | +| `--config ` | The configuration module under the root, `bounda.config.ts` by default | + +Exit codes: `0` migrated, `1` bad arguments, `2` the configuration could not be loaded or a +migration failed. + ## `bounda forgotten` and `bounda forget` Apply again, by hand, the forgettings a restored backup undid, for an app without a @@ -297,7 +383,9 @@ It throws `ConventionError` when the layout breaks a convention; `formatConventi `formatWarnings` format the error and the warnings as the command prints them. `bounda rebuild` is `rebuildReadModel` from `@bounda-dev/core` over a project loaded with `loadProject` from `@bounda-dev/core/node`, with `actor` for `--actor`; an app exposes the same as `app.rebuildReadModel(name)`, and -`app.as(actor).rebuildReadModel(name)`. `bounda dead-letters` is `app.deadLetters` on a booted app, +`app.as(actor).rebuildReadModel(name)`. `bounda status` is `app.status()`, and its failures' text +`app.as(actor).subscriberFailures.list()`; `bounda migrate` is `migrateTables({ config })` from +`@bounda-dev/core`. `bounda dead-letters` is `app.deadLetters` on a booted app, or `app.as(actor).deadLetters`, and `bounda forget` is `app.forgetAgain(entries, { actor })`. The commands boot the app with `createConsoleLogger({ stream: process.stderr })` from `@bounda-dev/core/node`, which writes every level to the stream it is given. diff --git a/docs/src/content/docs/reference/errors.md b/docs/src/content/docs/reference/errors.md index 1e952a08..a98497b9 100644 --- a/docs/src/content/docs/reference/errors.md +++ b/docs/src/content/docs/reference/errors.md @@ -81,7 +81,7 @@ dead letter with `errorType: "retriable_exhausted"`. the app does not start, since a person a restored backup brought back could be read. `cause` is the ledger's error. Starting again tries again. - **`STORAGE_SCHEMA_TOO_NEW`** (`StorageSchemaTooNewError`; a start's answer). Thrown by - `createApp`, and so by `boot` and the CLI, when a newer version of Bounda has + `createApp`, and so by `boot` and the CLI, and by `migrateTables` and `bounda migrate`, when a newer version of Bounda has [migrated the database's Bounda tables](/guides/deployment/#schema) past what this version knows: `scope` (`storage`, `checkpoints` or `forget-ledger`), its `version`, and `supported`, the newest this version knows. The app does not start and nothing is written: Bounda never takes diff --git a/docs/src/content/docs/reference/limitations.md b/docs/src/content/docs/reference/limitations.md index 4a875d0b..a87f8194 100644 --- a/docs/src/content/docs/reference/limitations.md +++ b/docs/src/content/docs/reference/limitations.md @@ -22,6 +22,10 @@ get today. Each item says why, so nobody discovers it the hard way: `pollInterval`; only the event dispatcher is woken by `NOTIFY`. See [Tuning](/guides/deployment/#tuning). - **Every read of personal data in the access log.** See [The access log](#the-access-log). +- **A restore made while the app was stopped, told apart.** No running instance remembers what the + database held, so the jobs start from the copy as from any start, and the holds of failures it + brought back end on their own, within an hour. See + [Restores while running](/guides/personal-data/#restores-while-running). ## The access log @@ -54,8 +58,10 @@ What a production app does get, and where it is explained: reaction, [forgetting them](/guides/personal-data/#retention) after a retention or when a purpose ends, their keys [wrapped under a master key](/guides/personal-data/#a-master-key) kept out of the database and out of JavaScript memory, their forgetting [applied again](/guides/personal-data/#backups) when a backup is -restored, what was written before a field was marked [sealed in -place](/guides/personal-data/#marking-a-field-later), +restored, and found when restored [under the running app](/guides/personal-data/#restores-while-running), +what was written before a field was marked [sealed in +place](/guides/personal-data/#marking-a-field-later), [the status of the background +work](/reference/cli/#bounda-status) and [migrations run before a deploy](/reference/cli/#bounda-migrate), [dead letters with a way out](/guides/dead-letters/), [upcasters](/guides/changing-events/) for events whose payload changed, [snapshots](/concepts/long-streams/#snapshots) whose validity the runtime checks itself, [observability](/reference/observability/) through OpenTelemetry, and a diff --git a/docs/src/content/docs/reference/observability.md b/docs/src/content/docs/reference/observability.md index 114ef872..2a06b7e3 100644 --- a/docs/src/content/docs/reference/observability.md +++ b/docs/src/content/docs/reference/observability.md @@ -73,14 +73,21 @@ default no span and no log line of the runtime carries the text of an error: `exception` event has neither `exception.message` nor `exception.stacktrace`; - a log line names the error with `errorName` and, for a `BoundaError`, `errorCode`, next to the ids of what failed: the event, the command, the subscriber; -- `app.getLag()` says what a failing subscriber is stuck on with `errorName` and `errorCode`. +- `app.getLag()` says what a failing subscriber is stuck on with `errorName` and `errorCode`, and + `app.status()` (and `bounda status`) its last failure, by any instance, with the same identity. An error whose message Bounda writes from names and ids alone, such as `ConcurrencyError`, `ClaimLostError`, `HandlerTimeoutError` or a dead letter not found, keeps its message, never its stack. To find out what a reaction failed on, follow the log line that filed its dead letter: `policy dead-lettered`, `process dead-lettered` and `scheduled command dropped` carry `letterId`, and `app.deadLetters.get(letterId)` opens the text the letter keeps encrypted under the person -(see [what a letter holds](/guides/dead-letters/#what-a-letter-holds)). Your own logger calls +(see [what a letter holds](/guides/dead-letters/#what-a-letter-holds)). A projection has no +letter: its batch is delivered again until it goes through, so the runtime keeps the last failure +of each subscriber instead, its text encrypted under the person the event names, or in the clear +for an event that names nobody, whatever `errorText` says. `app.subscriberFailures.list()` and +`.get(subscriber)`, or `bounda status --error-text`, open it, and each call is recorded in the +[access log](/guides/personal-data/#the-access-log); once the person is forgotten it reads `null`. +Your own logger calls around the app, in a request handler, can follow the same rule with `errorFields`: ```ts From 77deb2284f2856c6c02fe2c0f0562d40b0f0e8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:40:38 +0100 Subject: [PATCH 06/19] fix: findings of the review of the background jobs, the status and abandoned processes --- packages/cli/src/status-output.test.ts | 5 ++-- packages/core/src/adapter/adapter.ts | 3 ++ .../testing/checkpoint-store.contract.ts | 5 +++- packages/core/src/kernel/app.ts | 1 - .../dispatch/dispatcher-backoff.test.ts | 26 ++++++++++++++++ .../core/src/kernel/dispatch/dispatcher.ts | 1 + .../src/kernel/dispatch/last-failures.test.ts | 21 +++++++++++++ .../core/src/kernel/dispatch/last-failures.ts | 4 ++- .../src/kernel/personal-data/forget-ledger.ts | 10 +++++-- .../src/kernel/personal-data/retention.ts | 2 ++ .../src/kernel/personal-data/rotation.test.ts | 5 +++- .../core/src/kernel/personal-data/rotation.ts | 2 ++ .../core/src/kernel/process/deliver-event.ts | 24 +++++++++++++-- .../core/src/kernel/process/parked.test.ts | 30 +++++++++++++++++++ packages/core/src/kernel/shared/lease.ts | 2 ++ packages/core/src/kernel/status.test.ts | 28 +++++++++++++++++ packages/core/src/kernel/status.ts | 10 +++---- 17 files changed, 163 insertions(+), 16 deletions(-) create mode 100644 packages/core/src/kernel/status.test.ts diff --git a/packages/cli/src/status-output.test.ts b/packages/cli/src/status-output.test.ts index e54e44d8..2b8de18e 100644 --- a/packages/cli/src/status-output.test.ts +++ b/packages/cli/src/status-output.test.ts @@ -16,6 +16,8 @@ const failure = { resolvedAt: null, }; +const { errorCode: _code, ...withoutCode } = failure; + const status: AppStatus = { schema: [ { database: "storage", scope: "storage", version: 1, latest: 2 }, @@ -37,9 +39,8 @@ const status: AppStatus = { subscriberFailures: [ { ...failure, current: true }, { - ...failure, + ...withoutCode, subscriber: "policies", - errorCode: undefined, errorName: "Error", attempts: 1, resolvedAt: "2026-05-02T00:00:00.000Z", diff --git a/packages/core/src/adapter/adapter.ts b/packages/core/src/adapter/adapter.ts index bd92479e..4d36ee57 100644 --- a/packages/core/src/adapter/adapter.ts +++ b/packages/core/src/adapter/adapter.ts @@ -238,6 +238,9 @@ export interface Adapter */ export type SchemaScopeName = "storage" | "checkpoints" | "forget-ledger"; +/** + * Where one scope of Bounda's tables stands in a database, as `Adapter.schemaVersions` reads it. + */ export interface SchemaVersion { readonly scope: string; /** diff --git a/packages/core/src/adapter/testing/checkpoint-store.contract.ts b/packages/core/src/adapter/testing/checkpoint-store.contract.ts index 40878bcf..4f50cae2 100644 --- a/packages/core/src/adapter/testing/checkpoint-store.contract.ts +++ b/packages/core/src/adapter/testing/checkpoint-store.contract.ts @@ -117,7 +117,10 @@ export const checkpointStoreContract: CheckpointStoreContractFunction = ({ creat expect(await one.take(start)).toBe(true); await one.release(); expect(await store.get("job:lease")).toBe(start); - expect(await leaseOver().take(start)).toBe(true); + const other = leaseOver(); + expect(await other.take(start)).toBe(true); + expect(await one.take(start)).toBe(false); + expect(await store.get("job:lease")).toBe(start + LEASE_MS); }); it("takes back as its own a lease of its own a restored copy brought back", async () => { diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 925ee5a8..ece69cbf 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -794,7 +794,6 @@ export const assembleApp: AssembleAppFunction = async ({ lag: getLag, failures: written.subscriberFailures, restores, - backfill, reconciliation, rotation, retention, diff --git a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts index 4c118408..de9baf43 100644 --- a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts +++ b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts @@ -98,6 +98,32 @@ describe("the dispatcher's backoff", () => { expect(attemptsAt).toEqual([100, 1_100, 3_100, 7_100, 11_100]); }); + it("resolves the stored failure again after a restore, which may have brought it back", async () => { + const resolved: string[] = []; + const { eventStore, checkpointStore } = await memory().createStorage(); + const recovering = scripted("projection:x", [{ outcome: "advanced" }, { outcome: "advanced" }]); + const dispatcher = createDispatcher({ + clock: createFixedClock(), + eventStore, + checkpointStore, + subscribers: [recovering], + batchSize: 10, + pollIntervalMs: 100, + lastFailures: { + record: async () => undefined, + resolve: async (subscriber) => { + resolved.push(subscriber); + }, + }, + logger: silentLogger, + errorText: "omit", + }); + await dispatcher.processOnce(); + dispatcher.restored(); + await dispatcher.processOnce(); + expect(resolved).toEqual(["projection:x", "projection:x"]); + }); + it("forgets every failure it met after a restore, so each is tried again at once", async () => { const poisoned = scripted("projection:poisoned", [failed, failed, failed]); const { clock, dispatcher } = await setUp([poisoned]); diff --git a/packages/core/src/kernel/dispatch/dispatcher.ts b/packages/core/src/kernel/dispatch/dispatcher.ts index efc1c50f..6157c056 100644 --- a/packages/core/src/kernel/dispatch/dispatcher.ts +++ b/packages/core/src/kernel/dispatch/dispatcher.ts @@ -373,6 +373,7 @@ export const createDispatcher: CreateDispatcherFunction = ({ }, restored: () => { failing.clear(); + checked.clear(); }, getLag: async () => { const lastPosition = await eventStore.lastPosition(); diff --git a/packages/core/src/kernel/dispatch/last-failures.test.ts b/packages/core/src/kernel/dispatch/last-failures.test.ts index cfa237b4..f7e24095 100644 --- a/packages/core/src/kernel/dispatch/last-failures.test.ts +++ b/packages/core/src/kernel/dispatch/last-failures.test.ts @@ -61,4 +61,25 @@ describe("createLastFailures", () => { }, ]); }); + + it("seals the text under the person the store names when the marks name nobody", async () => { + const storage = await memory().createStorage(); + const lastFailures = createLastFailures({ + store: storage.subscriberFailures, + aggregates: buildAggregates({ registry: { aggregates: {}, readModels: {} }, ports: {} }), + keys: createPersonalKeyWrapping({ wrapper: null, clock: createFixedClock() }).over( + storage.personalKeys, + ), + logger: createRecordingLogger("omit").logger, + }); + await lastFailures.record({ + subscriber: "projection:x", + event: { ...event, person: "c-9" }, + error: new Error("cannot reach ana@example.com"), + at: new Date("2026-01-01T00:00:00.000Z"), + }); + const stored = await storage.subscriberFailures.get("projection:x"); + expect(stored).toMatchObject({ person: "c-9", errorMessage: { $personal: { person: "c-9" } } }); + expect(JSON.stringify(stored)).not.toContain("ana@example.com"); + }); }); diff --git a/packages/core/src/kernel/dispatch/last-failures.ts b/packages/core/src/kernel/dispatch/last-failures.ts index 48223081..6e7756bc 100644 --- a/packages/core/src/kernel/dispatch/last-failures.ts +++ b/packages/core/src/kernel/dispatch/last-failures.ts @@ -52,7 +52,9 @@ export const createLastFailures: CreateLastFailuresFunction = ({ }) => ({ record: async ({ subscriber, event, error, at }) => { try { - const person = personOfEvent({ aggregates, event }); + // The store's `person` when the marks name nobody, as for a person forgotten since: the text + // is then sealed under a key no store keeps. + const person = personOfEvent({ aggregates, event }) ?? event.person; await store.record({ subscriber, position: event.position, diff --git a/packages/core/src/kernel/personal-data/forget-ledger.ts b/packages/core/src/kernel/personal-data/forget-ledger.ts index c1f76cc5..a1c75b97 100644 --- a/packages/core/src/kernel/personal-data/forget-ledger.ts +++ b/packages/core/src/kernel/personal-data/forget-ledger.ts @@ -75,9 +75,13 @@ export interface OpenForgetLedgerFunction { (args: OpenForgetLedgerArgs): Promise; } -// The same adapter, or one of the same kind with the same options: a database a restore rolls back -// together with the storage. Two memory adapters are two stores whatever their options. -const sharesStorage = (source: Adapter, storage: AdapterDefinition): boolean => +export interface SharesStorageFunction { + (source: Adapter, storage: AdapterDefinition): boolean; +} + +// The same adapter, or one of the same kind with the same options: the same database. Two memory +// adapters are two stores whatever their options. +export const sharesStorage: SharesStorageFunction = (source, storage) => source === storage || (source.name !== "memory" && source.name === storage.name && diff --git a/packages/core/src/kernel/personal-data/retention.ts b/packages/core/src/kernel/personal-data/retention.ts index a9a4c7e6..c33ed5c5 100644 --- a/packages/core/src/kernel/personal-data/retention.ts +++ b/packages/core/src/kernel/personal-data/retention.ts @@ -256,6 +256,8 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ const sweep = async (now: Date): Promise => { const at = now.getTime(); + // While another instance holds the lease, its row is one read, the persons a batch. + if (at < lease.heldElsewhereUntil() && !(await lease.take(at))) return 0; const due = await storage.persons.listIdle({ activeBefore: new Date(at - retentionMs), releasedBy: now, diff --git a/packages/core/src/kernel/personal-data/rotation.test.ts b/packages/core/src/kernel/personal-data/rotation.test.ts index ae6bec18..92f8069f 100644 --- a/packages/core/src/kernel/personal-data/rotation.test.ts +++ b/packages/core/src/kernel/personal-data/rotation.test.ts @@ -738,8 +738,11 @@ describe("the key rotation across instances", () => { expect(await one.sweep(clock.now())).toBe(1); await addUnder(store, "old", "c-2"); clock.advance(ROTATION_CHECK_INTERVAL_MS); - const other = rotationOver({ keys: store, checkpoints, wrapper: wrapper() }); + const { keys, reads } = counted(store); + const other = rotationOver({ keys, checkpoints, wrapper: wrapper() }); + expect(await other.sweep(clock.now())).toBe(0); expect(await other.sweep(clock.now())).toBe(0); + expect(reads.count).toBe(1); expect(await kidsOf(store, ["c-2"])).toEqual(["old"]); expect(await other.nextDueAt()).toEqual(new Date(leased)); await one.release(); diff --git a/packages/core/src/kernel/personal-data/rotation.ts b/packages/core/src/kernel/personal-data/rotation.ts index 769b3069..91a8b07e 100644 --- a/packages/core/src/kernel/personal-data/rotation.ts +++ b/packages/core/src/kernel/personal-data/rotation.ts @@ -200,6 +200,8 @@ export const createKeyRotation: CreateKeyRotationFunction = ({ completeUntil = markedAt + ROTATION_CHECK_INTERVAL_MS; return 0; } + // While another instance holds the lease, its row is one read, the keys a batch. + if (at < lease.heldElsewhereUntil() && !(await lease.take(at))) return 0; answer = null; const due = await keys.listWrappedUnder({ kids, releasedBy: now, limit: ROTATION_BATCH }); if (due.length === 0) { diff --git a/packages/core/src/kernel/process/deliver-event.ts b/packages/core/src/kernel/process/deliver-event.ts index 1170608f..7b93fff4 100644 --- a/packages/core/src/kernel/process/deliver-event.ts +++ b/packages/core/src/kernel/process/deliver-event.ts @@ -13,7 +13,12 @@ import type { UnitOfWork } from "../unit-of-work/unit-of-work.ts"; import type { ProcessRuntime } from "./build-processes.ts"; import type { ProcessFailures } from "./failures.ts"; import type { ProcessHandlers } from "./handlers.ts"; -import { type LifecycleEntry, lifecycleEntries, type ProcessInstance } from "./lifecycle.ts"; +import { + instanceContext, + type LifecycleEntry, + lifecycleEntries, + type ProcessInstance, +} from "./lifecycle.ts"; import { actsOn, completesOn, followsUp, handledEntries, handlerOf, startsOn } from "./routes.ts"; import type { DeadlineSchedule } from "./schedule.ts"; import type { ProcessUnits } from "./units.ts"; @@ -72,6 +77,14 @@ export const createEventDelivery: CreateEventDeliveryFunction = ({ !instance.handledEventIds.has(event.id) && !instance.parked.some((parked) => parked.eventId === event.id); + // A failure whose letter was discarded before `ProcessAbandoned` existed: the instance is + // abandoned now, while the letter still says so, rather than parking for good. + const discardedBefore = async (unit: UnitOfWork, instance: ProcessInstance): Promise => { + const letterId = instance.failure?.letterId; + if (letterId === undefined) return false; + return (await unit.deadLetterStore.get(letterId))?.status === "discarded"; + }; + // The instance with its start staged in `entries` when `event` starts it; as it is otherwise. const started = ( process: ProcessRuntime, @@ -98,7 +111,14 @@ export const createEventDelivery: CreateEventDeliveryFunction = ({ const instance = started(process, event, await within.load(process, instanceId), entries); if (!instance.exists) return false; let parked = false; - if (instance.status === "failed") { + if (instance.status === "failed" && (await discardedBefore(unit, instance))) { + entries.push( + lifecycleEntries.abandoned( + instance.failure?.letterId ?? "", + instanceContext(process, instanceId, instance), + ), + ); + } else if (instance.status === "failed") { if (parks(process, event, instance)) { entries.push(lifecycleEntries.parked(event)); parked = true; diff --git a/packages/core/src/kernel/process/parked.test.ts b/packages/core/src/kernel/process/parked.test.ts index d66c1868..d6f2ab52 100644 --- a/packages/core/src/kernel/process/parked.test.ts +++ b/packages/core/src/kernel/process/parked.test.ts @@ -457,6 +457,36 @@ describe("events of a failed process", () => { ).toHaveLength(1); }); + it("abandon an instance whose letter was discarded before ProcessAbandoned existed", async () => { + const context = await setUp(); + const { harness, deadLetters, settle, types } = context; + await failOnFirstPayment(context); + const [letter] = await deadLetters.list(); + // As a discard before this release left it: the letter discarded, the instance failed. + await harness.storage.deadLetterStore.updateStatus( + letter?.id ?? "", + "discarded", + harness.clock.now(), + ); + const order = await harness.storage.eventStore.load({ + aggregateType: "order", + aggregateId: "o-1", + }); + const last = order.events.at(-1); + if (last === undefined) throw new Error("no order"); + await harness.storage.eventStore.append({ + aggregateType: "order", + aggregateId: "o-1", + expectedVersion: order.events.length, + events: [{ ...last, id: "after-upgrade", version: order.events.length + 1 }], + }); + await settle(); + expect((await types()).slice(-2)).toEqual([PROCESS_EVENTS.failed, PROCESS_EVENTS.abandoned]); + expect((await harness.storage.eventStore.load(stream)).events.at(-1)?.payload).toEqual({ + letterId: letter?.id, + }); + }); + it("abandon nothing for a letter its instance is not failed on", async () => { const context = await setUp(); const { harness, deadLetters, types } = context; diff --git a/packages/core/src/kernel/shared/lease.ts b/packages/core/src/kernel/shared/lease.ts index 96896307..7c2098ad 100644 --- a/packages/core/src/kernel/shared/lease.ts +++ b/packages/core/src/kernel/shared/lease.ts @@ -84,6 +84,8 @@ export const createLease: CreateLeaseFunction = ({ checkpoints, row, clock, logg if (held === null) return; const until = held; held = null; + // Given back, it is no longer this instance's: another may write the same time next. + written = written.filter((value) => value !== until); const lapsed = Math.min(until, clock.now().getTime()); await checkpoints.compareAndSet(row, until, lapsed).catch((error: unknown) => { logger.error("background job could not give its lease back", { lease: row, error }); diff --git a/packages/core/src/kernel/status.test.ts b/packages/core/src/kernel/status.test.ts new file mode 100644 index 00000000..7c44430b --- /dev/null +++ b/packages/core/src/kernel/status.test.ts @@ -0,0 +1,28 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createNodeSqliteAdapter } from "../adapter/sqlite/node-sqlite.ts"; +import { resolveConfig } from "../config/schema.ts"; +import { memory } from "../memory/index.ts"; +import { databasesOf } from "./status.ts"; + +describe("databasesOf", () => { + it("names each database once, an adapter over the storage's own database included", () => { + const directory = mkdtempSync(join(tmpdir(), "bounda-databases-")); + const storage = createNodeSqliteAdapter(join(directory, "storage.db")).adapter; + const alsoStorage = createNodeSqliteAdapter(join(directory, "storage.db")).adapter; + const reports = createNodeSqliteAdapter(join(directory, "reports.db")).adapter; + const names = databasesOf( + resolveConfig({ + storage, + readModels: { orders: alsoStorage, reports, totals: reports, notes: memory() }, + }), + ).map(({ database, scopes }) => [database, scopes]); + expect(names).toEqual([ + ["storage", ["storage", "checkpoints"]], + ["readModels.reports", ["checkpoints"]], + ["readModels.notes", ["checkpoints"]], + ]); + }); +}); diff --git a/packages/core/src/kernel/status.ts b/packages/core/src/kernel/status.ts index fe4b91c7..48eb865e 100644 --- a/packages/core/src/kernel/status.ts +++ b/packages/core/src/kernel/status.ts @@ -7,7 +7,7 @@ import type { import type { ResolvedConfig } from "../config/types.ts"; import type { AppLag } from "./app.ts"; import type { DeadLetterRetention, DeadLetterRetentionStatus } from "./dead-letters/retention.ts"; -import type { PersonalDataBackfill } from "./personal-data/backfill.ts"; +import { sharesStorage } from "./personal-data/forget-ledger.ts"; import type { ForgetLedgerReconciliation, ForgetLedgerReconciliationStatus, @@ -76,7 +76,6 @@ export interface ReadAppStatusArgs { readonly lag: () => Promise; readonly failures: SubscriberFailureStore; readonly restores: RestoreWatch; - readonly backfill?: PersonalDataBackfill | undefined; readonly reconciliation?: ForgetLedgerReconciliation | undefined; readonly rotation?: KeyRotation | undefined; readonly retention?: RetentionSweep | undefined; @@ -108,7 +107,9 @@ export const databasesOf: DatabasesOfFunction = (config) => { }); } for (const [name, adapter] of Object.entries(config.readModels)) { - if (!isAdapter(adapter) || databases.some((known) => known.adapter === adapter)) continue; + if (!isAdapter(adapter) || databases.some((known) => sharesStorage(adapter, known.adapter))) { + continue; + } databases.push({ database: `readModels.${name}`, adapter, scopes: ["checkpoints"] }); } const ledger = config.runtime.personalData.forgetLedger?.store; @@ -133,7 +134,6 @@ export const readAppStatus: ReadAppStatusFunction = async ({ lag, failures, restores, - backfill, reconciliation, rotation, retention, @@ -168,7 +168,7 @@ export const readAppStatus: ReadAppStatusFunction = async ({ failure.resolvedAt === null && position !== undefined && position < failure.position, }; }), - personalDataBackfill: backfill === undefined ? null : await backfill.status(), + personalDataBackfill: current.personalDataBackfill, forgetLedger, keyRotation, retention: retained, From 970a059adeb571d68be3151de12f13422fc128f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:48:05 +0100 Subject: [PATCH 07/19] test: list the subscriber failures table and the second storage migration --- packages/postgresql/src/postgresql.test.ts | 1 + packages/sqlite/src/sqlite.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/postgresql/src/postgresql.test.ts b/packages/postgresql/src/postgresql.test.ts index 83c958d8..6272a666 100644 --- a/packages/postgresql/src/postgresql.test.ts +++ b/packages/postgresql/src/postgresql.test.ts @@ -605,6 +605,7 @@ describe.skipIf(container === null)("postgresql adapter", () => { "rm_order_summary", "scheduled_commands", "snapshots", + "subscriber_failures", ]); }); diff --git a/packages/sqlite/src/sqlite.test.ts b/packages/sqlite/src/sqlite.test.ts index 3e2227ba..86cdc414 100644 --- a/packages/sqlite/src/sqlite.test.ts +++ b/packages/sqlite/src/sqlite.test.ts @@ -357,6 +357,7 @@ describe("sqlite adapter on a file", () => { expect(rows.map((row) => [row.scope, Number(row.version)])).toEqual([ ["checkpoints", 1], ["storage", 1], + ["storage", 2], ]); }); From c7e07748cd915ad9f26e1317efde9cf75cfbf7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:17:25 +0100 Subject: [PATCH 08/19] test: kill the surviving mutants of the background jobs, and cover the object's status --- .../cloudflare/test/bounda-object.test.ts | 29 ++++++++ .../src/kernel/dead-letters/retention.test.ts | 12 ++++ .../dispatch/dispatcher-backoff.test.ts | 10 ++- .../src/kernel/dispatch/last-failures.test.ts | 11 +-- .../core/src/kernel/dispatch/last-failures.ts | 6 +- .../dispatch/subscriber-failures.test.ts | 42 +++++++++++ .../personal-data/reconciliation.test.ts | 11 +++ .../kernel/personal-data/retention.test.ts | 32 ++++++++- .../src/kernel/personal-data/rotation.test.ts | 20 ++++++ packages/core/src/kernel/shared/lease.test.ts | 70 +++++++++++++++++++ .../core/src/kernel/shared/restores.test.ts | 57 +++++++++++++++ packages/core/src/kernel/status.test.ts | 70 ++++++++++++++++++- 12 files changed, 359 insertions(+), 11 deletions(-) create mode 100644 packages/core/src/kernel/dispatch/subscriber-failures.test.ts create mode 100644 packages/core/src/kernel/shared/lease.test.ts create mode 100644 packages/core/src/kernel/shared/restores.test.ts diff --git a/packages/cloudflare/test/bounda-object.test.ts b/packages/cloudflare/test/bounda-object.test.ts index d4406187..27b87b46 100644 --- a/packages/cloudflare/test/bounda-object.test.ts +++ b/packages/cloudflare/test/bounda-object.test.ts @@ -522,6 +522,35 @@ describe("a Bounda Durable Object", () => { }); }); + it("reports its status and keeps a projection's last failure, opened through a recorded call", async () => { + const { store } = open(); + outage.projection = true; + await store.commands.placeOrder({ orderId: "o-1", total: 5, customer: "ada", note: "x" }); + outage.projection = false; + const status = await store.status(); + expect(status.schema).toEqual([ + { database: "storage", scope: "storage", version: 2, latest: 2 }, + { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, + ]); + expect(status.subscriberFailures).toContainEqual( + expect.objectContaining({ + subscriber: "projection:orders", + errorName: "Error", + current: true, + }), + ); + expect(JSON.stringify(status)).not.toContain("projection is down"); + const [failure] = await store.as("ops-1").subscriberFailures.list(); + expect(failure).toMatchObject({ + subscriber: "projection:orders", + errorMessage: "projection is down", + }); + expect(await store.subscriberFailures.get("projection:orders")).toMatchObject({ + errorMessage: "projection is down", + }); + expect(await store.subscriberFailures.get("projection:nothing")).toBeNull(); + }); + it("dead-letters a failing policy and lets it be discarded", async () => { const { stub, store } = open(); await store.commands.placeOrder({ orderId: "fail-1", total: 3, customer: "ada" }); diff --git a/packages/core/src/kernel/dead-letters/retention.test.ts b/packages/core/src/kernel/dead-letters/retention.test.ts index 628b31e2..532eb2a5 100644 --- a/packages/core/src/kernel/dead-letters/retention.test.ts +++ b/packages/core/src/kernel/dead-letters/retention.test.ts @@ -192,6 +192,18 @@ describe("the retention of dead letters across instances", () => { expect(await store.count()).toBe(1); }); + it("purges again at once after a restore, though the store had paused it", async () => { + const { store, retention, resolve } = await setUp(); + await resolve("a", later(-8 * DAY)); + const { firstResolvedAt } = store; + store.firstResolvedAt = () => Promise.reject(new Error("down")); + expect(await retention.sweep(start)).toBe(0); + store.firstResolvedAt = firstResolvedAt; + expect(await retention.sweep(start)).toBe(0); + retention.restored(); + expect(await retention.sweep(start)).toBe(1); + }); + it("reads the store again at once after a restore", async () => { const { store, retention, resolve } = await setUp(); await retention.sweep(start); diff --git a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts index de9baf43..07e9a09a 100644 --- a/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts +++ b/packages/core/src/kernel/dispatch/dispatcher-backoff.test.ts @@ -101,7 +101,12 @@ describe("the dispatcher's backoff", () => { it("resolves the stored failure again after a restore, which may have brought it back", async () => { const resolved: string[] = []; const { eventStore, checkpointStore } = await memory().createStorage(); - const recovering = scripted("projection:x", [{ outcome: "advanced" }, { outcome: "advanced" }]); + const recovering = scripted("projection:x", [ + { outcome: "advanced" }, + { outcome: "advanced" }, + { outcome: "moved" }, + { outcome: "advanced" }, + ]); const dispatcher = createDispatcher({ clock: createFixedClock(), eventStore, @@ -119,6 +124,9 @@ describe("the dispatcher's backoff", () => { errorText: "omit", }); await dispatcher.processOnce(); + await dispatcher.processOnce(); + await dispatcher.processOnce(); + expect(resolved).toEqual(["projection:x"]); dispatcher.restored(); await dispatcher.processOnce(); expect(resolved).toEqual(["projection:x", "projection:x"]); diff --git a/packages/core/src/kernel/dispatch/last-failures.test.ts b/packages/core/src/kernel/dispatch/last-failures.test.ts index f7e24095..26898aa7 100644 --- a/packages/core/src/kernel/dispatch/last-failures.test.ts +++ b/packages/core/src/kernel/dispatch/last-failures.test.ts @@ -62,7 +62,7 @@ describe("createLastFailures", () => { ]); }); - it("seals the text under the person the store names when the marks name nobody", async () => { + it("keeps the text in the clear when the marks name nobody, whoever the store names", async () => { const storage = await memory().createStorage(); const lastFailures = createLastFailures({ store: storage.subscriberFailures, @@ -75,11 +75,12 @@ describe("createLastFailures", () => { await lastFailures.record({ subscriber: "projection:x", event: { ...event, person: "c-9" }, - error: new Error("cannot reach ana@example.com"), + error: new Error("cannot total 5"), at: new Date("2026-01-01T00:00:00.000Z"), }); - const stored = await storage.subscriberFailures.get("projection:x"); - expect(stored).toMatchObject({ person: "c-9", errorMessage: { $personal: { person: "c-9" } } }); - expect(JSON.stringify(stored)).not.toContain("ana@example.com"); + expect(await storage.subscriberFailures.get("projection:x")).toMatchObject({ + person: null, + errorMessage: "cannot total 5", + }); }); }); diff --git a/packages/core/src/kernel/dispatch/last-failures.ts b/packages/core/src/kernel/dispatch/last-failures.ts index 6e7756bc..c0c60c5a 100644 --- a/packages/core/src/kernel/dispatch/last-failures.ts +++ b/packages/core/src/kernel/dispatch/last-failures.ts @@ -52,9 +52,9 @@ export const createLastFailures: CreateLastFailuresFunction = ({ }) => ({ record: async ({ subscriber, event, error, at }) => { try { - // The store's `person` when the marks name nobody, as for a person forgotten since: the text - // is then sealed under a key no store keeps. - const person = personOfEvent({ aggregates, event }) ?? event.person; + // Whom the marks name, as a dead letter's: an event whose marked fields hold nothing, or + // nothing any more, names nobody, and a person without a key would seal the text for good. + const person = personOfEvent({ aggregates, event }); await store.record({ subscriber, position: event.position, diff --git a/packages/core/src/kernel/dispatch/subscriber-failures.test.ts b/packages/core/src/kernel/dispatch/subscriber-failures.test.ts new file mode 100644 index 00000000..b3a42c17 --- /dev/null +++ b/packages/core/src/kernel/dispatch/subscriber-failures.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { memory } from "../../memory/index.ts"; +import type { RecordAccessArgs } from "../personal-data/access.ts"; +import { createPersonalKeyWrapping } from "../personal-data/wrapped-keys.ts"; +import { createSubscriberFailures } from "./subscriber-failures.ts"; + +describe("createSubscriberFailures", () => { + it("names no person for a failure on an event that names nobody, and keeps a failure without a stack so", async () => { + const storage = await memory().createStorage(); + await storage.subscriberFailures.record({ + subscriber: "projection:x", + position: 1, + eventId: "e-1", + eventType: "OrderPlaced", + person: null, + errorName: "TypeError", + errorMessage: "x is not a function", + failedAt: "2026-01-01T00:00:00.000Z", + }); + const records: RecordAccessArgs<"subscriberFailures.list" | "subscriberFailures.get">[] = []; + const failures = createSubscriberFailures({ + store: storage.subscriberFailures, + keys: createPersonalKeyWrapping({ wrapper: null, clock: createFixedClock() }).over( + storage.personalKeys, + ), + actor: "ops-1", + recordAccess: async (args) => { + records.push(args as (typeof records)[number]); + }, + errorText: "omit", + }); + const [listed] = await failures.list(); + expect(listed).toMatchObject({ person: null, errorMessage: "x is not a function" }); + expect(listed).not.toHaveProperty("errorStack"); + expect(await failures.get("projection:x")).not.toHaveProperty("errorStack"); + expect(records.map(({ operation, persons }) => [operation, [...persons]])).toEqual([ + ["subscriberFailures.list", []], + ["subscriberFailures.get", []], + ]); + }); +}); diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index b0eec13f..e5ff120f 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -624,6 +624,17 @@ describe("the reconciliation's status and restores", () => { lastRunAt: NOW.toISOString(), held: null, }); + await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 60_000); + expect((await reconciliation.status()).held).toEqual({ + until: new Date(NOW.getTime() + 60_000).toISOString(), + attempts: 0, + }); + await storage.checkpointStore.remove(FORGET_LEDGER_HELD_UNTIL); + await storage.checkpointStore.set(FORGET_LEDGER_ATTEMPTS, 1); + expect((await reconciliation.status()).held).toEqual({ + until: new Date(0).toISOString(), + attempts: 1, + }); await storage.checkpointStore.set(FORGET_LEDGER_ATTEMPTS, 2); await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 60_000); expect((await reconciliation.status()).held).toEqual({ diff --git a/packages/core/src/kernel/personal-data/retention.test.ts b/packages/core/src/kernel/personal-data/retention.test.ts index c1c5fb91..13eb0bdd 100644 --- a/packages/core/src/kernel/personal-data/retention.test.ts +++ b/packages/core/src/kernel/personal-data/retention.test.ts @@ -810,8 +810,20 @@ describe("the retention across instances", () => { await touch("c-2"); await record(); advance(31); - const other = sweepOver(storage); + let lists = 0; + const other = sweepOver({ + ...storage, + persons: { + ...storage.persons, + listIdle: (args) => { + lists += 1; + return storage.persons.listIdle(args); + }, + }, + }); expect(await other.sweep(clock.now())).toBe(0); + expect(await other.sweep(clock.now())).toBe(0); + expect(lists).toBe(1); expect(await other.nextDueAt()).toEqual(new Date(leased)); await one.release(); expect(await other.sweep(clock.now())).toBe(1); @@ -853,6 +865,24 @@ describe("the retention across instances", () => { expect(await sweep.sweep(clock.now())).toBe(1); }); + it("sweeps again at once after a restore, though the store had paused it", async () => { + const { storage, sweepOver, clock } = await idle(["c-1"]); + let down = true; + const sweep = sweepOver({ + ...storage, + persons: { + ...storage.persons, + listIdle: (args) => + down ? Promise.reject(new Error("down")) : storage.persons.listIdle(args), + }, + }); + expect(await sweep.sweep(clock.now())).toBe(0); + down = false; + expect(await sweep.sweep(clock.now())).toBe(0); + await sweep.restored(false); + expect(await sweep.sweep(clock.now())).toBe(1); + }); + it("logs holds it could not clear after a restore, and goes on", async () => { const { storage, sweepOver, entries } = await idle([]); await sweepOver({ diff --git a/packages/core/src/kernel/personal-data/rotation.test.ts b/packages/core/src/kernel/personal-data/rotation.test.ts index 92f8069f..c7112287 100644 --- a/packages/core/src/kernel/personal-data/rotation.test.ts +++ b/packages/core/src/kernel/personal-data/rotation.test.ts @@ -797,6 +797,26 @@ describe("the key rotation across instances", () => { expect(await kidsOf(store, ["c-1", "c-2"])).toEqual(["new", "new"]); }); + it("rewraps again at once after a restore, though the store had paused it", async () => { + const { store, checkpoints } = await setUp(); + await addUnder(store, "old", "c-1"); + let down = true; + const rotation = rotationOver({ + keys: { + ...store, + listWrappedUnder: (args) => + down ? Promise.reject(new Error("down")) : store.listWrappedUnder(args), + }, + checkpoints, + wrapper: wrapper(), + }); + expect(await rotation.sweep(clock.now())).toBe(0); + down = false; + expect(await rotation.sweep(clock.now())).toBe(0); + await rotation.restored(false); + expect(await rotation.sweep(clock.now())).toBe(1); + }); + it("leaves the holds to the instance that found the restore first", async () => { const { store, checkpoints } = await setUp(); const held = await addUnder(store, "old", "c-1"); diff --git a/packages/core/src/kernel/shared/lease.test.ts b/packages/core/src/kernel/shared/lease.test.ts new file mode 100644 index 00000000..3e871310 --- /dev/null +++ b/packages/core/src/kernel/shared/lease.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { memory } from "../../memory/index.ts"; +import { createRecordingLogger } from "../test-support.ts"; +import { createLease, LEASE_MS } from "./lease.ts"; + +const start = Date.parse("2026-01-01T00:00:00.000Z"); + +const setUp = async (wrap: (store: CheckpointStore) => CheckpointStore = (store) => store) => { + const { checkpointStore } = await memory().createStorage(); + const writes: number[] = []; + const counted: CheckpointStore = { + ...checkpointStore, + compareAndSet: (row, expected, position) => { + writes.push(position); + return checkpointStore.compareAndSet(row, expected, position); + }, + }; + const { logger, entries } = createRecordingLogger("omit"); + const clock = createFixedClock(new Date(start)); + const lease = createLease({ checkpoints: wrap(counted), row: "job:lease", clock, logger }); + return { checkpointStore, writes, lease, entries, clock }; +}; + +describe("createLease", () => { + it("renews only once half of it is spent, writing nothing before", async () => { + const { lease, writes } = await setUp(); + expect(await lease.take(start)).toBe(true); + expect(await lease.take(start + LEASE_MS / 2 - 1)).toBe(true); + expect(await lease.take(start + LEASE_MS / 2)).toBe(true); + expect(writes).toEqual([start + LEASE_MS, start + LEASE_MS / 2 + LEASE_MS]); + }); + + it("takes a lease written more than a lease ahead, by a clock that runs ahead", async () => { + const { lease, checkpointStore } = await setUp(); + await checkpointStore.set("job:lease", start + LEASE_MS + 1); + expect(await lease.take(start)).toBe(true); + expect(await checkpointStore.get("job:lease")).toBe(start + LEASE_MS); + await checkpointStore.set("job:lease", start + 2 * LEASE_MS); + lease.reset(); + expect(await lease.take(start + LEASE_MS)).toBe(false); + expect(lease.heldElsewhereUntil()).toBe(start + 2 * LEASE_MS); + }); + + it("gives back nothing it does not hold, and logs a lease it could not give back", async () => { + let refusing = false; + const { lease, writes, entries } = await setUp((store) => ({ + ...store, + compareAndSet: (row, expected, position) => + refusing + ? Promise.reject(new Error("cannot write")) + : store.compareAndSet(row, expected, position), + })); + await lease.release(); + expect(writes).toEqual([]); + expect(await lease.take(start)).toBe(true); + refusing = true; + await lease.release(); + expect(entries).toEqual([ + { + level: "error", + message: "background job could not give its lease back", + fields: expect.objectContaining({ lease: "job:lease", errorName: "Error" }), + }, + ]); + await lease.release(); + expect(entries).toHaveLength(1); + }); +}); diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts new file mode 100644 index 00000000..c3a2f2f2 --- /dev/null +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import { memory } from "../../memory/index.ts"; +import { createRecordingLogger } from "../test-support.ts"; +import { + createRestoreWatch, + GENERATION_INTERVAL_MS, + RESTORE_GENERATION, + RESTORES_FOUND, +} from "./restores.ts"; + +const at = (ms: number) => new Date(Date.parse("2026-01-01T00:00:00.000Z") + ms); + +describe("createRestoreWatch", () => { + it("moves the generation at most once an interval", async () => { + const { checkpointStore } = await memory().createStorage(); + const { logger } = createRecordingLogger("omit"); + const watch = createRestoreWatch({ checkpoints: checkpointStore, logger }); + await watch.check(at(0)); + await watch.check(at(GENERATION_INTERVAL_MS - 1)); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); + await watch.check(at(GENERATION_INTERVAL_MS)); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(2); + }); + + it("counts a restore once when two instances find it at the same moment, and then nothing more", async () => { + const { checkpointStore } = await memory().createStorage(); + const { logger } = createRecordingLogger("omit"); + const one = createRestoreWatch({ checkpoints: checkpointStore, logger }); + const other = createRestoreWatch({ checkpoints: checkpointStore, logger }); + await one.check(at(0)); + await other.check(at(0)); + await checkpointStore.set(RESTORE_GENERATION, 0); + const found = await Promise.all([one.check(at(1)), other.check(at(1))]); + expect(found.map((each) => each?.first).sort()).toEqual([false, true]); + expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); + expect(await Promise.all([one.check(at(2)), other.check(at(2))])).toEqual([null, null]); + }); + + it("logs a store it could not read, and finds nothing", async () => { + const { checkpointStore } = await memory().createStorage(); + const { logger, entries } = createRecordingLogger("omit"); + const failing: CheckpointStore = { + ...checkpointStore, + get: () => Promise.reject(new Error("store is down")), + }; + const watch = createRestoreWatch({ checkpoints: failing, logger }); + expect(await watch.check(at(0))).toBeNull(); + expect(entries).toEqual([ + { + level: "error", + message: "the database could not be checked for a restore", + fields: expect.objectContaining({ errorName: "Error" }), + }, + ]); + }); +}); diff --git a/packages/core/src/kernel/status.test.ts b/packages/core/src/kernel/status.test.ts index 7c44430b..e04fd592 100644 --- a/packages/core/src/kernel/status.test.ts +++ b/packages/core/src/kernel/status.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from "vitest"; import { createNodeSqliteAdapter } from "../adapter/sqlite/node-sqlite.ts"; import { resolveConfig } from "../config/schema.ts"; import { memory } from "../memory/index.ts"; -import { databasesOf } from "./status.ts"; +import { databasesOf, readAppStatus } from "./status.ts"; describe("databasesOf", () => { it("names each database once, an adapter over the storage's own database included", () => { @@ -26,3 +26,71 @@ describe("databasesOf", () => { ]); }); }); + +describe("readAppStatus", () => { + const failure = (subscriber: string, position: number, resolvedAt: string | null = null) => ({ + subscriber, + position, + eventId: `e-${position}`, + eventType: "OrderPlaced", + person: null, + errorName: "Error", + errorMessage: "boom", + attempts: 1, + firstFailedAt: "2026-01-01T00:00:00.000Z", + lastFailedAt: "2026-01-01T00:00:00.000Z", + resolvedAt, + }); + + it("says a failure is current only while its subscriber is behind it, unresolved and still run", async () => { + const { subscriberFailures } = await memory().createStorage(); + for (const each of [ + failure("projection:behind", 5), + failure("projection:on", 4), + failure("projection:past", 3), + failure("projection:resolved", 9, "2026-01-02T00:00:00.000Z"), + failure("projection:gone", 9), + ]) { + const { attempts: _a, firstFailedAt, lastFailedAt: _l, resolvedAt, ...recorded } = each; + await subscriberFailures.record({ ...recorded, failedAt: firstFailedAt }); + if (resolvedAt !== null) { + await subscriberFailures.resolve(each.subscriber, new Date(resolvedAt)); + } + } + const lag = (subscriber: string, position: number) => ({ + subscriber, + position, + lag: 9 - position, + }); + const status = await readAppStatus({ + // A definition without factories, as a Cloudflare one is outside its object: no tables to read. + config: resolveConfig({ storage: { kind: "bounda-adapter", name: "x", options: {} } }), + lag: async () => ({ + lastPosition: 9, + maxLag: 5, + subscribers: [ + lag("projection:behind", 4), + lag("projection:on", 4), + lag("projection:past", 4), + lag("projection:resolved", 4), + ], + waitingDeadlines: 0, + forgetLedger: null, + personalDataBackfill: null, + }), + failures: subscriberFailures, + restores: { check: async () => null, found: async () => 0 }, + }); + expect( + Object.fromEntries(status.subscriberFailures.map((each) => [each.subscriber, each.current])), + ).toEqual({ + "projection:behind": true, + "projection:gone": false, + "projection:on": false, + "projection:past": false, + "projection:resolved": false, + }); + expect(status.schema).toEqual([]); + expect(status.subscriberFailures[0]).not.toHaveProperty("errorMessage"); + }); +}); From 03085011f999b0cc72bb6f3512ca8253250bd477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:17:47 +0100 Subject: [PATCH 09/19] test: type the access records of the subscriber failures' test --- .../core/src/kernel/dispatch/subscriber-failures.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/kernel/dispatch/subscriber-failures.test.ts b/packages/core/src/kernel/dispatch/subscriber-failures.test.ts index b3a42c17..d02f4b89 100644 --- a/packages/core/src/kernel/dispatch/subscriber-failures.test.ts +++ b/packages/core/src/kernel/dispatch/subscriber-failures.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from "vitest"; import { createFixedClock } from "../../contracts/clock.ts"; import { memory } from "../../memory/index.ts"; -import type { RecordAccessArgs } from "../personal-data/access.ts"; import { createPersonalKeyWrapping } from "../personal-data/wrapped-keys.ts"; import { createSubscriberFailures } from "./subscriber-failures.ts"; @@ -18,15 +17,15 @@ describe("createSubscriberFailures", () => { errorMessage: "x is not a function", failedAt: "2026-01-01T00:00:00.000Z", }); - const records: RecordAccessArgs<"subscriberFailures.list" | "subscriberFailures.get">[] = []; + const records: { readonly operation: string; readonly persons: Iterable }[] = []; const failures = createSubscriberFailures({ store: storage.subscriberFailures, keys: createPersonalKeyWrapping({ wrapper: null, clock: createFixedClock() }).over( storage.personalKeys, ), actor: "ops-1", - recordAccess: async (args) => { - records.push(args as (typeof records)[number]); + recordAccess: async ({ operation, persons }) => { + records.push({ operation, persons }); }, errorText: "omit", }); From 529dd84c6ff2500cc156823097b076b9e4efaf2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:31:15 +0100 Subject: [PATCH 10/19] feat(core): inspect the subscriber failures for expectForgotten --- packages/core/src/adapter/storage/inspection.ts | 2 ++ .../core/src/adapter/testing/inspection.contract.ts | 11 +++++++++++ packages/core/src/memory/index.ts | 8 +++++++- packages/core/src/memory/subscriber-failure-store.ts | 10 +++++++++- packages/core/src/testing/expect-forgotten.ts | 2 ++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/core/src/adapter/storage/inspection.ts b/packages/core/src/adapter/storage/inspection.ts index 4b853be5..25405fd6 100644 --- a/packages/core/src/adapter/storage/inspection.ts +++ b/packages/core/src/adapter/storage/inspection.ts @@ -12,6 +12,7 @@ export type InspectedStore = | "personalKeys" | "persons" | "snapshots" + | "subscriberFailures" | "personRows" | "forgetLedger" | "migrations" @@ -30,6 +31,7 @@ export const STORAGE_STORES: readonly Exclude[] = [ "personalKeys", "persons", "snapshots", + "subscriberFailures", "personRows", "forgetLedger", "migrations", diff --git a/packages/core/src/adapter/testing/inspection.contract.ts b/packages/core/src/adapter/testing/inspection.contract.ts index cd1ebda7..410f903a 100644 --- a/packages/core/src/adapter/testing/inspection.contract.ts +++ b/packages/core/src/adapter/testing/inspection.contract.ts @@ -71,6 +71,16 @@ export const inspectionContract: InspectionContractFunction = ({ create }) => { state: "{}", persons: [], }); + await storage.subscriberFailures.record({ + subscriber: "projection:orders", + position: 1, + eventId: "e-1", + eventType: "OrderPlaced", + person: null, + errorName: "Error", + errorMessage: "boom", + failedAt: now.toISOString(), + }); const tables = await adapter.inspect(); for (const { name } of tables) expect(name.startsWith("bounda_")).toBe(true); for (const [name, store] of [ @@ -83,6 +93,7 @@ export const inspectionContract: InspectionContractFunction = ({ create }) => { ["bounda_personal_keys", "personalKeys"], ["bounda_persons", "persons"], ["bounda_snapshots", "snapshots"], + ["bounda_subscriber_failures", "subscriberFailures"], ] as const) { const table = tableOf(tables, name); expect(table.store, name).toBe(store); diff --git a/packages/core/src/memory/index.ts b/packages/core/src/memory/index.ts index 2a379d55..18719dfc 100644 --- a/packages/core/src/memory/index.ts +++ b/packages/core/src/memory/index.ts @@ -135,6 +135,7 @@ export const memory: MemoryFunction = (options = {}) => { const persons = createKeptPersonStore(); const idempotencyStore = createKeptIdempotencyStore({ holds: personalKeys.holds }); const snapshotStore = createMemorySnapshotStore({ holds: personalKeys.holds }); + const subscriberFailures = createMemorySubscriberFailureStore(); const stores = { eventStore: createMemoryEventStore({ onAppend: notifier.notify, @@ -156,6 +157,11 @@ export const memory: MemoryFunction = (options = {}) => { key: ["aggregateType", "aggregateId"], rows: snapshotStore.rows(), }), + storageTable({ + table: "subscriberFailures", + key: ["subscriber"], + rows: subscriberFailures.rows(), + }), storageTable({ table: "inbox", key: ["handler", "eventId"], @@ -189,7 +195,7 @@ export const memory: MemoryFunction = (options = {}) => { notifier, checkpointStore, snapshotStore, - subscriberFailures: createMemorySubscriberFailureStore(), + subscriberFailures, transact: createMemoryStorageTransaction({ ...stores, entries: { diff --git a/packages/core/src/memory/subscriber-failure-store.ts b/packages/core/src/memory/subscriber-failure-store.ts index e76ad248..16676a84 100644 --- a/packages/core/src/memory/subscriber-failure-store.ts +++ b/packages/core/src/memory/subscriber-failure-store.ts @@ -4,8 +4,15 @@ import type { } from "../adapter/storage/subscriber-failure-store.ts"; import { jsonCopy } from "./json-copy.ts"; +export interface MemorySubscriberFailureStore extends SubscriberFailureStore { + /** + * Every row as kept, for `inspect`. + */ + rows(): readonly StoredSubscriberFailure[]; +} + export interface CreateMemorySubscriberFailureStoreFunction { - (): SubscriberFailureStore; + (): MemorySubscriberFailureStore; } export const createMemorySubscriberFailureStore: CreateMemorySubscriberFailureStoreFunction = @@ -39,5 +46,6 @@ export const createMemorySubscriberFailureStore: CreateMemorySubscriberFailureSt [...failures.values()] .sort((a, b) => (a.subscriber < b.subscriber ? -1 : 1)) .map((kept) => jsonCopy(kept)), + rows: () => [...failures.values()], }; }; diff --git a/packages/core/src/testing/expect-forgotten.ts b/packages/core/src/testing/expect-forgotten.ts index d3a06082..80181410 100644 --- a/packages/core/src/testing/expect-forgotten.ts +++ b/packages/core/src/testing/expect-forgotten.ts @@ -208,6 +208,8 @@ const STORE_ADVICE: Readonly>> = { "A remembered answer quotes the value: keep personal data out of a rejection's message and say what went wrong with its code and ids.", snapshots: "A snapshot holds the value from a field left unmarked: mark it personal() where the state's value comes from.", + subscriberFailures: + "A subscriber's last failure quotes the value in the clear: its event names the person by no marked field holding a value, so mark the field the value comes from personal(), and keep personal data out of error messages.", }; const opened = async (witness: TestWitness, personId: string, held: string | null) => { From 5bbf2573832dfeec7c17c4c962e278c3b83139c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:41:27 +0100 Subject: [PATCH 11/19] feat: find a restore made while the app was stopped, watch for one in the web role, and move the generation only after activity --- .changeset/personal-data-operations.md | 4 +- docs/src/content/docs/adapters/index.mdx | 2 +- docs/src/content/docs/guides/deployment.mdx | 5 +- docs/src/content/docs/guides/personal-data.md | 58 +++-- docs/src/content/docs/reference/cli.md | 27 ++- .../src/content/docs/reference/limitations.md | 6 +- packages/cli/src/run.test.ts | 26 +++ packages/cli/src/run.ts | 25 +++ packages/cloudflare/src/forget-ledger.ts | 115 +++++----- .../core/src/adapter/sql/forget-ledger.ts | 39 ++++ packages/core/src/adapter/sql/index.ts | 7 +- .../src/adapter/sqlite/migrations.test.ts | 7 +- .../core/src/adapter/sqlite/migrations.ts | 3 +- .../core/src/adapter/sqlite/sqlite.test.ts | 6 +- .../core/src/adapter/storage/forget-ledger.ts | 10 + .../core/src/adapter/storage/inspection.ts | 2 + .../testing/checkpoint-store.contract.ts | 52 ++++- .../adapter/testing/forget-ledger.contract.ts | 13 ++ .../testing/storage-schema.contract.ts | 7 +- packages/core/src/config/schema.test.ts | 7 +- packages/core/src/kernel/app.test.ts | 5 +- packages/core/src/kernel/app.ts | 58 ++++- .../dispatch/subscriber-failures-app.test.ts | 2 +- packages/core/src/kernel/migrate.test.ts | 8 +- .../personal-data/ledger-append.test.ts | 6 + .../personal-data/ledger-outbox.test.ts | 18 +- .../personal-data/reconciliation-app.test.ts | 22 ++ .../personal-data/reconciliation.test.ts | 6 + packages/core/src/kernel/scheduler/worker.ts | 15 +- .../src/kernel/shared/restores-app.test.ts | 129 ++++++++++++ .../core/src/kernel/shared/restores.test.ts | 140 ++++++++++-- packages/core/src/kernel/shared/restores.ts | 199 +++++++++++++++--- packages/core/src/kernel/status.test.ts | 8 +- packages/core/src/memory/forget-ledger.ts | 6 + packages/core/test-types/config.test-d.ts | 8 + packages/core/test-types/inference.test-d.ts | 9 + packages/postgresql/src/migrations.ts | 2 + 37 files changed, 894 insertions(+), 168 deletions(-) diff --git a/.changeset/personal-data-operations.md b/.changeset/personal-data-operations.md index 12832d9e..3c022679 100644 --- a/.changeset/personal-data-operations.md +++ b/.changeset/personal-data-operations.md @@ -9,8 +9,8 @@ Operate the runtime's background work as one: share a lease, start over after a restore, and say where it stands. - The backfill, the key rotation, the retention and the purge of resolved dead letters each run in one instance at a time, under a lease kept in `bounda_checkpoints`; an instance that stalls past it stops at its next item. -- A database restored under the running app is found by every instance on its next pass: the forget ledger is reconciled at once, the jobs forget what they knew, and the first instance clears the holds the copy brought back. `PersonStore.releaseHolds`/`countIdle` and `PersonalKeyStore.releaseHolds`/`countWrappedUnder` are new (breaking for custom adapters). +- A restored database is found and the background work starts over from it: the forget ledger is reconciled at once, the jobs forget what they knew, and the first instance clears the holds the copy brought back. Under the running app every instance finds it on its next pass, the `web` role through a watch of its own; restored while the app was stopped, the forget ledger's mark tells it at start, and without a ledger `bounda restored` (`app.restored()`) declares it. The generation behind it moves only after activity. `ForgetLedger.readMark`/`raiseMark` (forget-ledger migration 2), `PersonStore.releaseHolds`/`countIdle` and `PersonalKeyStore.releaseHolds`/`countWrappedUnder` are new (breaking for custom adapters and ledgers). - Discarding a process dead letter ends its instance with `ProcessAbandoned` in the same transaction; `ProcessStatus` gains `"abandoned"`, and the purge of resolved letters no longer decides whether an instance was given up. - Each subscriber's last failure is kept in `bounda_subscriber_failures` (storage migration 2), its text sealed under the person the event names whatever `runtime.telemetry.errorText` says. `app.subscriberFailures.list()`/`.get()` open it and are recorded in the access log (`subscriberFailures.list`, `subscriberFailures.get`); the export includes it as `subscriberFailures`. `Storage.subscriberFailures` is new (breaking for custom adapters), and `Delivery` failures carry the event and the error. - `app.status()`, also on `connect` clients, reports the tables' versions by database, the restores found, every subscriber's lag and last failure without its text, and the backfill, ledger, rotation, retention and purge. `Adapter.schemaVersions()` and `Adapter.migrate(scopes)` are new (breaking for custom adapters), and `migrateTables({ config })` migrates every database a configuration names. -- `bounda status` prints it, with `--json`, and `--error-text` the failures' text through the recorded call; `bounda migrate` migrates the tables before a deploy, reading only the configuration. +- `bounda status` prints it, with `--json`, and `--error-text` the failures' text through the recorded call; `bounda migrate` migrates the tables before a deploy, reading only the configuration; `bounda restored` declares a restore. diff --git a/docs/src/content/docs/adapters/index.mdx b/docs/src/content/docs/adapters/index.mdx index f172bca3..b2f743ff 100644 --- a/docs/src/content/docs/adapters/index.mdx +++ b/docs/src/content/docs/adapters/index.mdx @@ -86,7 +86,7 @@ it brings them up to the version the installed Bounda expects | Table | Holds | | --- | --- | | `bounda_events` | Every event, with its stream version, a global position and the person it names, if any, which [`exportPersonalData`](/guides/personal-data/#exporting-a-persons-data) reads by | -| `bounda_checkpoints` | How far each projection, the policy runner, the process runner, the retention and the [forget ledger](/guides/personal-data/#backups)'s copy have read, where a paused rebuild stands, the time of the last ledger entry the database applied, when the last reconciliation ran and the wait of one that failed, when the [key rotation](/guides/personal-data/#rotating-the-master-key) was last found complete for a set of master keys, the leases of the [background jobs](/guides/deployment/#more-than-one-instance), and the counters that tell a [restore under the running app](/guides/personal-data/#restores-while-running) | +| `bounda_checkpoints` | How far each projection, the policy runner, the process runner, the retention and the [forget ledger](/guides/personal-data/#backups)'s copy have read, where a paused rebuild stands, the time of the last ledger entry the database applied, when the last reconciliation ran and the wait of one that failed, when the [key rotation](/guides/personal-data/#rotating-the-master-key) was last found complete for a set of master keys, the leases of the [background jobs](/guides/deployment/#more-than-one-instance), and the counters that tell a [restore](/guides/personal-data/#restores) | | `bounda_inbox` | Which handler already completed for which event, so a retry does not run it again | | `bounda_scheduled_commands` | Scheduled commands, process deadlines and timeouts, and delayed policy runs | | `bounda_dead_letters` | Handler runs that gave up, with the error and the attempt count | diff --git a/docs/src/content/docs/guides/deployment.mdx b/docs/src/content/docs/guides/deployment.mdx index 0cc5c21c..7ee551ea 100644 --- a/docs/src/content/docs/guides/deployment.mdx +++ b/docs/src/content/docs/guides/deployment.mdx @@ -39,7 +39,7 @@ everything — the right answer until it is not. | Role | What `start()` does | | --- | --- | | `all` (default) | runs the dispatcher and the scheduled command worker | -| `web` | nothing | +| `web` | watches for a [restore](/guides/personal-data/#restores) of the database, nothing else | | `worker` | runs the dispatcher and the scheduled command worker | Serving HTTP is your app's business, not Bounda's: the role only decides whether this process also @@ -359,7 +359,8 @@ as long as the backups. Restoring one taken before someone was forgotten brings the app has a [forget ledger](/guides/personal-data/#backups) in another database, which forgets them again when the app starts on the restored copy, or at once when the copy is restored under the running app: every running instance finds that the database went back, and its background jobs -start over (see [restores while running](/guides/personal-data/#restores-while-running)). Set the +start over (see [restores](/guides/personal-data/#restores)). Restored while the app was stopped, +the ledger tells it at the next start, and without a ledger run `bounda restored`. Set the ledger's `keepFor` longer than the oldest backup you keep. ## Observability diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index 16e273e3..6a24b8a1 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -885,7 +885,9 @@ Anything else is a ledger of your own through `defineForgetLedger`, which receiv environment, the tenant and `keepForMs` when the app starts: its `read` returns entries in the order of their time, then of their event's id, and `byForgetLedgerOrder` from `@bounda-dev/core/adapter` sorts them so; its optional `prune` removes the entries older than a -time; `forgetLedgerContract` from `@bounda-dev/core/adapter/testing` is the test it has to pass. +time; its `readMark` and `raiseMark` keep numbers that only go up, which the runtime uses to tell +a copy [restored while the app was stopped](#restores); `forgetLedgerContract` from +`@bounda-dev/core/adapter/testing` is the test it has to pass. ### What the runtime does with it @@ -922,7 +924,7 @@ kept in the database, so a restore takes it back too and cannot hold the next ro round runs on the worker's passes, in every instance whose role runs the worker; a Durable Object runs it on the passes it wakes for, never waking for it alone, since restoring its storage restarts it anyway. A database restored under a running app is reconciled at once, on the next -pass that [finds the restore](#restores-while-running). +pass that [finds the restore](#restores). A ledger that cannot be read when the app starts does not stop it: the app logs an error, starts, and its worker tries again after 30 seconds, then twice as long after each failure, up to an hour. @@ -988,39 +990,53 @@ bounda forget --from forgotten.jsonl unless the database holds the forgetting a line names. A line may also be a bare person's id, for a list kept some other way, which is forgotten whenever the person holds a key: list only those forgotten after the copy was taken. `--actor ` names who does it on each `PersonalDataForgotten` -it appends. `app.forgetAgain(entries, { actor })` is the same in code. +it appends. `app.forgetAgain(entries, { actor })` is the same in code. Then run `bounda restored`, +so the jobs start over from the copy ([restores](#restores)). Without either, a restored backup brings back the data of everyone forgotten after it was taken, until they are forgotten again. -### Restores while running +### Restores -Every instance that runs the worker moves a counter in `bounda_checkpoints` one forward, at most -every five seconds, and remembers what it read. A copy restored under the running app holds less -than that, so the next pass of each instance finds the database went back; the first to find it -counts the restore in another row, which every other instance then sees changed, whatever its own -reading shows. Each instance then: +A copy of the database brings back more than forgotten persons: the waits of the jobs that had +failed, which count failures against data the copy holds as it was. So the runtime finds a +restore, wherever it happens, and then: -- reconciles the forget ledger at once, without waiting for `reconcileEvery` or for a hold the copy +- reconciles the forget ledger at once, without waiting for `reconcileEvery` or for a wait the copy brought back; -- forgets what it knew of its background jobs, the backfill caught up, the rotation complete, the - waits after a failure, the dispatcher's backoff, and so reads the copy again on that pass; -- and, on the instance that found it first, clears the holds the copy brought back, on persons, - keys, the backfill and the reconciliation: they count failures against data the copy no longer - holds as it was. +- has every instance forget what it knew of its background jobs, the backfill caught up, the + rotation complete, the pauses after a failure, the dispatcher's backoff, and read the copy again; +- and, on the instance that found it first, clears the waits the copy brought back, on persons, + keys, the backfill and the reconciliation. The attempts of handlers, a policy's, a process's or a scheduled command's, stay as the copy has -them: those runs happened, and what they did through a port is not undone. It logs: +them: those runs happened, and what they did through a port is not undone. Each find logs ``` the database went back to an earlier copy; background jobs start over ``` -and `bounda status` counts the restores found. A copy taken within five seconds of being restored -may hold the counter every instance read last, and goes unnoticed: what it undid is that recent. -A copy restored while the app was stopped is not told apart from a database left alone, since no -running instance remembers what it held; the start reconciles the ledger whatever its hold says, -and the holds the copy brought back end on their own, within an hour at most. +and `bounda status` counts the restores found. How it is found depends on when the copy came back: + +| The copy was restored | Found by | What is guaranteed | +| --- | --- | --- | +| under the running app | every instance, on its next pass, or within five seconds in the `web` role | Any copy taken before the last write the instances saw. A copy taken after it differs from the database in nothing but waits, which the jobs read again anyway. | +| while the app was stopped, with a forget ledger | the first instance to start, before it serves anything | Any copy taken before the last mark the ledger holds: raised at most once a minute while the app writes, and when it stops. A copy taken in the last minute before a crash goes unseen, as above. | +| while the app was stopped, without a ledger | `bounda restored`, which you run after restoring | What you declare: the database alone cannot tell a copy from itself. | + +Under the running app, every instance moves a counter in `bounda_checkpoints` one forward after +it wrote, at most every five seconds, and remembers what it read: a copy holds less, and the +first instance to see it counts the restore in another row, which every other instance then sees +changed. An idle instance writes nothing, so neither does its watch. With a forget ledger the +counter is also raised in the ledger, as a mark outside the database, which is how a start tells +a copy restored while the app was stopped. Without one, run, after restoring: + +```bash +bounda restored +``` + +`app.restored()` is the same in code. It counts the restore, so running instances learn of it +too, clears the waits, and reconciles the ledger at once when there is one. ### Old backups diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index bf37c33b..821d7aab 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -1,6 +1,6 @@ --- title: CLI -description: "The bounda command: generate the registry and the types, rebuild a read model, deal with dead letters, read the runtime's status and migrate its tables." +description: "The bounda command: generate the registry and the types, rebuild a read model, deal with dead letters, read the runtime's status, migrate its tables and declare a restore." sidebar: order: 0 --- @@ -277,7 +277,7 @@ dead letter retention: 30d, next purge 2026-11-20T08:00:00.000Z the database holds of the one this release migrates it to; one behind says `(migrations pending: bounda migrate)`. A store without tables, as `memory()`, has none. - `restores found while running`: how many times a running instance found the database gone back - to an earlier copy ([restores while running](/guides/personal-data/#restores-while-running)). + to an earlier copy ([restores](/guides/personal-data/#restores)), or declared by `bounda restored`. - each subscriber's checkpoint and lag, and its last failure, by any instance: `failing` while it is still stuck on it, `last failed, resolved …` once it got past. Its class and code, the event, how many deliveries in a row failed and since when; never its text. @@ -298,6 +298,27 @@ person was forgotten says its text went with them. | `--root`, `--config`, `--registry`, `--error-text` | As for `bounda rebuild` | | `--actor ` | Who reads the failures' text with `--error-text`, named in the access log | +## `bounda restored` + +Says the database was restored from a copy while the app was stopped, for an app without a +[forget ledger](/guides/personal-data/#backups), whose mark tells it on its own. It boots the +project as `bounda dead-letters` does. + +```bash +bounda restored +``` + +``` +restore recorded: 1 found so far; the background jobs start over from the copy, without the holds it brought back +``` + +The restore is counted, so every running instance starts its background jobs over; the holds of +failures the copy brought back are cleared; and with a ledger, it is reconciled at once. Run it +after `bounda forget --from` when restoring without a ledger (see +[restores](/guides/personal-data/#restores)). Options: `--root`, `--config`, `--registry`, +`--error-text`, as for `bounda rebuild`. Exit codes: `0` recorded, `2` the project could not be +loaded. + ## `bounda migrate` Brings Bounda's tables to this release's version in every database the configuration names, as @@ -385,7 +406,7 @@ It throws `ConventionError` when the layout breaks a convention; `formatConventi `actor` for `--actor`; an app exposes the same as `app.rebuildReadModel(name)`, and `app.as(actor).rebuildReadModel(name)`. `bounda status` is `app.status()`, and its failures' text `app.as(actor).subscriberFailures.list()`; `bounda migrate` is `migrateTables({ config })` from -`@bounda-dev/core`. `bounda dead-letters` is `app.deadLetters` on a booted app, +`@bounda-dev/core`; `bounda restored` is `app.restored()`. `bounda dead-letters` is `app.deadLetters` on a booted app, or `app.as(actor).deadLetters`, and `bounda forget` is `app.forgetAgain(entries, { actor })`. The commands boot the app with `createConsoleLogger({ stream: process.stderr })` from `@bounda-dev/core/node`, which writes every level to the stream it is given. diff --git a/docs/src/content/docs/reference/limitations.md b/docs/src/content/docs/reference/limitations.md index a87f8194..f579a968 100644 --- a/docs/src/content/docs/reference/limitations.md +++ b/docs/src/content/docs/reference/limitations.md @@ -22,10 +22,6 @@ get today. Each item says why, so nobody discovers it the hard way: `pollInterval`; only the event dispatcher is woken by `NOTIFY`. See [Tuning](/guides/deployment/#tuning). - **Every read of personal data in the access log.** See [The access log](#the-access-log). -- **A restore made while the app was stopped, told apart.** No running instance remembers what the - database held, so the jobs start from the copy as from any start, and the holds of failures it - brought back end on their own, within an hour. See - [Restores while running](/guides/personal-data/#restores-while-running). ## The access log @@ -58,7 +54,7 @@ What a production app does get, and where it is explained: reaction, [forgetting them](/guides/personal-data/#retention) after a retention or when a purpose ends, their keys [wrapped under a master key](/guides/personal-data/#a-master-key) kept out of the database and out of JavaScript memory, their forgetting [applied again](/guides/personal-data/#backups) when a backup is -restored, and found when restored [under the running app](/guides/personal-data/#restores-while-running), +restored, and [found](/guides/personal-data/#restores) whether restored under the running app or while it was stopped, what was written before a field was marked [sealed in place](/guides/personal-data/#marking-a-field-later), [the status of the background work](/reference/cli/#bounda-status) and [migrations run before a deploy](/reference/cli/#bounda-migrate), diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 8f3c51f7..9370f794 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -924,3 +924,29 @@ export default { storage: adapter }; } }); }); + +describe("bounda restored", () => { + const fixture = resolve(import.meta.dirname, "fixtures/rebuild-project"); + + it("records a restore, which the status counts", async () => { + const before = JSON.parse((await cli(["status", "--json"], fixture)).stdout).restoresFound; + const result = await cli(["restored"], fixture); + expect(result.code).toBe(EXIT_OK); + expect(besidesLogs(result.stderr)).toBe(""); + expect(result.stdout).toBe( + `restore recorded: ${before + 1} found so far; the background jobs start over from the copy, without the holds it brought back\n`, + ); + expect(result.stderr).toContain( + '[bounda] warn the database went back to an earlier copy; background jobs start over {"first":true}', + ); + }); + + it("documents itself in its help", async () => { + const help = await cli(["restored", "--help"], fixture); + expect(help.code).toBe(EXIT_OK); + expect(help.stdout.replace(/\s+/g, " ")).toContain( + "say the database was restored from a copy while the app was stopped", + ); + expect(help.stdout).toContain("--registry "); + }); +}); diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index acee39f3..0bd68158 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -399,6 +399,21 @@ const runStatus = ( return EXIT_OK; }); +const runRestored = ( + options: ProjectOptions, + cwd: string, + stdout: Output, + stderr: Output, +): Promise => + withApp(options, cwd, stderr, async (app) => { + await app.restored(); + line( + stdout, + `restore recorded: ${(await app.status()).restoresFound} found so far; the background jobs start over from the copy, without the holds it brought back`, + ); + return EXIT_OK; + }); + // Only the configuration is read: the tables are migrated before any app of this release starts. const NO_REGISTRY: Registry = { aggregates: {}, readModels: {} }; @@ -598,6 +613,16 @@ export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal exitCode = await runStatus(options, cwd, stdout, stderr); }); + projectOptions( + program + .command("restored") + .description( + "say the database was restored from a copy while the app was stopped: every instance starts its background jobs over, without the holds the copy brought back", + ), + ).action(async (options: ProjectOptions) => { + exitCode = await runRestored(options, cwd, stdout, stderr); + }); + program .command("migrate") .description( diff --git a/packages/cloudflare/src/forget-ledger.ts b/packages/cloudflare/src/forget-ledger.ts index 98bbacfd..12f0d4f4 100644 --- a/packages/cloudflare/src/forget-ledger.ts +++ b/packages/cloudflare/src/forget-ledger.ts @@ -15,6 +15,7 @@ export const KV_LIST_DELAY_MS = 60_000; // entry lists first. Any time from 1970 on leaves sixteen digits, which sort as numbers do. const LATEST_MS = 8_640_000_000_000_000; const LIST_LIMIT = 1_000; +const MARKS_PREFIX = "bounda/forget-ledger-marks/"; // The soonest KV lets a key expire, in seconds from now. const KV_MIN_TTL_S = 60; @@ -53,59 +54,71 @@ export const createKvForgetLedger: CreateKvForgetLedgerFunction = ({ namespace, prefix, keepForMs, -}) => ({ - visibilityDelayMs: KV_LIST_DELAY_MS, - append: async (entries) => { - const now = Date.now(); - await Promise.all( - entries.flatMap(({ personId, eventId, forgottenAt }) => { - const metadata: EntryMetadata = { personId, forgottenAt }; - const key = `${prefix}${timeKey(forgottenAt)}:${eventId}`; - if (keepForMs === undefined) return [namespace.put(key, "", { metadata })]; - const expiresAt = Date.parse(forgottenAt) + keepForMs; - if (!(expiresAt > now)) return []; - const expiration = Math.max( - Math.ceil(expiresAt / 1000), - Math.floor(now / 1000) + KV_MIN_TTL_S, - ); - return [namespace.put(key, "", { metadata, expiration })]; - }), - ); - }, - read: async ({ after, limit }) => { - const oldest = after === undefined ? null : timeKey(after.forgottenAt); - const found: ForgetLedgerEntry[] = []; - let cursor: string | undefined; - for (;;) { - const page = await namespace.list({ - prefix, - limit: LIST_LIMIT, - ...(cursor === undefined ? {} : { cursor }), - }); - let past = false; - for (const { name, metadata } of page.keys) { - const rest = name.slice(prefix.length); - const separator = rest.indexOf(":"); - if (oldest !== null && rest.slice(0, separator) > oldest) { - past = true; - break; - } - if (metadata === null || metadata === undefined) continue; - found.push({ - personId: metadata.personId, - eventId: rest.slice(separator + 1), - forgottenAt: metadata.forgottenAt, +}) => { + // Apart from the entries' keys, so no list of entries meets one. + const markKey = (name: string): string => `${MARKS_PREFIX}${prefix}${name}`; + return { + visibilityDelayMs: KV_LIST_DELAY_MS, + append: async (entries) => { + const now = Date.now(); + await Promise.all( + entries.flatMap(({ personId, eventId, forgottenAt }) => { + const metadata: EntryMetadata = { personId, forgottenAt }; + const key = `${prefix}${timeKey(forgottenAt)}:${eventId}`; + if (keepForMs === undefined) return [namespace.put(key, "", { metadata })]; + const expiresAt = Date.parse(forgottenAt) + keepForMs; + if (!(expiresAt > now)) return []; + const expiration = Math.max( + Math.ceil(expiresAt / 1000), + Math.floor(now / 1000) + KV_MIN_TTL_S, + ); + return [namespace.put(key, "", { metadata, expiration })]; + }), + ); + }, + read: async ({ after, limit }) => { + const oldest = after === undefined ? null : timeKey(after.forgottenAt); + const found: ForgetLedgerEntry[] = []; + let cursor: string | undefined; + for (;;) { + const page = await namespace.list({ + prefix, + limit: LIST_LIMIT, + ...(cursor === undefined ? {} : { cursor }), }); + let past = false; + for (const { name, metadata } of page.keys) { + const rest = name.slice(prefix.length); + const separator = rest.indexOf(":"); + if (oldest !== null && rest.slice(0, separator) > oldest) { + past = true; + break; + } + if (metadata === null || metadata === undefined) continue; + found.push({ + personId: metadata.personId, + eventId: rest.slice(separator + 1), + forgottenAt: metadata.forgottenAt, + }); + } + if (past || page.list_complete) break; + cursor = page.cursor; } - if (past || page.list_complete) break; - cursor = page.cursor; - } - return found - .filter((entry) => after === undefined || byForgetLedgerOrder(entry, after) > 0) - .sort(byForgetLedgerOrder) - .slice(0, limit); - }, -}); + return found + .filter((entry) => after === undefined || byForgetLedgerOrder(entry, after) > 0) + .sort(byForgetLedgerOrder) + .slice(0, limit); + }, + readMark: async (name) => Number((await namespace.get(markKey(name))) ?? 0), + // KV has no compare: of two instances raising at once the lower may win, which the next raise, + // of a mark that only grows, puts right. + raiseMark: async (name, value) => { + const current = Number((await namespace.get(markKey(name))) ?? 0); + if (current >= value) return; + await namespace.put(markKey(name), String(value)); + }, + }; +}; export interface KvForgetLedgerOptions { /** diff --git a/packages/core/src/adapter/sql/forget-ledger.ts b/packages/core/src/adapter/sql/forget-ledger.ts index f8346d91..d7f464dc 100644 --- a/packages/core/src/adapter/sql/forget-ledger.ts +++ b/packages/core/src/adapter/sql/forget-ledger.ts @@ -28,6 +28,29 @@ export const forgetLedgerStatements: ForgetLedgerStatementsFunction = (table, { ]; }; +export interface ForgetLedgerMarksTableForFunction { + (table: string): string; +} + +/** + * The table of the ledger's marks, beside the ledger's own `table`, both quoted. + */ +export const forgetLedgerMarksTableFor: ForgetLedgerMarksTableForFunction = (table) => + quoteIdentifier(`${table.slice(1, -1)}_marks`); + +export interface ForgetLedgerMarksStatementFunction { + (table: string): string; +} + +// The second migration of the ledger's scope: its marks, one row per tenant and name. +export const forgetLedgerMarksStatement: ForgetLedgerMarksStatementFunction = (table) => + `CREATE TABLE IF NOT EXISTS ${forgetLedgerMarksTableFor(table)} ( + "tenant" TEXT NOT NULL, + "name" TEXT NOT NULL, + "value" BIGINT NOT NULL, + PRIMARY KEY ("tenant", "name") + )`; + export interface CreateSqlForgetLedgerArgs { readonly db: SqlExecutor; /** @@ -66,6 +89,7 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ close, }) => { const p = dialect.placeholder; + const marks = forgetLedgerMarksTableFor(table); let ready: Promise | undefined; const prepared = (): Promise => { ready ??= ensureSchema().catch((error: unknown) => { @@ -115,6 +139,21 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ before, ]); }, + readMark: async (name) => { + await prepared(); + const [row] = await db.all( + `SELECT "value" FROM ${marks} WHERE "tenant" = ${p(1)} AND "name" = ${p(2)}`, + [tenant, name], + ); + return Number(row?.value ?? 0); + }, + raiseMark: async (name, value) => { + await prepared(); + await db.run( + `INSERT INTO ${marks} ("tenant", "name", "value") VALUES (${p(1)}, ${p(2)}, ${p(3)}) ON CONFLICT ("tenant", "name") DO UPDATE SET "value" = EXCLUDED."value" WHERE ${marks}."value" < EXCLUDED."value"`, + [tenant, name, value], + ); + }, close, }; }; diff --git a/packages/core/src/adapter/sql/index.ts b/packages/core/src/adapter/sql/index.ts index 3c1ac128..b582384a 100644 --- a/packages/core/src/adapter/sql/index.ts +++ b/packages/core/src/adapter/sql/index.ts @@ -20,7 +20,12 @@ export type { CreateSqlForgetLedgerFunction, ForgetLedgerStatementsFunction, } from "./forget-ledger.ts"; -export { createSqlForgetLedger, forgetLedgerStatements } from "./forget-ledger.ts"; +export { + createSqlForgetLedger, + forgetLedgerMarksStatement, + forgetLedgerMarksTableFor, + forgetLedgerStatements, +} from "./forget-ledger.ts"; export { DEFAULT_TABLE_PREFIX, quoteIdentifier, diff --git a/packages/core/src/adapter/sqlite/migrations.test.ts b/packages/core/src/adapter/sqlite/migrations.test.ts index 2a5906b1..3290ae69 100644 --- a/packages/core/src/adapter/sqlite/migrations.test.ts +++ b/packages/core/src/adapter/sqlite/migrations.test.ts @@ -118,7 +118,12 @@ describe("migrateSqlite", () => { .prepare("SELECT name FROM sqlite_master WHERE name LIKE 'app_%' ORDER BY name") .all() .map((row) => row.name), - ).toEqual(["app_forget_ledger", "app_forget_ledger_order_idx", "app_migrations"]); + ).toEqual([ + "app_forget_ledger", + "app_forget_ledger_marks", + "app_forget_ledger_order_idx", + "app_migrations", + ]); }); }); diff --git a/packages/core/src/adapter/sqlite/migrations.ts b/packages/core/src/adapter/sqlite/migrations.ts index 5f82f938..8fc20368 100644 --- a/packages/core/src/adapter/sqlite/migrations.ts +++ b/packages/core/src/adapter/sqlite/migrations.ts @@ -2,7 +2,7 @@ import { ConfigurationError } from "../../contracts/errors.ts"; import type { SqlDatabase, SqlTransaction } from "../sql/database.ts"; import { resolvedAtOfEarlierLetters } from "../sql/dead-letters.ts"; import { sqliteDialect } from "../sql/dialect.ts"; -import { forgetLedgerStatements } from "../sql/forget-ledger.ts"; +import { forgetLedgerMarksStatement, forgetLedgerStatements } from "../sql/forget-ledger.ts"; import { quoteIdentifier } from "../sql/identifiers.ts"; import { migrateSchemas, @@ -152,6 +152,7 @@ export const sqliteForgetLedgerScope: SqliteForgetLedgerScopeFunction = (table) } }, }, + { version: 2, migrate: (tx) => tx.run(forgetLedgerMarksStatement(table), []) }, ], }); diff --git a/packages/core/src/adapter/sqlite/sqlite.test.ts b/packages/core/src/adapter/sqlite/sqlite.test.ts index ad1402b8..bd0f3e88 100644 --- a/packages/core/src/adapter/sqlite/sqlite.test.ts +++ b/packages/core/src/adapter/sqlite/sqlite.test.ts @@ -681,6 +681,10 @@ describe("the SQLite forget ledger", () => { const ledger = await adapter.createForgetLedger({ tenant: "t-1" }); expect(tableNames(db, "app_")).toEqual([]); await ledger.read({ limit: 1 }); - expect(tableNames(db, "app_")).toEqual(["app_forget_ledger", "app_migrations"]); + expect(tableNames(db, "app_")).toEqual([ + "app_forget_ledger", + "app_forget_ledger_marks", + "app_migrations", + ]); }); }); diff --git a/packages/core/src/adapter/storage/forget-ledger.ts b/packages/core/src/adapter/storage/forget-ledger.ts index a2d1f39c..a6ec584f 100644 --- a/packages/core/src/adapter/storage/forget-ledger.ts +++ b/packages/core/src/adapter/storage/forget-ledger.ts @@ -69,6 +69,16 @@ export interface ForgetLedger { * and expires each entry `keepForMs` after its `forgottenAt` instead. */ prune?(args: PruneForgetLedgerArgs): Promise; + /** + * The highest value raised under `name` for this ledger's store, `0` for none. What the runtime + * keeps of the database outside it: a copy restored while the app was stopped holds less than + * the ledger says the database reached. + */ + readMark(name: string): Promise; + /** + * Raises the mark `name` to `value`, unless it is already that high: a mark never goes down. + */ + raiseMark(name: string, value: number): Promise; /** * Releases what the ledger holds open; called when the app stops. */ diff --git a/packages/core/src/adapter/storage/inspection.ts b/packages/core/src/adapter/storage/inspection.ts index 25405fd6..8be6a517 100644 --- a/packages/core/src/adapter/storage/inspection.ts +++ b/packages/core/src/adapter/storage/inspection.ts @@ -15,6 +15,7 @@ export type InspectedStore = | "subscriberFailures" | "personRows" | "forgetLedger" + | "forgetLedgerMarks" | "migrations" | "readModel"; @@ -34,6 +35,7 @@ export const STORAGE_STORES: readonly Exclude[] = [ "subscriberFailures", "personRows", "forgetLedger", + "forgetLedgerMarks", "migrations", ]; diff --git a/packages/core/src/adapter/testing/checkpoint-store.contract.ts b/packages/core/src/adapter/testing/checkpoint-store.contract.ts index 4f50cae2..c2c159da 100644 --- a/packages/core/src/adapter/testing/checkpoint-store.contract.ts +++ b/packages/core/src/adapter/testing/checkpoint-store.contract.ts @@ -134,15 +134,21 @@ export const checkpointStoreContract: CheckpointStoreContractFunction = ({ creat expect(await leaseOver().take(renewed + 1)).toBe(false); }); + // A stream whose head the test moves, as appends do. + const stream = { head: 0 }; + const events = { lastPosition: async () => stream.head }; + const watchOver = () => + createRestoreWatch({ checkpoints: store, events, marks: null, logger: silentLogger }); + const at = (ms: number) => new Date(start + ms); + it("tells every instance that the database went back to an earlier copy, and one that it was first", async () => { - const one = createRestoreWatch({ checkpoints: store, logger: silentLogger }); - const other = createRestoreWatch({ checkpoints: store, logger: silentLogger }); - const at = (ms: number) => new Date(start + ms); - expect(await one.check(at(0))).toBeNull(); - expect(await other.check(at(0))).toBeNull(); + const one = watchOver(); + const other = watchOver(); + expect(await one.check(at(0), { active: true })).toBeNull(); + expect(await other.check(at(0), { active: true })).toBeNull(); const copy = await store.get(RESTORE_GENERATION); - expect(await one.check(at(GENERATION_INTERVAL_MS))).toBeNull(); - expect(await other.check(at(GENERATION_INTERVAL_MS))).toBeNull(); + expect(await one.check(at(GENERATION_INTERVAL_MS), { active: true })).toBeNull(); + expect(await other.check(at(GENERATION_INTERVAL_MS), { active: true })).toBeNull(); expect(await store.get(RESTORE_GENERATION)).toBe(copy + 2); await store.set(RESTORE_GENERATION, copy); expect(await one.check(at(GENERATION_INTERVAL_MS + 1))).toEqual({ first: true }); @@ -154,15 +160,41 @@ export const checkpointStoreContract: CheckpointStoreContractFunction = ({ creat }); it("tells a restore that took the count of restores back too", async () => { - const one = createRestoreWatch({ checkpoints: store, logger: silentLogger }); + const one = watchOver(); await store.set(RESTORES_FOUND, 3); await store.set(RESTORE_GENERATION, 10); - expect(await one.check(new Date(start))).toBeNull(); + expect(await one.check(at(0))).toBeNull(); await store.set(RESTORES_FOUND, 2); await store.set(RESTORE_GENERATION, 11); - expect(await one.check(new Date(start + 1))).toEqual({ first: true }); + expect(await one.check(at(1))).toEqual({ first: true }); expect(await store.get(RESTORES_FOUND)).toBe(4); }); + + it("moves the generation only after activity: a write to the stream or work of its own", async () => { + const one = watchOver(); + await one.check(at(0)); + await one.check(at(GENERATION_INTERVAL_MS)); + expect(await store.get(RESTORE_GENERATION)).toBe(0); + stream.head = 3; + await one.check(at(2 * GENERATION_INTERVAL_MS)); + expect(await store.get(RESTORE_GENERATION)).toBe(1); + await one.check(at(3 * GENERATION_INTERVAL_MS)); + expect(await store.get(RESTORE_GENERATION)).toBe(1); + await one.check(at(3 * GENERATION_INTERVAL_MS + 1), { active: true }); + expect(await store.get(RESTORE_GENERATION)).toBe(2); + await one.check(at(3 * GENERATION_INTERVAL_MS + 2), { active: true }); + expect(await store.get(RESTORE_GENERATION)).toBe(2); + await one.check(at(4 * GENERATION_INTERVAL_MS + 1)); + expect(await store.get(RESTORE_GENERATION)).toBe(3); + }); + + it("counts a restore an operator declares, which every running instance then learns", async () => { + const running = watchOver(); + await running.check(at(0)); + expect(await watchOver().declare()).toEqual({ first: true }); + expect(await running.check(at(1))).toEqual({ first: false }); + expect(await running.found()).toBe(1); + }); }); }); }; diff --git a/packages/core/src/adapter/testing/forget-ledger.contract.ts b/packages/core/src/adapter/testing/forget-ledger.contract.ts index 95b47796..040a51d4 100644 --- a/packages/core/src/adapter/testing/forget-ledger.contract.ts +++ b/packages/core/src/adapter/testing/forget-ledger.contract.ts @@ -135,5 +135,18 @@ export const forgetLedgerContract: ForgetLedgerContractFunction = ({ create }) = ]); expect(await third.read({ limit: 10 })).toEqual([]); }); + + it("raises a mark only upward, keeps it apart from the entries, and each tenant's apart", async () => { + const other = await ledgerFor("tenant-b"); + expect(await ledger.readMark("restore:generation")).toBe(0); + await ledger.raiseMark("restore:generation", 7); + await ledger.raiseMark("restore:generation", 5); + expect(await ledger.readMark("restore:generation")).toBe(7); + await ledger.raiseMark("restore:generation", 9); + expect(await ledger.readMark("restore:generation")).toBe(9); + expect(await ledger.readMark("another")).toBe(0); + expect(await other.readMark("restore:generation")).toBe(0); + expect(await ledger.read({ limit: 10 })).toEqual([]); + }); }); }; diff --git a/packages/core/src/adapter/testing/storage-schema.contract.ts b/packages/core/src/adapter/testing/storage-schema.contract.ts index c29b938f..f6fa5a94 100644 --- a/packages/core/src/adapter/testing/storage-schema.contract.ts +++ b/packages/core/src/adapter/testing/storage-schema.contract.ts @@ -149,18 +149,18 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ expect(versions).toEqual([ { scope: "storage", version: 0, latest: 2 }, { scope: "checkpoints", version: 0, latest: 1 }, - { scope: "forget-ledger", version: 0, latest: 1 }, + { scope: "forget-ledger", version: 0, latest: 2 }, ]); expect(await shapeOf(target, dialect)).toEqual([]); expect(await target.adapter.migrate(["checkpoints"])).toEqual([ { scope: "storage", version: 0, latest: 2 }, { scope: "checkpoints", version: 1, latest: 1 }, - { scope: "forget-ledger", version: 0, latest: 1 }, + { scope: "forget-ledger", version: 0, latest: 2 }, ]); expect(await target.adapter.migrate(["storage", "forget-ledger"])).toEqual([ { scope: "storage", version: 2, latest: 2 }, { scope: "checkpoints", version: 1, latest: 1 }, - { scope: "forget-ledger", version: 1, latest: 1 }, + { scope: "forget-ledger", version: 2, latest: 2 }, ]); const shape = await shapeOf(target, dialect); await opened(target.adapter, async () => undefined); @@ -307,6 +307,7 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ } expect((await versionsOf(target)).map(({ scope, version }) => ({ scope, version }))).toEqual([ { scope: "forget-ledger", version: 1 }, + { scope: "forget-ledger", version: 2 }, ]); }); diff --git a/packages/core/src/config/schema.test.ts b/packages/core/src/config/schema.test.ts index 4888d160..b503efe5 100644 --- a/packages/core/src/config/schema.test.ts +++ b/packages/core/src/config/schema.test.ts @@ -20,7 +20,12 @@ const postgres: AdapterDefinition<"postgresql", { url: string }> = { }; const keyWrapper = masterKey({ from: "BOUNDA_MASTER_KEY" }); -const forgetLedger = defineForgetLedger(() => ({ append: async () => {}, read: async () => [] })); +const forgetLedger = defineForgetLedger(() => ({ + append: async () => {}, + read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, +})); const message = (config: unknown): string => { try { diff --git a/packages/core/src/kernel/app.test.ts b/packages/core/src/kernel/app.test.ts index 1608c189..f7c5bf89 100644 --- a/packages/core/src/kernel/app.test.ts +++ b/packages/core/src/kernel/app.test.ts @@ -446,11 +446,12 @@ describe("createApp", () => { await fresh.stop(); }); - it("does not start background work in the web role but still serves commands and queries", async () => { + it("starts no background work in the web role but its watch for a restore, and still serves commands and queries", async () => { const { app, clock } = await start("web"); expect(app.role).toBe("web"); app.start(); - expect(clock.pending()).toBe(0); + app.start(); + expect(clock.pending()).toBe(1); await app.commands.placeOrder({ orderId: "o-1", total: 10 }); clock.advance(1_000); expect(await app.queries.getOrder({ orderId: "o-1" })).toBeNull(); diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index ece69cbf..5dc27b7a 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -73,7 +73,7 @@ import { import { createScheduledCommandWorker } from "./scheduler/worker.ts"; import { guardedLogger } from "./shared/guarded-logger.ts"; import { ignoredRetries, type PendingRetries } from "./shared/pending-retries.ts"; -import { createRestoreWatch } from "./shared/restores.ts"; +import { createRestoreWatch, GENERATION_INTERVAL_MS } from "./shared/restores.ts"; import { createSnapshotSubscriber, takesSnapshots } from "./snapshot/subscriber.ts"; import { type AppStatus, readAppStatus } from "./status.ts"; import { ATTRIBUTES, METRICS, meter, tracedOperation } from "./telemetry.ts"; @@ -190,8 +190,9 @@ export interface BoundaApp extends BoundaClien readonly config: ResolvedConfig; readonly role: RuntimeRole; /** - * Starts background work: the dispatcher and the scheduled command worker. Does nothing for - * `role: "web"`. + * Starts background work: the dispatcher and the scheduled command worker. For `role: "web"`, + * only the watch for a database restored under the app, which forgets again at once whom the + * copy brought back. */ start(): void; /** @@ -262,6 +263,14 @@ export interface BoundaApp extends BoundaClien entries: readonly ForgetAgainEntry[], options?: ForgetAgainOptions, ): Promise; + /** + * Says the database was restored from a copy: what `bounda restored` runs after restoring while + * the app was stopped, without a forget ledger that tells it. Counts the restore, so every + * running instance starts its background jobs over, clears the holds the copy brought back, and + * reconciles the forget ledger at once when there is one. A restore made under a running app, + * or while it was stopped with a ledger, is found without it. + */ + restored(): Promise; } /** @@ -659,7 +668,12 @@ export const assembleApp: AssembleAppFunction = async ({ logger, errorText, }); - const restores = createRestoreWatch({ checkpoints: written.checkpointStore, logger }); + const restores = createRestoreWatch({ + checkpoints: written.checkpointStore, + events: written.eventStore, + marks: ledger, + logger, + }); const dispatcher = createDispatcher({ eventStore: storage.eventStore, checkpointStore: storage.checkpointStore, @@ -799,6 +813,29 @@ export const assembleApp: AssembleAppFunction = async ({ retention, letterRetention, }); + // A copy restored while the app was stopped holds less than the forget ledger's mark: the + // holds it brought back are cleared before anything runs, and the reconciliation below forgets + // again whom it brought back. + const cold = await restores.start(clock.now()); + if (cold !== null) await worker.restored(cold.first); + // An instance that runs no worker still watches for a restore, and reconciles the ledger at + // once when it finds one, since no pass of its own would. + const watchForRestores = (): (() => void) => { + let cancel: (() => void) | undefined; + const next = (): void => { + cancel = clock.after(GENERATION_INTERVAL_MS, async () => { + const found = await restores.check(clock.now()); + if (found !== null) { + await worker.restored(found.first); + await reconciliation?.reconcile(clock.now()); + } + next(); + }); + }; + next(); + return () => cancel?.(); + }; + let unwatch: (() => void) | undefined; // Before the app serves anything: a backup restored brings back persons forgotten since, and // the read models that show them hear of it by the time the start resolves. A ledger that // does not answer in time is left to finish behind the start, as one that fails is retried, @@ -964,7 +1001,11 @@ export const assembleApp: AssembleAppFunction = async ({ config, role, start: () => { - if (role === "web" || stopping !== undefined) return; + if (stopping !== undefined) return; + if (role === "web") { + unwatch ??= watchForRestores(); + return; + } dispatcher.start(); worker.start(); }, @@ -972,7 +1013,9 @@ export const assembleApp: AssembleAppFunction = async ({ stopping ??= (async () => { try { lag.removeCallback(observeLag); + unwatch?.(); await Promise.all([dispatcher.stop(), worker.stop()]); + await restores.flush(); try { await readModels.close(); } finally { @@ -1034,6 +1077,11 @@ export const assembleApp: AssembleAppFunction = async ({ } return forgotten; }, + restored: async () => { + const { first } = await restores.declare(); + await worker.restored(first); + await reconciliation?.reconcile(clock.now()); + }, pendingRebuilds: async () => { const paused = await Promise.all( Object.values(readModels.byName).map(async ({ name, storage }) => diff --git a/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts index d2a3abfc..d339c678 100644 --- a/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts +++ b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts @@ -340,7 +340,7 @@ describe("the status of the app", () => { { database: "storage", scope: "storage", version: 2, latest: 2 }, { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, - { database: "forgetLedger", scope: "forget-ledger", version: 1, latest: 1 }, + { database: "forgetLedger", scope: "forget-ledger", version: 2, latest: 2 }, ]); expect(status).toMatchObject({ forgetLedger: { reconciled: true, held: null }, diff --git a/packages/core/src/kernel/migrate.test.ts b/packages/core/src/kernel/migrate.test.ts index b6a34025..3434d064 100644 --- a/packages/core/src/kernel/migrate.test.ts +++ b/packages/core/src/kernel/migrate.test.ts @@ -28,10 +28,14 @@ describe("migrateTables", () => { { database: "storage", scope: "storage", version: 2, latest: 2 }, { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, - { database: "forgetLedger", scope: "forget-ledger", version: 1, latest: 1 }, + { database: "forgetLedger", scope: "forget-ledger", version: 2, latest: 2 }, ]); expect(tablesIn(readModels.db)).toEqual(["bounda_checkpoints", "bounda_migrations"]); - expect(tablesIn(ledger.db)).toEqual(["bounda_forget_ledger", "bounda_migrations"]); + expect(tablesIn(ledger.db)).toEqual([ + "bounda_forget_ledger", + "bounda_forget_ledger_marks", + "bounda_migrations", + ]); expect(tablesIn(storage.db)).toContain("bounda_subscriber_failures"); expect(tablesIn(storage.db)).not.toContain("bounda_forget_ledger"); expect(await migrateTables({ config })).toHaveLength(4); diff --git a/packages/core/src/kernel/personal-data/ledger-append.test.ts b/packages/core/src/kernel/personal-data/ledger-append.test.ts index 4722affd..22ed466e 100644 --- a/packages/core/src/kernel/personal-data/ledger-append.test.ts +++ b/packages/core/src/kernel/personal-data/ledger-append.test.ts @@ -37,6 +37,8 @@ const setUp = async ({ appended.push(entries); }, read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, ...ledger, }, keepForMs, @@ -114,6 +116,8 @@ describe("the append to the forget ledger as a transaction commits", () => { appended.push(entries); }, read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, }, keepForMs: null, clock: createFixedClock(NOW), @@ -204,6 +208,8 @@ describe("the append to the forget ledger as a transaction commits", () => { : Promise.resolve(); }, read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, }, keepForMs: null, clock, diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts index 6c39319a..34c417be 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts @@ -30,6 +30,8 @@ describe("the forget ledger's outbox", () => { appended.push(entries); }, read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, }, keepForMs: null, clock: createFixedClock(new Date("2030-01-01T00:00:00.000Z")), @@ -89,6 +91,8 @@ describe("the forget ledger's outbox", () => { appended.push(entries); }, read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, }, keepForMs: 60_000, clock: createFixedClock(new Date("2026-01-01T00:02:00.000Z")), @@ -109,7 +113,12 @@ describe("the forget ledger's outbox", () => { describe("a forget ledger's source", () => { it("is an adapter that opens ledgers or a ledger definition, nothing else", () => { - const definition = defineForgetLedger(() => ({ append: async () => {}, read: async () => [] })); + const definition = defineForgetLedger(() => ({ + append: async () => {}, + read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, + })); expect(isForgetLedgerSource(memory())).toBe(true); expect(isForgetLedgerSource(definition)).toBe(true); for (const value of [ @@ -127,7 +136,12 @@ describe("a forget ledger's source", () => { const seen: unknown[] = []; const definition = defineForgetLedger((args) => { seen.push(args); - return { append: async () => {}, read: async () => [] }; + return { + append: async () => {}, + read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, + }; }); const env = { REGION: "eu" }; await openForgetLedger({ source: definition, storage: memory(), env, tenant: "t-1" }); diff --git a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts index a77a0555..83bd6636 100644 --- a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts @@ -342,6 +342,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { if (failures > 0) { failures -= 1; @@ -387,6 +389,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { reads += 1; if (down) throw new Error("ledger is down"); @@ -435,6 +439,8 @@ describe("the forget ledger", () => { read: async () => { throw new Error("ledger is down"); }, + readMark: async () => 0, + raiseMark: async () => undefined, })); const { app: first } = await start({ storage, forgetLedger: broken }); // A fresh store's first pass finds nothing written before the marks. @@ -465,6 +471,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { await answered; return inner.read(args); @@ -530,6 +538,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, append: async (entries) => { if (refusals > 0) { refusals -= 1; @@ -596,6 +606,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, append: async (entries) => { appends.push(...entries.map(({ personId }) => personId)); await inner.append(entries); @@ -636,6 +648,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, append: () => new Promise(() => undefined), }; }); @@ -666,6 +680,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { reads += 1; return inner.read(args); @@ -706,6 +722,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { reads += 1; if (failures > 0) { @@ -803,6 +821,8 @@ describe("the forget ledger", () => { const down = new Error("ledger is down"); let closed = 0; const broken = defineForgetLedger(async () => ({ + readMark: async () => 0, + raiseMark: async () => undefined, append: async () => {}, read: async () => { throw down; @@ -826,6 +846,8 @@ describe("the forget ledger", () => { const inner = await opened(ledgerAdapter); return { append: (entries) => inner.append(entries), + readMark: (name) => inner.readMark(name), + raiseMark: (name, value) => inner.raiseMark(name, value), read: async (args) => { await answered; return inner.read(args); diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index e5ff120f..2152b225 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -51,6 +51,8 @@ const recordingLedger = (entries: readonly ForgetLedgerEntry[], visibilityDelayM const inner = memory(); const ledger: ForgetLedger = { ...(visibilityDelayMs === undefined ? {} : { visibilityDelayMs }), + readMark: async () => 0, + raiseMark: async () => undefined, append: async () => {}, read: async (args) => { reads.push(args); @@ -199,6 +201,8 @@ describe("the forget ledger's reconciliation", () => { await inner.append([entry("c-1", "e-1", at(0))]); const reconciliation = createForgetLedgerReconciliation({ ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, append: async () => {}, read: async (args) => { if (failures > 0) { @@ -243,6 +247,8 @@ describe("the forget ledger's reconciliation", () => { let reads = 0; const reconciliation = createForgetLedgerReconciliation({ ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, append: async () => {}, read: async () => { reads += 1; diff --git a/packages/core/src/kernel/scheduler/worker.ts b/packages/core/src/kernel/scheduler/worker.ts index 8109a2bf..1784d7a0 100644 --- a/packages/core/src/kernel/scheduler/worker.ts +++ b/packages/core/src/kernel/scheduler/worker.ts @@ -61,6 +61,11 @@ export interface ScheduledCommandWorker { * How many due deadlines and restricted retries wait for the reactive runners to catch up. */ waitingDeadlines(): number; + /** + * Starts every background job over from what the database holds, as after a restore, between + * passes; `first` also clears the holds the copy brought back. Never rejects. + */ + restored(first: boolean): Promise; } export interface CreateScheduledCommandWorkerArgs { @@ -570,9 +575,12 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction ]); }; + // Whether the last pass did anything, which may have written no event: a key rewrapped, a hold. + let worked = false; + const runOnce = (): Promise => mutex.run(async () => { - const restored = await restores?.check(clock.now()); + const restored = await restores?.check(clock.now(), { active: worked }); if (restored !== undefined && restored !== null) await restart(restored.first); const claimedAt = clock.now(); const due = await claimDue(claimedAt); @@ -614,7 +622,9 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction const purged = (await letterRetention?.sweep(claimedAt)) ?? 0; const backfilled = (await backfill?.sweep(claimedAt)) ?? 0; if (unreadable !== undefined) throw unreadable.error; - return due.length + forgotten + rewrapped + forgottenAgain + purged + backfilled; + const done = due.length + forgotten + rewrapped + forgottenAgain + purged + backfilled; + worked = done > 0; + return done; }); const schedule = (): void => { @@ -647,5 +657,6 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction runOnce, leaseMs, waitingDeadlines: () => waits.size, + restored: (first) => mutex.run(() => restart(first)), }; }; diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index e1de476c..63b084e1 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -16,6 +16,7 @@ import { type BoundaApp, createApp } from "../app.ts"; import { HOLD_MAX_MS } from "../personal-data/hold.ts"; import { masterKey } from "../personal-data/master-key.ts"; import { ROTATION_CHECK_INTERVAL_MS } from "../personal-data/rotation.ts"; +import { advanceUntilWaiting } from "../test-support.ts"; import { GENERATION_INTERVAL_MS } from "./restores.ts"; const START = new Date("2026-04-01T00:00:00.000Z"); @@ -239,6 +240,8 @@ describe("a database restored while the app runs", () => { await first.app.runUntilIdle(); await second.app.runUntilIdle(); const copy = copyOf(one.db); + // What the copy undoes: activity after it, which moves the generation. + await register(first.app, "c-1"); first.clock.advance(GENERATION_INTERVAL_MS); second.clock.advance(GENERATION_INTERVAL_MS); await first.app.runUntilIdle(); @@ -256,3 +259,129 @@ describe("a database restored while the app runs", () => { expect((await second.app.status()).restoresFound).toBe(1); }); }); + +describe("a database restored while the app was stopped", () => { + const files = () => { + const directory = mkdtempSync(join(tmpdir(), "bounda-cold-")); + return { + storage: join(directory, "store.db"), + ledger: join(directory, "ledger.db"), + }; + }; + + const restarts = (logger: { readonly logged: Logged[] }) => + logger.logged.filter( + ({ message }) => + message === "the database went back to an earlier copy; background jobs start over", + ); + + it("is found at start from the forget ledger's mark, which clears the copy's holds and forgets again", async () => { + const paths = files(); + const store = createNodeSqliteAdapter(paths.storage); + const ledger = createNodeSqliteAdapter(paths.ledger).adapter; + const runtime: RuntimeConfig = { + personalData: { forgetLedger: { store: ledger }, retention: "3650d" }, + }; + const before = await start({ adapter: store.adapter, runtime }); + await register(before.app, "c-1"); + await before.app.runUntilIdle(); + // The copy: c-1 registered, and a person held for long after failures. + store.db + .prepare( + `INSERT INTO "bounda_persons" ("person", "position", "last_active_at", "held_until", "attempts") VALUES ('c-9', 1, ?, ?, 7)`, + ) + .run(START.toISOString(), new Date(START.getTime() + HOLD_MAX_MS).toISOString()); + const copy = copyOf(store.db); + await before.app.forget("c-1"); + before.clock.advance(GENERATION_INTERVAL_MS); + await before.app.runUntilIdle(); + await before.app.stop(); + restore(store.db, copy); + + const logger = recordingLogger(); + const after = await start({ adapter: store.adapter, runtime, logger }); + expect(restarts(logger).map(({ fields }) => fields)).toEqual([{ first: true }]); + expect( + store.db + .prepare(`SELECT "held_until", "attempts" FROM "bounda_persons" WHERE "person" = 'c-9'`) + .get(), + ).toEqual({ held_until: null, attempts: 0 }); + const reasons = (await after.app.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => (event.payload as PersonalDataForgottenPayload).reason); + expect(reasons).toEqual(["restored"]); + expect((await after.app.status()).restoresFound).toBe(1); + await after.app.stop(); + const again = recordingLogger(); + await start({ adapter: store.adapter, runtime, logger: again }); + expect(restarts(again)).toEqual([]); + }); + + it("is declared without a ledger by bounda restored, which every running instance learns", async () => { + const path = files().storage; + const one = createNodeSqliteAdapter(path); + const runningLogger = recordingLogger(); + const runtime: RuntimeConfig = { personalData: { retention: "3650d" } }; + const running = await start({ + adapter: createNodeSqliteAdapter(path).adapter, + runtime, + logger: runningLogger, + }); + await running.app.runUntilIdle(); + one.db + .prepare( + `INSERT INTO "bounda_persons" ("person", "position", "last_active_at", "held_until", "attempts") VALUES ('c-9', 1, ?, ?, 7)`, + ) + .run(START.toISOString(), new Date(START.getTime() + HOLD_MAX_MS).toISOString()); + const operatorLogger = recordingLogger(); + const operator = await start({ adapter: one.adapter, runtime, logger: operatorLogger }); + await operator.app.restored(); + expect(restarts(operatorLogger).map(({ fields }) => fields)).toEqual([{ first: true }]); + expect( + one.db + .prepare(`SELECT "held_until", "attempts" FROM "bounda_persons" WHERE "person" = 'c-9'`) + .get(), + ).toEqual({ held_until: null, attempts: 0 }); + await running.app.runUntilIdle(); + expect(restarts(runningLogger).map(({ fields }) => fields)).toEqual([{ first: false }]); + expect((await running.app.status()).restoresFound).toBe(1); + }); +}); + +describe("an instance in the web role", () => { + it("watches for a restore and forgets again at once whom the copy brings back", async () => { + const directory = mkdtempSync(join(tmpdir(), "bounda-web-")); + const store = createNodeSqliteAdapter(join(directory, "store.db")); + const ledger = memory(); + const runtime: RuntimeConfig = { + role: "web", + personalData: { forgetLedger: { store: ledger } }, + }; + const logger = recordingLogger(); + const { app, clock } = await start({ adapter: store.adapter, runtime, logger }); + app.start(); + await register(app, "c-1"); + await advanceUntilWaiting(clock, GENERATION_INTERVAL_MS); + const copy = copyOf(store.db); + await app.forget("c-1"); + await advanceUntilWaiting(clock, GENERATION_INTERVAL_MS); + restore(store.db, copy); + expect(store.db.prepare(`SELECT count(*) AS "keys" FROM "bounda_personal_keys"`).get()).toEqual( + { + keys: 1, + }, + ); + await advanceUntilWaiting(clock, GENERATION_INTERVAL_MS); + expect(store.db.prepare(`SELECT count(*) AS "keys" FROM "bounda_personal_keys"`).get()).toEqual( + { + keys: 0, + }, + ); + expect( + logger.logged.filter( + ({ message }) => + message === "the database went back to an earlier copy; background jobs start over", + ), + ).toHaveLength(1); + }); +}); diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts index c3a2f2f2..3f0f131c 100644 --- a/packages/core/src/kernel/shared/restores.test.ts +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -1,35 +1,57 @@ import { describe, expect, it } from "vitest"; import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; import { memory } from "../../memory/index.ts"; import { createRecordingLogger } from "../test-support.ts"; import { createRestoreWatch, GENERATION_INTERVAL_MS, + MARK_INTERVAL_MS, RESTORE_GENERATION, + RESTORE_MARK, RESTORES_FOUND, } from "./restores.ts"; const at = (ms: number) => new Date(Date.parse("2026-01-01T00:00:00.000Z") + ms); +const setUp = async ( + options: { + readonly marks?: boolean; + readonly wrap?: (store: CheckpointStore) => CheckpointStore; + readonly ledger?: (ledger: ForgetLedger) => Pick; + } = {}, +) => { + const { checkpointStore } = await memory().createStorage(); + const ledger = await memory().createForgetLedger({}); + const { logger, entries } = createRecordingLogger("omit"); + const stream = { head: 0 }; + const watchOver = () => + createRestoreWatch({ + checkpoints: options.wrap?.(checkpointStore) ?? checkpointStore, + events: { lastPosition: async () => stream.head }, + marks: options.marks === false ? null : (options.ledger?.(ledger) ?? ledger), + logger, + }); + return { checkpointStore, ledger, entries, stream, watchOver }; +}; + describe("createRestoreWatch", () => { it("moves the generation at most once an interval", async () => { - const { checkpointStore } = await memory().createStorage(); - const { logger } = createRecordingLogger("omit"); - const watch = createRestoreWatch({ checkpoints: checkpointStore, logger }); - await watch.check(at(0)); - await watch.check(at(GENERATION_INTERVAL_MS - 1)); + const { checkpointStore, watchOver } = await setUp({ marks: false }); + const watch = watchOver(); + await watch.check(at(0), { active: true }); + await watch.check(at(GENERATION_INTERVAL_MS - 1), { active: true }); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); await watch.check(at(GENERATION_INTERVAL_MS)); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(2); }); it("counts a restore once when two instances find it at the same moment, and then nothing more", async () => { - const { checkpointStore } = await memory().createStorage(); - const { logger } = createRecordingLogger("omit"); - const one = createRestoreWatch({ checkpoints: checkpointStore, logger }); - const other = createRestoreWatch({ checkpoints: checkpointStore, logger }); - await one.check(at(0)); - await other.check(at(0)); + const { checkpointStore, watchOver } = await setUp({ marks: false }); + const one = watchOver(); + const other = watchOver(); + await one.check(at(0), { active: true }); + await other.check(at(0), { active: true }); await checkpointStore.set(RESTORE_GENERATION, 0); const found = await Promise.all([one.check(at(1)), other.check(at(1))]); expect(found.map((each) => each?.first).sort()).toEqual([false, true]); @@ -38,20 +60,102 @@ describe("createRestoreWatch", () => { }); it("logs a store it could not read, and finds nothing", async () => { - const { checkpointStore } = await memory().createStorage(); - const { logger, entries } = createRecordingLogger("omit"); - const failing: CheckpointStore = { - ...checkpointStore, - get: () => Promise.reject(new Error("store is down")), - }; - const watch = createRestoreWatch({ checkpoints: failing, logger }); + const { entries, watchOver } = await setUp({ + marks: false, + wrap: (store) => ({ ...store, get: () => Promise.reject(new Error("store is down")) }), + }); + const watch = watchOver(); expect(await watch.check(at(0))).toBeNull(); + expect(await watch.start(at(0))).toBeNull(); expect(entries).toEqual([ { level: "error", message: "the database could not be checked for a restore", fields: expect.objectContaining({ errorName: "Error" }), }, + { + level: "error", + message: "the database could not be compared with the forget ledger's mark", + fields: expect.objectContaining({ errorName: "Error" }), + }, ]); }); + + it("finds at start a copy restored while the app was stopped, from the ledger's mark", async () => { + const { checkpointStore, ledger, watchOver } = await setUp(); + const running = watchOver(); + expect(await running.start(at(0))).toBeNull(); + const copy = await checkpointStore.get(RESTORE_GENERATION); + await running.check(at(0), { active: true }); + await running.check(at(GENERATION_INTERVAL_MS), { active: true }); + await running.flush(); + expect(await ledger.readMark(RESTORE_MARK)).toBe(2); + await checkpointStore.set(RESTORE_GENERATION, copy); + const restarted = watchOver(); + expect(await restarted.start(at(MARK_INTERVAL_MS))).toEqual({ first: true }); + expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(3); + expect(await ledger.readMark(RESTORE_MARK)).toBe(3); + expect(await watchOver().start(at(MARK_INTERVAL_MS + 1))).toBeNull(); + }); + + it("raises the mark as the generation moves, at most once a mark interval, and on a flush", async () => { + const { ledger, watchOver } = await setUp(); + const watch = watchOver(); + await watch.start(at(0)); + await watch.check(at(1), { active: true }); + expect(await ledger.readMark(RESTORE_MARK)).toBe(0); + await watch.check(at(MARK_INTERVAL_MS), { active: true }); + expect(await ledger.readMark(RESTORE_MARK)).toBe(2); + await watch.check(at(MARK_INTERVAL_MS + GENERATION_INTERVAL_MS), { active: true }); + expect(await ledger.readMark(RESTORE_MARK)).toBe(2); + await watch.flush(); + expect(await ledger.readMark(RESTORE_MARK)).toBe(3); + }); + + it("compares with the mark later when the ledger could not be read at start, moving nothing meanwhile", async () => { + let down = true; + const { checkpointStore, ledger, watchOver } = await setUp({ + ledger: (inner) => ({ + readMark: (name) => (down ? Promise.reject(new Error("down")) : inner.readMark(name)), + raiseMark: (name, value) => inner.raiseMark(name, value), + }), + }); + await ledger.raiseMark(RESTORE_MARK, 5); + const watch = watchOver(); + expect(await watch.start(at(0))).toBeNull(); + expect(await watch.check(at(1), { active: true })).toBeNull(); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(0); + await watch.flush(); + expect(await ledger.readMark(RESTORE_MARK)).toBe(5); + down = false; + expect(await watch.check(at(2))).toEqual({ first: true }); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(6); + }); + + it("logs a mark it could not raise, and goes on", async () => { + const { entries, watchOver } = await setUp({ + ledger: (inner) => ({ + readMark: (name) => inner.readMark(name), + raiseMark: () => Promise.reject(new Error("down")), + }), + }); + const watch = watchOver(); + expect(await watch.start(at(0))).toBeNull(); + await watch.flush(); + expect(entries.map(({ message }) => message)).toEqual([ + "the forget ledger's mark could not be raised", + "the forget ledger's mark could not be raised", + ]); + }); + + it("declares a restore, the first, and raises the mark past it", async () => { + const { checkpointStore, ledger, watchOver } = await setUp(); + const watch = watchOver(); + await watch.start(at(0)); + expect(await watch.declare()).toEqual({ first: true }); + expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); + expect(await ledger.readMark(RESTORE_MARK)).toBe(1); + }); }); diff --git a/packages/core/src/kernel/shared/restores.ts b/packages/core/src/kernel/shared/restores.ts index 754a8274..f732b414 100644 --- a/packages/core/src/kernel/shared/restores.ts +++ b/packages/core/src/kernel/shared/restores.ts @@ -1,10 +1,13 @@ import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; +import type { EventStore } from "../../adapter/storage/event-store.ts"; +import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; import type { Logger } from "../../contracts/logger.ts"; /** - * The checkpoint row every running instance moves one forward, at most once every - * `GENERATION_INTERVAL_MS`: it only ever grows, so a database that holds less than an instance - * read from it went back to an earlier copy. + * The checkpoint row the running instances move one forward, each at most once every + * `GENERATION_INTERVAL_MS` and only after activity: it only ever grows, so a database that holds + * less than an instance read from it, or than the forget ledger's mark, went back to an earlier + * copy. */ export const RESTORE_GENERATION = "runtime:generation"; @@ -14,8 +17,17 @@ export const RESTORE_GENERATION = "runtime:generation"; */ export const RESTORES_FOUND = "runtime:restores"; +/** + * The forget ledger's mark the generation is raised to, outside the database: what tells a copy + * restored while the app was stopped. + */ +export const RESTORE_MARK = "restore:generation"; + export const GENERATION_INTERVAL_MS = 5_000; +// How often a moved generation is raised in the forget ledger at most, which a stop also does. +export const MARK_INTERVAL_MS = 60_000; + export interface Restored { /** * Whether this instance found it first: the one that clears what the copy brought back of the @@ -26,20 +38,50 @@ export interface Restored { export interface RestoreWatch { /** - * Reads the rows and resolves to what this instance learnt: `null` while the database holds - * at least what it read last. A database restored while this instance was stopped is not - * told apart from one left alone. Never rejects: a store that fails is logged, and the next - * check reads again. + * Compares the database with what this instance read from it at start, against the forget + * ledger's mark: `null` while it holds at least that. Never rejects: a ledger that cannot be + * read is logged and asked again by `check`. */ - check(now: Date): Promise; + start(now: Date): Promise; + /** + * Reads the rows and resolves to what this instance learnt: `null` while the database holds at + * least what it read last. Moves the generation after activity, a write to the stream since the + * last move or `active`. Never rejects: a store that fails is logged, and the next check reads + * again. + */ + check(now: Date, options?: CheckRestoreOptions): Promise; + /** + * Records a restore an operator made: counted as found, so every running instance learns of + * it, and this one first. + */ + declare(): Promise; + /** + * Raises the forget ledger's mark to the generation this instance read last; what a stop does. + * Never rejects. + */ + flush(): Promise; /** * How many restores the instances found, as the database counts them. */ found(): Promise; } +export interface CheckRestoreOptions { + /** + * Whether this instance wrote anything besides events since its last check, such as a key + * rewrapped or a hold. + */ + readonly active?: boolean; +} + export interface CreateRestoreWatchArgs { readonly checkpoints: CheckpointStore; + readonly events: Pick; + /** + * The forget ledger, outside the database; `null` without one, and a copy restored while the + * app was stopped is then found only when declared. + */ + readonly marks: Pick | null; readonly logger: Logger; } @@ -47,54 +89,147 @@ export interface CreateRestoreWatchFunction { (args: CreateRestoreWatchArgs): RestoreWatch; } +interface Seen { + readonly generation: number; + readonly restores: number; +} + /** * A generation that went back, or a count of restores that changed, means a restore. The first * instance to see the generation go back moves the count past every value an instance can have * read, so each other instance finds it changed, whether its own reading of the generation shows - * the restore or not. A copy taken within one interval of the restore may hold the generation - * every instance read last, and goes unseen: what it undid is that recent. + * the restore or not. A copy taken after the last move goes unseen: nothing was written since. */ -export const createRestoreWatch: CreateRestoreWatchFunction = ({ checkpoints, logger }) => { - let seen: { readonly generation: number; readonly restores: number } | null = null; +export const createRestoreWatch: CreateRestoreWatchFunction = ({ + checkpoints, + events, + marks, + logger, +}) => { + let seen: Seen | null = null; let movedAt = Number.NEGATIVE_INFINITY; + let markedAt = Number.NEGATIVE_INFINITY; + let head: number | null = null; + let pending = false; + // While the ledger's mark could not be read, the generation stays where it is: moving it could + // carry a copy restored while the app was stopped past the mark before it is compared. + let unmarked = marks !== null; - const check = async (now: Date): Promise => { - const at = now.getTime(); + const read = async (): Promise => { const [generation, restores] = await Promise.all([ checkpoints.get(RESTORE_GENERATION), checkpoints.get(RESTORES_FOUND), ]); + return { generation, restores }; + }; + + // Counts the restore unless another instance did first, and moves the generation past + // `floor`, so the copy's own value never reads as current again. + const count = async (stored: Seen, past: number, floor: number): Promise => { + const counted = past + 1; + const first = await checkpoints.compareAndSet(RESTORES_FOUND, stored.restores, counted); + const restores = first ? counted : await checkpoints.get(RESTORES_FOUND); + const generation = Math.max(stored.generation, floor) + 1; + const moved = await checkpoints.compareAndSet( + RESTORE_GENERATION, + stored.generation, + generation, + ); + seen = { generation: moved ? generation : await checkpoints.get(RESTORE_GENERATION), restores }; + movedAt = Number.NEGATIVE_INFINITY; + return { first }; + }; + + const raise = async (at: number, generation: number): Promise => { + if (marks === null) return; + try { + await marks.raiseMark(RESTORE_MARK, generation); + markedAt = at; + } catch (error) { + logger.error("the forget ledger's mark could not be raised", { error }); + } + }; + + // A mark past the database's generation: the database is a copy from before it. + const compareWithMark = async (stored: Seen): Promise => { + if (marks === null) return null; + const mark = await marks.readMark(RESTORE_MARK); + unmarked = false; + if (stored.generation >= mark) return null; + const restored = await count(stored, stored.restores, mark); + await marks.raiseMark(RESTORE_MARK, seen?.generation ?? mark); + return restored; + }; + + const check = async (now: Date, active: boolean): Promise => { + const at = now.getTime(); + const [stored, last] = await Promise.all([read(), events.lastPosition()]); + pending ||= active || (head !== null && last !== head); + head = last; + if (unmarked) { + const restored = await compareWithMark(stored); + if (restored !== null) return restored; + } let restored: Restored | null = null; - let counted = restores; - if (seen !== null && (generation < seen.generation || restores !== seen.restores)) { + let current = stored; + if ( + seen !== null && + (stored.generation < seen.generation || stored.restores !== seen.restores) + ) { // A count past what this instance read was moved by the instance that found it first. - if (restores > seen.restores) { + if (stored.restores > seen.restores) { restored = { first: false }; } else { - counted = seen.restores + 1; - const first = await checkpoints.compareAndSet(RESTORES_FOUND, restores, counted); - if (!first) counted = await checkpoints.get(RESTORES_FOUND); - restored = { first }; + restored = await count(stored, seen.restores, 0); + current = seen; } - movedAt = Number.NEGATIVE_INFINITY; + pending = true; } - let current = generation; - if (at - movedAt >= GENERATION_INTERVAL_MS) { - if (await checkpoints.compareAndSet(RESTORE_GENERATION, generation, generation + 1)) { - current = generation + 1; + if (pending && !unmarked && at - movedAt >= GENERATION_INTERVAL_MS) { + const next = current.generation + 1; + if (await checkpoints.compareAndSet(RESTORE_GENERATION, current.generation, next)) { + current = { ...current, generation: next }; + if (at - markedAt >= MARK_INTERVAL_MS) await raise(at, next); } movedAt = at; + pending = false; } - seen = { generation: current, restores: counted }; + seen = current; return restored; }; + const logged = + (what: string) => + (error: unknown): null => { + logger.error(what, { error }); + return null; + }; + return { - check: (now) => - check(now).catch((error: unknown) => { - logger.error("the database could not be checked for a restore", { error }); - return null; - }), + start: async (now) => { + try { + const stored = await read(); + head = await events.lastPosition(); + seen = stored; + const restored = await compareWithMark(stored); + if (restored === null && marks !== null) await raise(now.getTime(), stored.generation); + return restored; + } catch (error) { + return logged("the database could not be compared with the forget ledger's mark")(error); + } + }, + check: (now, { active = false } = {}) => + check(now, active).catch(logged("the database could not be checked for a restore")), + declare: async () => { + const stored = await read(); + const restored = await count(stored, Math.max(stored.restores, seen?.restores ?? 0), 0); + if (marks !== null && seen !== null) await marks.raiseMark(RESTORE_MARK, seen.generation); + return restored; + }, + flush: async () => { + if (seen === null || unmarked) return; + await raise(Number.NEGATIVE_INFINITY, seen.generation); + }, found: () => checkpoints.get(RESTORES_FOUND), }; }; diff --git a/packages/core/src/kernel/status.test.ts b/packages/core/src/kernel/status.test.ts index e04fd592..e2e22bcf 100644 --- a/packages/core/src/kernel/status.test.ts +++ b/packages/core/src/kernel/status.test.ts @@ -79,7 +79,13 @@ describe("readAppStatus", () => { personalDataBackfill: null, }), failures: subscriberFailures, - restores: { check: async () => null, found: async () => 0 }, + restores: { + start: async () => null, + check: async () => null, + declare: async () => ({ first: true }), + flush: async () => undefined, + found: async () => 0, + }, }); expect( Object.fromEntries(status.subscriberFailures.map((each) => [each.subscriber, each.current])), diff --git a/packages/core/src/memory/forget-ledger.ts b/packages/core/src/memory/forget-ledger.ts index 1ef7664c..fcec9ed3 100644 --- a/packages/core/src/memory/forget-ledger.ts +++ b/packages/core/src/memory/forget-ledger.ts @@ -21,6 +21,7 @@ export interface CreateMemoryForgetLedgersFunction { */ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () => { const byTenant = new Map>(); + const marks = new Map(); return { rows: () => [...byTenant.entries()].flatMap(([tenant, entries]) => @@ -47,6 +48,11 @@ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () = if (entry.forgottenAt < before) entries.delete(eventId); } }, + readMark: async (name) => marks.get(`${key}\u0000${name}`) ?? 0, + raiseMark: async (name, value) => { + const mark = `${key}\u0000${name}`; + marks.set(mark, Math.max(marks.get(mark) ?? 0, value)); + }, }; }, }; diff --git a/packages/core/test-types/config.test-d.ts b/packages/core/test-types/config.test-d.ts index 1ae7624a..fca7c628 100644 --- a/packages/core/test-types/config.test-d.ts +++ b/packages/core/test-types/config.test-d.ts @@ -363,6 +363,14 @@ describe("defineConfig", () => { prune: async (args) => { expectTypeOf(args).toEqualTypeOf(); }, + readMark: async (name) => { + expectTypeOf(name).toEqualTypeOf(); + return 0; + }, + raiseMark: async (name, value) => { + expectTypeOf(name).toEqualTypeOf(); + expectTypeOf(value).toEqualTypeOf(); + }, }; }); expectTypeOf(own).toEqualTypeOf(); diff --git a/packages/core/test-types/inference.test-d.ts b/packages/core/test-types/inference.test-d.ts index cb9ede16..8f6f3ad9 100644 --- a/packages/core/test-types/inference.test-d.ts +++ b/packages/core/test-types/inference.test-d.ts @@ -100,6 +100,7 @@ import { } from "@bounda-dev/core"; import type { Adapter, + ForgetLedger, SchemaScopeName, SchemaVersion, StoredSubscriberFailure, @@ -1370,6 +1371,14 @@ describe("operations", () => { expectTypeOf().toEqualTypeOf(); }); + it("declares a restore on the app, and keeps the ledger's marks outside the database", () => { + expectTypeOf().toEqualTypeOf<() => Promise>(); + expectTypeOf().toEqualTypeOf<(name: string) => Promise>(); + expectTypeOf().toEqualTypeOf< + (name: string, value: number) => Promise + >(); + }); + it("names a process given up as abandoned", () => { expectTypeOf().toEqualTypeOf< "started" | "completed" | "failed" | "timed_out" | "abandoned" diff --git a/packages/postgresql/src/migrations.ts b/packages/postgresql/src/migrations.ts index 1335c5e9..9be8b4f9 100644 --- a/packages/postgresql/src/migrations.ts +++ b/packages/postgresql/src/migrations.ts @@ -1,6 +1,7 @@ import { ConfigurationError } from "@bounda-dev/core"; import type { SchemaVersion } from "@bounda-dev/core/adapter"; import { + forgetLedgerMarksStatement, forgetLedgerStatements, migrateSchemas, postgresqlDialect, @@ -179,6 +180,7 @@ export const postgresqlForgetLedgerScope: PostgresqlForgetLedgerScopeFunction = } }, }, + { version: 2, migrate: (tx) => tx.run(forgetLedgerMarksStatement(table), []) }, ], }); From dda04df464b13023ed0b5e5f81e013c988d8f8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:41:41 +0100 Subject: [PATCH 12/19] docs: the ledger's marks in the configuration reference --- docs/src/content/docs/reference/configuration.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 4c777dec..92f5f1e6 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -128,7 +128,8 @@ a stopped one left: see [delivery guarantees](/concepts/delivery-guarantees/#the [backups](/guides/personal-data/#backups). It holds: - **`store`**, required. The ledger: an adapter pointed at another database than `storage`, such as `sqlite({ path: "./ledger/forget-ledger.db" })` or `postgresql({ url })`, which creates its - `bounda_forget_ledger` table there on first use; `kvForgetLedger({ binding })` from + `bounda_forget_ledger` table there on first use, with `bounda_forget_ledger_marks` for the mark + that tells a [restore](/guides/personal-data/#restores) made while the app was stopped; `kvForgetLedger({ binding })` from `@bounda-dev/cloudflare` on Cloudflare; or a ledger of your own through `defineForgetLedger`. The runtime appends each forgetting to it as the forgetting commits, and copies every `PersonalDataForgotten` to it again from the stream in the background. An adapter pointing at @@ -136,8 +137,8 @@ a stopped one left: see [delivery guarantees](/concepts/delivery-guarantees/#the - **`reconcileEvery`**, `5m` by default (`DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS` in `@bounda-dev/core/config`). The app reads the ledger when it starts and, while it runs, again this often, one instance a round, and forgets again, with `reason: "restored"`, whoever a - restored backup brought back. A Durable Object only runs a round on the passes it wakes for - anyway; it reconciles on every start. + restored backup brought back; a restore the instances find reconciles at once. A Durable Object + only runs a round on the passes it wakes for anyway; it reconciles on every start. - **`strict`**, `false` by default. When `false`, a ledger that cannot be read at start, or does not answer within ten seconds, does not fail it: the app logs an error and reconciles in the background, and `getLag()` reports `forgetLedger: { reconciled: false }` meanwhile. When From e9237de6863a9293e3368b28b6c5f038d157e59c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:45:23 +0100 Subject: [PATCH 13/19] test: list the forget ledger's marks table on PostgreSQL --- packages/postgresql/src/postgresql.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/postgresql/src/postgresql.test.ts b/packages/postgresql/src/postgresql.test.ts index 6272a666..8eacf44b 100644 --- a/packages/postgresql/src/postgresql.test.ts +++ b/packages/postgresql/src/postgresql.test.ts @@ -379,6 +379,7 @@ describe.skipIf(container === null)("postgresql adapter", () => { const tables = await holder`SELECT tablename FROM pg_tables WHERE schemaname = ${schema}`; expect(tables.map(({ tablename }) => tablename).sort()).toEqual([ `t${run}_${prefixes}_forget_ledger`, + `t${run}_${prefixes}_forget_ledger_marks`, `t${run}_${prefixes}_migrations`, ]); await holder`SELECT pg_advisory_lock(hashtext(${`bounda:${schema}:schema`}))`; From 6fb678d8864f3fee50dd92ab5f0fdc74a8a409c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:51:13 +0100 Subject: [PATCH 14/19] test: pin the restore watch without a ledger, before its mark is read, and after work that writes no event --- .../src/kernel/shared/restores-app.test.ts | 42 +++++++++++++++++++ .../core/src/kernel/shared/restores.test.ts | 35 ++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index 63b084e1..eaa68723 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -202,6 +202,48 @@ describe("a database restored while the app runs", () => { expect(await app.status()).toMatchObject({ restoresFound: 1 }); }); + it("moves the generation after a pass that wrote no event, as a key rewrapped, and not while idle", async () => { + const { adapter, db } = createNodeSqliteAdapter(); + const old = randomKey(); + const before = await start({ + adapter, + env: { BOUNDA_MASTER_KEY: old }, + runtime: { personalData: { keyWrapper: masterKey({ from: "BOUNDA_MASTER_KEY" }) } }, + }); + await register(before.app, "c-1"); + await before.app.stop(); + const { app, clock } = await start({ + adapter, + env: { BOUNDA_MASTER_KEY: randomKey(), BOUNDA_MASTER_KEY_PREVIOUS: old }, + runtime: { + personalData: { + keyWrapper: masterKey({ + from: "BOUNDA_MASTER_KEY", + previous: ["BOUNDA_MASTER_KEY_PREVIOUS"], + }), + }, + }, + }); + const generation = () => + ( + db + .prepare( + `SELECT "position" FROM "bounda_checkpoints" WHERE "subscriber" = 'runtime:generation'`, + ) + .get() as { position: number } | undefined + )?.position ?? 0; + await app.runUntilIdle({ maxPasses: 1 }); + const rewrapped = generation(); + clock.advance(GENERATION_INTERVAL_MS); + await app.runUntilIdle({ maxPasses: 1 }); + expect(generation()).toBe(rewrapped + 1); + clock.advance(GENERATION_INTERVAL_MS); + await app.runUntilIdle({ maxPasses: 1 }); + clock.advance(GENERATION_INTERVAL_MS); + await app.runUntilIdle({ maxPasses: 1 }); + expect(generation()).toBe(rewrapped + 1); + }); + it("forgets again at once whom the copy brings back, without waiting for the periodic reconciliation", async () => { const { adapter, db } = createNodeSqliteAdapter(); const ledger = memory(); diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts index 3f0f131c..bfdc2a6b 100644 --- a/packages/core/src/kernel/shared/restores.test.ts +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -158,4 +158,39 @@ describe("createRestoreWatch", () => { expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); expect(await ledger.readMark(RESTORE_MARK)).toBe(1); }); + + it("touches no ledger and logs nothing without one", async () => { + const { entries, watchOver } = await setUp({ marks: false }); + const watch = watchOver(); + expect(await watch.start(at(0))).toBeNull(); + await watch.check(at(1), { active: true }); + await watch.flush(); + expect(entries).toEqual([]); + }); + + it("raises nothing on a flush before the mark could be read", async () => { + let raises = 0; + const { watchOver } = await setUp({ + ledger: () => ({ + readMark: () => Promise.reject(new Error("down")), + raiseMark: async () => { + raises += 1; + }, + }), + }); + const watch = watchOver(); + await watch.start(at(0)); + await watch.flush(); + expect(raises).toBe(0); + }); + + it("declares a restore past the count it read, which the database may have taken back", async () => { + const { checkpointStore, watchOver } = await setUp({ marks: false }); + const watch = watchOver(); + await checkpointStore.set(RESTORES_FOUND, 4); + await watch.check(at(0)); + await checkpointStore.set(RESTORES_FOUND, 2); + expect(await watch.declare()).toEqual({ first: true }); + expect(await checkpointStore.get(RESTORES_FOUND)).toBe(5); + }); }); From 102551b9f2aa10d6b4538f031d6e8cea31e78a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:51:28 +0100 Subject: [PATCH 15/19] docs(core): document the forget ledger's marks statement --- packages/core/src/adapter/sql/forget-ledger.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/src/adapter/sql/forget-ledger.ts b/packages/core/src/adapter/sql/forget-ledger.ts index d7f464dc..bcbd106e 100644 --- a/packages/core/src/adapter/sql/forget-ledger.ts +++ b/packages/core/src/adapter/sql/forget-ledger.ts @@ -42,7 +42,10 @@ export interface ForgetLedgerMarksStatementFunction { (table: string): string; } -// The second migration of the ledger's scope: its marks, one row per tenant and name. +/** + * The table of the ledger's marks, one row per tenant and name, beside the ledger's own `table`, + * quoted: the second migration of the ledger's scope. + */ export const forgetLedgerMarksStatement: ForgetLedgerMarksStatementFunction = (table) => `CREATE TABLE IF NOT EXISTS ${forgetLedgerMarksTableFor(table)} ( "tenant" TEXT NOT NULL, From 2b939c21e2b88380485ad762dd00eff9aeecf6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:55:21 +0100 Subject: [PATCH 16/19] feat(core): raise the ledger's mark with every move, on a stop after writes, and take a forgetting the database lost for a restore --- docs/src/content/docs/guides/personal-data.md | 12 ++++- packages/core/src/kernel/app.ts | 4 ++ .../personal-data/reconciliation-app.test.ts | 7 ++- packages/core/src/kernel/scheduler/worker.ts | 17 +++++-- .../src/kernel/shared/restores-app.test.ts | 29 ++++++++++++ .../core/src/kernel/shared/restores.test.ts | 18 +++---- packages/core/src/kernel/shared/restores.ts | 47 +++++++++++-------- 7 files changed, 98 insertions(+), 36 deletions(-) diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index 6a24b8a1..091d5972 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -1021,7 +1021,7 @@ and `bounda status` counts the restores found. How it is found depends on when t | The copy was restored | Found by | What is guaranteed | | --- | --- | --- | | under the running app | every instance, on its next pass, or within five seconds in the `web` role | Any copy taken before the last write the instances saw. A copy taken after it differs from the database in nothing but waits, which the jobs read again anyway. | -| while the app was stopped, with a forget ledger | the first instance to start, before it serves anything | Any copy taken before the last mark the ledger holds: raised at most once a minute while the app writes, and when it stops. A copy taken in the last minute before a crash goes unseen, as above. | +| while the app was stopped, with a forget ledger | the first instance to start, before it serves anything | Any copy taken before the last mark the ledger holds, which follows every move of the counter and a stop after any write, or before a forgetting the ledger holds and the copy does not. Only after a crash can a copy of the last five seconds of writes, with no forgetting among them, go unseen: it differs from the database in waits alone. | | while the app was stopped, without a ledger | `bounda restored`, which you run after restoring | What you declare: the database alone cannot tell a copy from itself. | Under the running app, every instance moves a counter in `bounda_checkpoints` one forward after @@ -1029,7 +1029,15 @@ it wrote, at most every five seconds, and remembers what it read: a copy holds l first instance to see it counts the restore in another row, which every other instance then sees changed. An idle instance writes nothing, so neither does its watch. With a forget ledger the counter is also raised in the ledger, as a mark outside the database, which is how a start tells -a copy restored while the app was stopped. Without one, run, after restoring: +a copy restored while the app was stopped; and a forgetting the ledger holds that the database +does not is a restore too, whatever the counter says, so the forgettings a copy undid are +applied again and found by the same reconciliation. A forgetting never depends on the mark: what +a restore that goes unseen leaves is waits, which end on their own within an hour. + +On Workers KV, `kvForgetLedger`, a read may lag a write by up to a minute in another location, so +a start there may read an older mark and miss a copy of the last minute, unless a forgetting +tells it; with only one object per tenant writing it, the mark itself never goes down. A ledger +in a SQL database reads what was written. Without a ledger, run, after restoring: ```bash bounda restored diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 5dc27b7a..d3c79eaf 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -853,6 +853,10 @@ export const assembleApp: AssembleAppFunction = async ({ } else if (typeof forgotten !== "number") { if (ledgerConfig?.strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); } else if (forgotten > 0) { + // The ledger held a forgetting the database does not: a restore, whatever the mark said. + if (cold === null) { + await worker.restored((await restores.declare()).first, { reconciled: true }); + } await dispatcher.catchUp("projection"); } } diff --git a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts index 83bd6636..a7936a96 100644 --- a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts @@ -703,14 +703,17 @@ describe("the forget ledger", () => { one.clock.advance(1); other.clock.advance(1); await one.app.runUntilIdle(); - await other.app.runUntilIdle(); expect(reads).toBe(3); + // The forgetting the ledger held is a restore the other instance learns of: it reads the + // ledger at once too. + await other.app.runUntilIdle(); + expect(reads).toBe(4); expect(await one.forgotten()).toEqual([["c-1", "restored"]]); expect(await one.customer("c-1")).toBeNull(); other.clock.advance(60_000); await other.app.runUntilIdle(); await one.app.runUntilIdle(); - expect(reads).toBe(4); + expect(reads).toBe(5); }); it("waits for another instance's failure before its periodic reconciliation", async () => { diff --git a/packages/core/src/kernel/scheduler/worker.ts b/packages/core/src/kernel/scheduler/worker.ts index 1784d7a0..35d5d7d8 100644 --- a/packages/core/src/kernel/scheduler/worker.ts +++ b/packages/core/src/kernel/scheduler/worker.ts @@ -63,9 +63,10 @@ export interface ScheduledCommandWorker { waitingDeadlines(): number; /** * Starts every background job over from what the database holds, as after a restore, between - * passes; `first` also clears the holds the copy brought back. Never rejects. + * passes; `first` also clears the holds the copy brought back, and `reconciled` leaves alone + * the reconciliation that found it. Never rejects. */ - restored(first: boolean): Promise; + restored(first: boolean, options?: { readonly reconciled?: boolean }): Promise; } export interface CreateScheduledCommandWorkerArgs { @@ -559,7 +560,8 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction // The jobs' holds the copy brought back count failures it undid: the instance that found the // restore first clears them, and every one forgets what it knew of the jobs. Each never rejects. - const restart = async (first: boolean): Promise => { + // `reconciled` when the reconciliation that just ran is what found the restore. + const restart = async (first: boolean, reconciled = false): Promise => { logger.warn("the database went back to an earlier copy; background jobs start over", { first, }); @@ -570,7 +572,7 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction await Promise.all([ retention?.restored(first), rotation?.restored(first), - reconciliation?.restored(first), + reconciled ? undefined : reconciliation?.restored(first), backfill?.restored(first), ]); }; @@ -619,6 +621,11 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction const forgotten = (await retention?.sweep(claimedAt)) ?? 0; const rewrapped = (await rotation?.sweep(claimedAt)) ?? 0; const forgottenAgain = (await reconciliation?.sweep(claimedAt)) ?? 0; + // A forgetting the ledger holds and the database does not: the database went back, though + // the generation did not show it, as for a copy taken just before a forget. + if (forgottenAgain > 0 && restores !== undefined && (restored ?? null) === null) { + await restart((await restores.declare()).first, true); + } const purged = (await letterRetention?.sweep(claimedAt)) ?? 0; const backfilled = (await backfill?.sweep(claimedAt)) ?? 0; if (unreadable !== undefined) throw unreadable.error; @@ -657,6 +664,6 @@ export const createScheduledCommandWorker: CreateScheduledCommandWorkerFunction runOnce, leaseMs, waitingDeadlines: () => waits.size, - restored: (first) => mutex.run(() => restart(first)), + restored: (first, { reconciled } = {}) => mutex.run(() => restart(first, reconciled)), }; }; diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index eaa68723..0426f56e 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -13,6 +13,7 @@ import type { PersonalDataForgottenPayload } from "../../modules/bounda-events.t import type { EventPayloadArgs, PayloadArgs } from "../../modules/payload.ts"; import type { Registry } from "../../modules/registry.ts"; import { type BoundaApp, createApp } from "../app.ts"; +import { defineForgetLedger } from "../personal-data/forget-ledger.ts"; import { HOLD_MAX_MS } from "../personal-data/hold.ts"; import { masterKey } from "../personal-data/master-key.ts"; import { ROTATION_CHECK_INTERVAL_MS } from "../personal-data/rotation.ts"; @@ -359,6 +360,34 @@ describe("a database restored while the app was stopped", () => { expect(restarts(again)).toEqual([]); }); + it("is found from a forgetting the ledger holds and the copy does not, though the mark reads stale", async () => { + const paths = files(); + const store = createNodeSqliteAdapter(paths.storage); + const ledgerAdapter = createNodeSqliteAdapter(paths.ledger).adapter; + // A store whose reads may lag its writes, as Workers KV's: the mark always reads as never raised. + const stale = defineForgetLedger(async () => { + const inner = await ledgerAdapter.createForgetLedger({}); + return { + append: (entries) => inner.append(entries), + read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: (name, value) => inner.raiseMark(name, value), + }; + }); + const runtime: RuntimeConfig = { personalData: { forgetLedger: { store: stale } } }; + const before = await start({ adapter: store.adapter, runtime }); + await register(before.app, "c-1"); + const copy = copyOf(store.db); + await before.app.forget("c-1"); + await before.app.stop(); + restore(store.db, copy); + const logger = recordingLogger(); + const after = await start({ adapter: store.adapter, runtime, logger }); + expect(restarts(logger).map(({ fields }) => fields)).toEqual([{ first: true }]); + expect((await after.app.status()).restoresFound).toBe(1); + expect((await after.app.getLag()).forgetLedger).toEqual({ reconciled: true }); + }); + it("is declared without a ledger by bounda restored, which every running instance learns", async () => { const path = files().storage; const one = createNodeSqliteAdapter(path); diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts index bfdc2a6b..4226f04a 100644 --- a/packages/core/src/kernel/shared/restores.test.ts +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -6,7 +6,6 @@ import { createRecordingLogger } from "../test-support.ts"; import { createRestoreWatch, GENERATION_INTERVAL_MS, - MARK_INTERVAL_MS, RESTORE_GENERATION, RESTORE_MARK, RESTORES_FOUND, @@ -92,23 +91,26 @@ describe("createRestoreWatch", () => { expect(await ledger.readMark(RESTORE_MARK)).toBe(2); await checkpointStore.set(RESTORE_GENERATION, copy); const restarted = watchOver(); - expect(await restarted.start(at(MARK_INTERVAL_MS))).toEqual({ first: true }); + expect(await restarted.start(at(GENERATION_INTERVAL_MS + 1))).toEqual({ first: true }); expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(3); expect(await ledger.readMark(RESTORE_MARK)).toBe(3); - expect(await watchOver().start(at(MARK_INTERVAL_MS + 1))).toBeNull(); + expect(await watchOver().start(at(GENERATION_INTERVAL_MS + 2))).toBeNull(); }); - it("raises the mark as the generation moves, at most once a mark interval, and on a flush", async () => { - const { ledger, watchOver } = await setUp(); + it("raises the mark with every move of the generation, and moves it on a stop after recent writes", async () => { + const { ledger, stream, watchOver } = await setUp(); const watch = watchOver(); await watch.start(at(0)); await watch.check(at(1), { active: true }); - expect(await ledger.readMark(RESTORE_MARK)).toBe(0); - await watch.check(at(MARK_INTERVAL_MS), { active: true }); + expect(await ledger.readMark(RESTORE_MARK)).toBe(1); + await watch.check(at(2), { active: true }); + expect(await ledger.readMark(RESTORE_MARK)).toBe(1); + await watch.flush(); expect(await ledger.readMark(RESTORE_MARK)).toBe(2); - await watch.check(at(MARK_INTERVAL_MS + GENERATION_INTERVAL_MS), { active: true }); + await watch.flush(); expect(await ledger.readMark(RESTORE_MARK)).toBe(2); + stream.head = 4; await watch.flush(); expect(await ledger.readMark(RESTORE_MARK)).toBe(3); }); diff --git a/packages/core/src/kernel/shared/restores.ts b/packages/core/src/kernel/shared/restores.ts index f732b414..fb16ac12 100644 --- a/packages/core/src/kernel/shared/restores.ts +++ b/packages/core/src/kernel/shared/restores.ts @@ -23,11 +23,10 @@ export const RESTORES_FOUND = "runtime:restores"; */ export const RESTORE_MARK = "restore:generation"; +// Also how often the forget ledger's mark follows it at most: Workers KV takes one write a second +// to a key. export const GENERATION_INTERVAL_MS = 5_000; -// How often a moved generation is raised in the forget ledger at most, which a stop also does. -export const MARK_INTERVAL_MS = 60_000; - export interface Restored { /** * Whether this instance found it first: the one that clears what the copy brought back of the @@ -56,8 +55,9 @@ export interface RestoreWatch { */ declare(): Promise; /** - * Raises the forget ledger's mark to the generation this instance read last; what a stop does. - * Never rejects. + * Moves the generation once more when this instance wrote since it last did, whatever the + * interval, and raises the forget ledger's mark to it: what a stop does, so a copy taken before + * the stop always holds less. Never rejects. */ flush(): Promise; /** @@ -108,7 +108,6 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ }) => { let seen: Seen | null = null; let movedAt = Number.NEGATIVE_INFINITY; - let markedAt = Number.NEGATIVE_INFINITY; let head: number | null = null; let pending = false; // While the ledger's mark could not be read, the generation stays where it is: moving it could @@ -140,11 +139,10 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ return { first }; }; - const raise = async (at: number, generation: number): Promise => { + const raise = async (generation: number): Promise => { if (marks === null) return; try { await marks.raiseMark(RESTORE_MARK, generation); - markedAt = at; } catch (error) { logger.error("the forget ledger's mark could not be raised", { error }); } @@ -161,6 +159,19 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ return restored; }; + // Moves the generation past what this instance read, and the mark with it: a copy taken before + // holds less than both from then on. + const move = async (at: number): Promise => { + if (seen === null) return; + const next = seen.generation + 1; + if (await checkpoints.compareAndSet(RESTORE_GENERATION, seen.generation, next)) { + seen = { ...seen, generation: next }; + await raise(next); + } + movedAt = at; + pending = false; + }; + const check = async (now: Date, active: boolean): Promise => { const at = now.getTime(); const [stored, last] = await Promise.all([read(), events.lastPosition()]); @@ -185,16 +196,8 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ } pending = true; } - if (pending && !unmarked && at - movedAt >= GENERATION_INTERVAL_MS) { - const next = current.generation + 1; - if (await checkpoints.compareAndSet(RESTORE_GENERATION, current.generation, next)) { - current = { ...current, generation: next }; - if (at - markedAt >= MARK_INTERVAL_MS) await raise(at, next); - } - movedAt = at; - pending = false; - } seen = current; + if (pending && !unmarked && at - movedAt >= GENERATION_INTERVAL_MS) await move(at); return restored; }; @@ -212,7 +215,7 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ head = await events.lastPosition(); seen = stored; const restored = await compareWithMark(stored); - if (restored === null && marks !== null) await raise(now.getTime(), stored.generation); + if (restored === null && marks !== null) await raise(stored.generation); return restored; } catch (error) { return logged("the database could not be compared with the forget ledger's mark")(error); @@ -228,7 +231,13 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ }, flush: async () => { if (seen === null || unmarked) return; - await raise(Number.NEGATIVE_INFINITY, seen.generation); + try { + const last = await events.lastPosition(); + if (pending || (head !== null && last !== head)) await move(Number.NEGATIVE_INFINITY); + else await raise(seen.generation); + } catch (error) { + logger.error("the database could not be checked for a restore", { error }); + } }, found: () => checkpoints.get(RESTORES_FOUND), }; From 563869fb1f2640bf78b07036fe0445ac14fe9406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:55:41 +0100 Subject: [PATCH 17/19] refactor(core): the restore watch's start takes no time --- packages/core/src/kernel/app.ts | 2 +- .../core/src/kernel/shared/restores.test.ts | 20 +++++++++---------- packages/core/src/kernel/shared/restores.ts | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index d3c79eaf..e8ec3fd6 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -816,7 +816,7 @@ export const assembleApp: AssembleAppFunction = async ({ // A copy restored while the app was stopped holds less than the forget ledger's mark: the // holds it brought back are cleared before anything runs, and the reconciliation below forgets // again whom it brought back. - const cold = await restores.start(clock.now()); + const cold = await restores.start(); if (cold !== null) await worker.restored(cold.first); // An instance that runs no worker still watches for a restore, and reconciles the ledger at // once when it finds one, since no pass of its own would. diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts index 4226f04a..86a1663f 100644 --- a/packages/core/src/kernel/shared/restores.test.ts +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -65,7 +65,7 @@ describe("createRestoreWatch", () => { }); const watch = watchOver(); expect(await watch.check(at(0))).toBeNull(); - expect(await watch.start(at(0))).toBeNull(); + expect(await watch.start()).toBeNull(); expect(entries).toEqual([ { level: "error", @@ -83,7 +83,7 @@ describe("createRestoreWatch", () => { it("finds at start a copy restored while the app was stopped, from the ledger's mark", async () => { const { checkpointStore, ledger, watchOver } = await setUp(); const running = watchOver(); - expect(await running.start(at(0))).toBeNull(); + expect(await running.start()).toBeNull(); const copy = await checkpointStore.get(RESTORE_GENERATION); await running.check(at(0), { active: true }); await running.check(at(GENERATION_INTERVAL_MS), { active: true }); @@ -91,17 +91,17 @@ describe("createRestoreWatch", () => { expect(await ledger.readMark(RESTORE_MARK)).toBe(2); await checkpointStore.set(RESTORE_GENERATION, copy); const restarted = watchOver(); - expect(await restarted.start(at(GENERATION_INTERVAL_MS + 1))).toEqual({ first: true }); + expect(await restarted.start()).toEqual({ first: true }); expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(3); expect(await ledger.readMark(RESTORE_MARK)).toBe(3); - expect(await watchOver().start(at(GENERATION_INTERVAL_MS + 2))).toBeNull(); + expect(await watchOver().start()).toBeNull(); }); it("raises the mark with every move of the generation, and moves it on a stop after recent writes", async () => { const { ledger, stream, watchOver } = await setUp(); const watch = watchOver(); - await watch.start(at(0)); + await watch.start(); await watch.check(at(1), { active: true }); expect(await ledger.readMark(RESTORE_MARK)).toBe(1); await watch.check(at(2), { active: true }); @@ -125,7 +125,7 @@ describe("createRestoreWatch", () => { }); await ledger.raiseMark(RESTORE_MARK, 5); const watch = watchOver(); - expect(await watch.start(at(0))).toBeNull(); + expect(await watch.start()).toBeNull(); expect(await watch.check(at(1), { active: true })).toBeNull(); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(0); await watch.flush(); @@ -143,7 +143,7 @@ describe("createRestoreWatch", () => { }), }); const watch = watchOver(); - expect(await watch.start(at(0))).toBeNull(); + expect(await watch.start()).toBeNull(); await watch.flush(); expect(entries.map(({ message }) => message)).toEqual([ "the forget ledger's mark could not be raised", @@ -154,7 +154,7 @@ describe("createRestoreWatch", () => { it("declares a restore, the first, and raises the mark past it", async () => { const { checkpointStore, ledger, watchOver } = await setUp(); const watch = watchOver(); - await watch.start(at(0)); + await watch.start(); expect(await watch.declare()).toEqual({ first: true }); expect(await checkpointStore.get(RESTORES_FOUND)).toBe(1); expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); @@ -164,7 +164,7 @@ describe("createRestoreWatch", () => { it("touches no ledger and logs nothing without one", async () => { const { entries, watchOver } = await setUp({ marks: false }); const watch = watchOver(); - expect(await watch.start(at(0))).toBeNull(); + expect(await watch.start()).toBeNull(); await watch.check(at(1), { active: true }); await watch.flush(); expect(entries).toEqual([]); @@ -181,7 +181,7 @@ describe("createRestoreWatch", () => { }), }); const watch = watchOver(); - await watch.start(at(0)); + await watch.start(); await watch.flush(); expect(raises).toBe(0); }); diff --git a/packages/core/src/kernel/shared/restores.ts b/packages/core/src/kernel/shared/restores.ts index fb16ac12..b7bd7d19 100644 --- a/packages/core/src/kernel/shared/restores.ts +++ b/packages/core/src/kernel/shared/restores.ts @@ -41,7 +41,7 @@ export interface RestoreWatch { * ledger's mark: `null` while it holds at least that. Never rejects: a ledger that cannot be * read is logged and asked again by `check`. */ - start(now: Date): Promise; + start(): Promise; /** * Reads the rows and resolves to what this instance learnt: `null` while the database holds at * least what it read last. Moves the generation after activity, a write to the stream since the @@ -209,7 +209,7 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ }; return { - start: async (now) => { + start: async () => { try { const stored = await read(); head = await events.lastPosition(); From 9b9caaf2b23002760ecdb5ce2e9f2c0baeaa2d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:58:55 +0100 Subject: [PATCH 18/19] test: pin the restore watch's stop and a restore counted once --- .../src/kernel/shared/restores-app.test.ts | 2 ++ .../core/src/kernel/shared/restores.test.ts | 27 +++++++++++++++++++ packages/core/src/kernel/shared/restores.ts | 20 +++++++------- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index 0426f56e..86388166 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -270,6 +270,8 @@ describe("a database restored while the app runs", () => { .filter((event) => event.type === "PersonalDataForgotten") .map((event) => (event.payload as PersonalDataForgottenPayload).reason); expect(reasons).toEqual(["restored"]); + // Found once, though the reconciliation of the same pass forgot someone again. + expect((await app.status()).restoresFound).toBe(1); }); it("tells every running instance, the first that found it once", async () => { diff --git a/packages/core/src/kernel/shared/restores.test.ts b/packages/core/src/kernel/shared/restores.test.ts index 86a1663f..dab7bc5e 100644 --- a/packages/core/src/kernel/shared/restores.test.ts +++ b/packages/core/src/kernel/shared/restores.test.ts @@ -195,4 +195,31 @@ describe("createRestoreWatch", () => { expect(await watch.declare()).toEqual({ first: true }); expect(await checkpointStore.get(RESTORES_FOUND)).toBe(5); }); + + it("moves nothing on a stop without a write since the last move, and logs a stream it cannot read", async () => { + let down = false; + const { checkpointStore } = await memory().createStorage(); + const ledger = await memory().createForgetLedger({}); + const { logger, entries } = createRecordingLogger("omit"); + const watch = createRestoreWatch({ + checkpoints: checkpointStore, + events: { + lastPosition: async () => { + if (down) throw new Error("down"); + return 0; + }, + }, + marks: ledger, + logger, + }); + await watch.start(); + await watch.check(at(0), { active: true }); + await watch.flush(); + expect(await checkpointStore.get(RESTORE_GENERATION)).toBe(1); + down = true; + await watch.flush(); + expect(entries.map(({ message }) => message)).toEqual([ + "the database could not be checked for a restore", + ]); + }); }); diff --git a/packages/core/src/kernel/shared/restores.ts b/packages/core/src/kernel/shared/restores.ts index b7bd7d19..617209e5 100644 --- a/packages/core/src/kernel/shared/restores.ts +++ b/packages/core/src/kernel/shared/restores.ts @@ -161,11 +161,11 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ // Moves the generation past what this instance read, and the mark with it: a copy taken before // holds less than both from then on. - const move = async (at: number): Promise => { - if (seen === null) return; - const next = seen.generation + 1; - if (await checkpoints.compareAndSet(RESTORE_GENERATION, seen.generation, next)) { - seen = { ...seen, generation: next }; + const move = async (from: Seen, at: number): Promise => { + const next = from.generation + 1; + seen = from; + if (await checkpoints.compareAndSet(RESTORE_GENERATION, from.generation, next)) { + seen = { ...from, generation: next }; await raise(next); } movedAt = at; @@ -197,7 +197,7 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ pending = true; } seen = current; - if (pending && !unmarked && at - movedAt >= GENERATION_INTERVAL_MS) await move(at); + if (pending && !unmarked && at - movedAt >= GENERATION_INTERVAL_MS) await move(current, at); return restored; }; @@ -215,7 +215,8 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ head = await events.lastPosition(); seen = stored; const restored = await compareWithMark(stored); - if (restored === null && marks !== null) await raise(stored.generation); + // A mark only goes up: past a restore it is already higher. + await raise(stored.generation); return restored; } catch (error) { return logged("the database could not be compared with the forget ledger's mark")(error); @@ -233,8 +234,9 @@ export const createRestoreWatch: CreateRestoreWatchFunction = ({ if (seen === null || unmarked) return; try { const last = await events.lastPosition(); - if (pending || (head !== null && last !== head)) await move(Number.NEGATIVE_INFINITY); - else await raise(seen.generation); + if (pending || (head !== null && last !== head)) { + await move(seen, Number.NEGATIVE_INFINITY); + } else await raise(seen.generation); } catch (error) { logger.error("the database could not be checked for a restore", { error }); } From 567fddc58de7fed3786f68adced57ad707222fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 22:49:02 +0100 Subject: [PATCH 19/19] docs: close the migrate example's code fence on its own line --- docs/src/content/docs/guides/deployment.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/guides/deployment.mdx b/docs/src/content/docs/guides/deployment.mdx index 7ee551ea..b2873615 100644 --- a/docs/src/content/docs/guides/deployment.mdx +++ b/docs/src/content/docs/guides/deployment.mdx @@ -182,7 +182,8 @@ database; read `failing`, and the `subscriber failed` log line that names the ev why. The last failure of each subscriber, by any instance, is also kept in the database, with its text encrypted under the person the event names: `bounda status` shows it without the text, and `app.subscriberFailures.get("projection:orderSummary")` opens it, a read the -[access log](/guides/personal-data/#who-opened-what) records. The event itself, payload included, is one read of the global stream away: +[access log](/guides/personal-data/#who-opened-what) records. The event itself, payload included, +is one read of the global stream away: `app.events.read({ after: 41, limit: 1 })` ([Reading the global stream](/guides/reading-the-stream/)). `waitingDeadlines`, next to `subscribers`, counts the process deadlines this instance's worker @@ -228,8 +229,9 @@ bounda status # "storage storage 1 of 2 (migrations pending: bound # stop the previous version bounda migrate # start the new one -``` Each table prefix -keeps its own versions, and each Durable Object, so each tenant, migrates its tables the first time +``` + +Each table prefix keeps its own versions, and each Durable Object, so each tenant, migrates its tables the first time it starts on the new code. A [forget ledger](/guides/personal-data/#backups) in another database is migrated there the first time the app uses it.