From aeef720acb6af6ccdd7cacc1ea311619c990c15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 20:53:01 +0100 Subject: [PATCH 1/6] feat: record every forgetting in the forget ledger before it commits A forgetting committed and then appended to the ledger was lost to a restore when the process stopped between the two steps. Every forgetting now writes its intention first, with its event's id and time set before the transaction, and a reconciliation commits again, under the same id, time and reason, each one the database does not hold while the person still holds a key it names by fingerprint. A strict ledger forgets nobody it has not recorded, every ledger call is bounded by forgetLedger.timeout, bounda forgotten reads the event store without starting the app, and bounda forget-ledger expire gives KV entries written before keepFor their expiration. --- .changeset/forget-ledger-first.md | 15 + docs/src/content/docs/adapters/cloudflare.md | 6 +- docs/src/content/docs/guides/personal-data.md | 143 +++-- docs/src/content/docs/reference/cli.md | 50 +- .../content/docs/reference/configuration.md | 34 +- docs/src/content/docs/reference/errors.md | 10 + packages/cli/package.json | 8 +- .../rebuild-project/strict-ledger.config.ts | 20 + packages/cli/src/forget-ledger-expire.test.ts | 210 ++++++++ packages/cli/src/forget-ledger-expire.ts | 157 ++++++ packages/cli/src/forgotten.test.ts | 72 ++- packages/cli/src/forgotten.ts | 53 +- .../cli/src/run-without-cloudflare.test.ts | 33 ++ packages/cli/src/run.test.ts | 173 +++++- packages/cli/src/run.ts | 146 ++++- packages/cloudflare/package.json | 4 + packages/cloudflare/src/forget-ledger.ts | 165 +++++- packages/cloudflare/src/index.ts | 20 +- .../cloudflare/test/forget-ledger.test.ts | 269 +++++++++- packages/cloudflare/tsdown.config.ts | 2 +- .../src/adapter/sql/forget-ledger.test.ts | 54 +- .../core/src/adapter/sql/forget-ledger.ts | 54 +- packages/core/src/adapter/sql/index.ts | 6 +- .../core/src/adapter/sqlite/migrations.ts | 8 +- .../core/src/adapter/storage/forget-ledger.ts | 27 +- .../adapter/testing/forget-ledger.contract.ts | 16 + .../testing/storage-schema.contract.ts | 8 +- packages/core/src/config/defaults.ts | 4 + packages/core/src/config/index.ts | 1 + packages/core/src/config/schema.test.ts | 16 +- packages/core/src/config/schema.ts | 3 + packages/core/src/config/types.ts | 30 +- packages/core/src/contracts/duration.ts | 4 + packages/core/src/contracts/errors.test.ts | 20 + packages/core/src/contracts/errors.ts | 35 ++ packages/core/src/contracts/index.ts | 5 +- packages/core/src/index.ts | 1 + packages/core/src/kernel/app.ts | 74 ++- .../kernel/command/reaction-commands.test.ts | 107 +++- .../src/kernel/command/reaction-commands.ts | 38 +- .../src/kernel/personal-data/commands.test.ts | 2 + .../src/kernel/personal-data/forget-ledger.ts | 69 ++- .../src/kernel/personal-data/forget.test.ts | 147 ++++- .../core/src/kernel/personal-data/forget.ts | 113 +++- .../kernel/personal-data/intention.test.ts | 168 ++++++ .../src/kernel/personal-data/intention.ts | 102 ++++ .../personal-data/ledger-append.test.ts | 267 ---------- .../src/kernel/personal-data/ledger-append.ts | 124 ----- .../personal-data/ledger-outbox.test.ts | 177 +++++- .../src/kernel/personal-data/ledger-outbox.ts | 71 ++- .../personal-data/reconciliation-app.test.ts | 502 +++++++++++++++--- .../personal-data/reconciliation.test.ts | 270 +++++++++- .../kernel/personal-data/reconciliation.ts | 161 ++++-- .../personal-data/retention-app.test.ts | 8 +- .../kernel/personal-data/retention.test.ts | 102 +++- .../src/kernel/personal-data/retention.ts | 104 +++- packages/core/src/kernel/policy/executor.ts | 4 + packages/core/src/kernel/process/handlers.ts | 4 + packages/core/src/kernel/process/runner.ts | 4 + packages/core/src/kernel/reactive-harness.ts | 4 + .../core/src/kernel/shared/runtime-text.ts | 4 + .../core/src/kernel/system-events.test.ts | 17 + packages/core/src/kernel/system-events.ts | 10 +- packages/core/src/kernel/test-support.ts | 14 +- .../kernel/unit-of-work/unit-of-work.test.ts | 41 ++ .../src/kernel/unit-of-work/unit-of-work.ts | 20 +- packages/core/src/memory/forget-ledger.ts | 14 +- packages/core/src/modules/bounda-events.ts | 19 +- packages/core/src/modules/policy.ts | 4 +- packages/core/test-types/config.test-d.ts | 12 +- packages/core/test-types/inference.test-d.ts | 7 +- packages/postgresql/src/migrations.ts | 7 + packages/sqlite/src/forget-ledger.test.ts | 89 +++- pnpm-lock.yaml | 6 + 74 files changed, 3963 insertions(+), 805 deletions(-) create mode 100644 .changeset/forget-ledger-first.md create mode 100644 packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts create mode 100644 packages/cli/src/forget-ledger-expire.test.ts create mode 100644 packages/cli/src/forget-ledger-expire.ts create mode 100644 packages/cli/src/run-without-cloudflare.test.ts create mode 100644 packages/core/src/kernel/personal-data/intention.test.ts create mode 100644 packages/core/src/kernel/personal-data/intention.ts delete mode 100644 packages/core/src/kernel/personal-data/ledger-append.test.ts delete mode 100644 packages/core/src/kernel/personal-data/ledger-append.ts diff --git a/.changeset/forget-ledger-first.md b/.changeset/forget-ledger-first.md new file mode 100644 index 00000000..d2331d9f --- /dev/null +++ b/.changeset/forget-ledger-first.md @@ -0,0 +1,15 @@ +--- +"@bounda-dev/core": minor +"@bounda-dev/sqlite": minor +"@bounda-dev/postgresql": minor +"@bounda-dev/cloudflare": minor +"@bounda-dev/cli": minor +--- + +With a forget ledger, every forgetting is in the ledger before it commits. `forget`, `forgetAgain`, a policy's or process's `forget` and the retention write the forgetting's intention first, its event's id, time and reason set before the transaction, which appends the event with them; a reconciliation commits again, under the same id, time and reason, every forgetting the database does not hold whose person still holds a key it names, so an intention whose commit never happened, because the process stopped or the transaction failed, is applied at the next start, after a restore too. A `forget` that rejects after its intention was written still forgets the person then. A forgetting the retention made is committed again only while the retention would still forget the person. A reconciliation leaves an entry younger than `FORGET_LEDGER_GRACE_MS` to a later one, due once it is old enough. + +Ledger entries and `PersonalDataForgotten` name the keys a forgetting destroys by their fingerprint: `ForgetLedgerEntry` gains `reason` and `fingerprints`, the payload gains `fingerprints`, and a reconciliation leaves a key the entry does not name, given to the person after it. The reason `"restored"` is gone: a forgetting committed again keeps its own. The SQL ledger table gains `reason` and `fingerprints` in version 2 of its `forget-ledger` scope, and the KV ledger keeps them in each key's metadata; an entry written before reads as a request that names no keys. + +`forgetLedger.strict` now also means nothing is forgotten that the ledger has not recorded: a forgetting whose intention the ledger does not take fails with `ForgetLedgerUnavailableError` (`FORGET_LEDGER_UNAVAILABLE`) and forgets nobody, so trying again is safe. `forgetLedger.timeout` (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS`, ten seconds) bounds each call to the ledger, given up with `ForgetLedgerTimeoutError` (`FORGET_LEDGER_TIMEOUT`), and how long a start that is not strict waits for the reconciliation; a strict start fails once a read runs past it. `parseDuration` is exported. + +`bounda forgotten` reads the event store without starting the app, so a strict ledger that is down does not stop it, and prints each forgetting's reason and fingerprints; `bounda forget --from` commits each whole line again under its own id, time and reason. `bounda forget-ledger expire --binding --keep-for ` gives every key of a Workers KV ledger the expiration `keepFor` sets, those written before it was set included, through the project's wrangler; `expireKvForgetLedger` and `planKvForgetLedgerExpiry` from `@bounda-dev/cloudflare`, and its new `./forget-ledger` subpath, do the same in code. diff --git a/docs/src/content/docs/adapters/cloudflare.md b/docs/src/content/docs/adapters/cloudflare.md index 5f6899f1..c8728ade 100644 --- a/docs/src/content/docs/adapters/cloudflare.md +++ b/docs/src/content/docs/adapters/cloudflare.md @@ -157,7 +157,11 @@ in `wrangler.jsonc` and name it in `runtime.personalData.forgetLedger.store`; an `ConfigurationError`. Each object keeps its entries under its own name, and reads the ledger once more a minute after it starts, since KV may take that long to list a key. The periodic reconciliation never wakes an object on its own, and with `keepFor` each key expires in KV by -itself. See [backups](/guides/personal-data/#backups). +itself; `bounda forget-ledger expire` gives the keys written before `keepFor` was set theirs. +Every forgetting writes its intention to KV before the object commits it, so one the object did not +get to commit, because it was evicted or its transaction failed, is committed at its next start; +with `strict`, a forgetting KV does not take fails and forgets nobody. See +[backups](/guides/personal-data/#backups). ## One object per tenant diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index 1456d05f..02f12da2 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -201,11 +201,12 @@ context of React Router. A client bound with `as` records who asked on `Personal `metadata.actor`, and so on `ProcessingRestricted` and `ProcessingRestrictionLifted`. In one transaction it deletes the person's key, cancels the scheduled commands that hold their data, removes the dead letters of those that gave up, and appends `PersonalDataForgotten` to the stream `bounda:`: the runtime's own aggregate, `bounda`, -has one stream per person it forgot. Its payload is `{ personId, reason, cancelledCommands, -removedDeadLetters }`: `reason` is `"requested"` for `forget`, `"retention"` when the -[retention](#retention) forgot them and `"restored"` when they were forgotten again after a -[backup was restored](#backups), and the rest are the ids and types of what went, the only -record of commands that never became events. It resolves like a command, with the event's +has one stream per person it forgot. Its payload is `{ personId, reason, fingerprints, +cancelledCommands, removedDeadLetters }`: `reason` is `"requested"` for `forget` and +`"retention"` when the [retention](#retention) forgot them, `fingerprints` names the key it +destroyed, by its check value, which opens nothing, none for a person who held none, and the rest +are the ids and types of what went, the only record of commands that never became events. A +forgetting a [forget ledger](#backups) commits again keeps its id, its time and its reason. It resolves like a command, with the event's position, so `catchUpReadModels({ through })` waits for the read models that handle it, and the Durable Object's `forget` and the one in React Router's `bounda` context wait for them themselves unless `consistency` is `"eventual"`. A person without a key is forgotten all the same, since a read model may hold what their events carried in @@ -815,9 +816,11 @@ is restored, to apply the erasures again. A forget ledger is how Bounda does the ### The forget ledger The ledger is a record of every forgetting kept outside the database: for each `PersonalDataForgotten`, -the person's id, the event's id and its time, and nothing else. It is a suppression list, the one -thing about a forgotten person that has to outlive their data, so it holds no field of theirs and -no reason. Give it a store a restore of the database does not touch: +the person's id, the event's id, its time, its reason, and the fingerprints of the keys it +destroys, and nothing else. It is a suppression list, the one thing about a forgotten person that +has to outlive their data, so it holds no field of theirs; a fingerprint is a check value of a key, +which opens nothing and names nothing once the key is gone. Give it a store a restore of the +database does not touch: ```ts import { defineConfig, masterKey } from "@bounda-dev/core/config"; @@ -877,32 +880,56 @@ time; `forgetLedgerContract` from `@bounda-dev/core/adapter/testing` is the test ### What the runtime does with it -Every transaction that forgets someone appends the forgetting to the ledger as soon as it commits, -before `forget` resolves, whoever forgot them: `forget`, a policy or process, the retention or a -reconciliation. The event store is also the ledger's outbox: a subscriber of the dispatcher, -`forget-ledger`, copies every `PersonalDataForgotten` to the ledger again in batches, with a -policy's guarantee, and the ledger keeps one entry per event however often it receives it. So a -forgetting the ledger refused, or did not take within ten seconds, which the app logs as a -warning, still reaches it: a batch the ledger refuses is delivered again, and the subscriber's -lag in `getLag` and `bounda.dispatcher.lag` shows how far behind it is. Configuring a ledger on an +Every forgetting writes its intention to the ledger first, and commits after: `forget`, a policy +or process, the retention, `forgetAgain`. The intention is the entry the forgetting will be: its +event's id and time are set before the transaction, and the transaction appends the event with +them, so the ledger and the database name one forgetting by one id. A policy's or process's +`forget` writes it just before its run's transaction opens, once the run has decided. + +What happens to an intention whose commit does not happen, because the process stopped in between +or the database refused the transaction, is the same in every case: the next reconciliation +commits it, since forgetting is what was asked. A `forget` that rejects after its intention was +written still forgets the person then; calling it again forgets them at once, and is never an +error. The retention is the one exception, as it decides on the person's activity: a forgetting +the retention made is committed again only while the retention would still forget the person, +read on the database as it is then, so a person who came back, or whose processing was +restricted, keeps their data. + +An intention names the key the person held when it was written, and a reconciliation commits it +only while the person still holds that key: a key they hold that it does not name was given to +them after, as when they come back and register again, and is kept. That also settles the order +between several intentions of one person, which never changes the result: each commits only while +the key it names is there, so the first to commit destroys it and the others find it gone. A +reconciliation leaves an entry younger than a minute, `FORGET_LEDGER_GRACE_MS` from +`@bounda-dev/core`, to the next one, due once it is old enough: it may be the intention of a +forgetting still committing, which then commits with its own actor. + +The event store is also the ledger's outbox: a subscriber of the dispatcher, `forget-ledger`, +copies every `PersonalDataForgotten` to the ledger again in batches, with a policy's guarantee, +and the ledger keeps one entry per event however often it receives it. So a forgetting whose +intention the ledger refused, or did not take within its `timeout`, which an app that is not +strict logs as a warning and commits all the same, still reaches it: a batch the ledger refuses is +delivered again, and the subscriber's lag in `getLag` and `bounda.dispatcher.lag` shows how far +behind it is. While an intention the ledger did not take in time is still pending, the next +forgettings do not wait for it, and leave their copy to the subscriber. Configuring a ledger on an app that already forgot people copies those forgettings first. When the app starts, before it serves anything, it reads the ledger past a cursor the database -keeps, the time of the last entry it applied, and for each entry whose forgetting the database -does not hold, it forgets the person again, with `reason: "restored"`, if they still hold a key. -After a normal start that read returns next to nothing. After a restore the cursor went back with -the copy, and the read returns exactly the forgettings made since the copy was taken; the read -models that show those persons hear of it before the start resolves, and the app logs how many it -forgot again: +keeps, the time of the last entry it applied, and commits again, under its own id, time and +reason, each forgetting the database does not hold whose person still holds a key it names. After +a normal start that read returns next to nothing. After a restore the cursor went back with the +copy, and the read returns exactly the forgettings made since the copy was taken, the intentions +whose commit never happened included; the read models that show those persons hear of it before +the start resolves, and the app logs how many it committed again: ``` -forgot again persons a restored backup brought back +committed again forgettings the database did not hold ``` An entry whose forgetting the database holds is skipped even when the person holds a key, since -that key was given to them after they were forgotten, as when they come back and register again. -The check and the forgetting are one transaction, so instances starting together forget a person -once. A Durable Object does it on every cold start; restoring its storage restarts it. +that key was given to them after they were forgotten. The check and the forgetting are one +transaction, so instances starting together forget a person once. A Durable Object does it on +every cold start; restoring its storage restarts it. While the app runs, it reads the ledger again every `reconcileEvery`, five minutes unless you set it, so a database restored under a running app forgets again within that time. One instance @@ -912,31 +939,39 @@ round runs on the worker's passes, in every instance whose role runs the worker; runs it on the passes it wakes for, never waking for it alone, since restoring its storage restarts it anyway. Restarting the app after a restore reconciles at once. -A ledger that cannot be read when the app starts does not stop it: the app logs an error, starts, -and its worker tries again after 30 seconds, then twice as long after each failure, up to an hour. +Each call to the ledger is given up after its `timeout`, ten seconds unless you set it, with +[`FORGET_LEDGER_TIMEOUT`](/reference/errors/#codes); the call may still land after it. A ledger +that cannot be read when the app starts does not stop it: the app logs an error, starts, and its +worker tries again after 30 seconds, then twice as long after each failure, up to an hour. The wait is kept in `bounda_checkpoints`, beside the cursor, so every instance waits for the longest and a restart backs off from where it was, though a start always tries once; the first reconciliation that succeeds clears it. -One that does not answer within ten seconds is left to finish in the background, with an error -logged, and the app starts. +A reconciliation that takes longer than the `timeout` as a whole, each call within it, is left to +finish in the background, with an error logged, and the app starts. Until then `getLag()` says `forgetLedger: { reconciled: false }`, and a person the restore brought back can be read; alert on it. An app without a ledger reports `forgetLedger: null`. -To never serve before the ledger was read, make it strict: +To never forget anyone the ledger has not recorded, nor serve before the ledger was read, make it +strict: ```ts forgetLedger: { store: sqlite({ path: "./ledger/forget-ledger.db" }), strict: true }, ``` -A strict app waits at start for the reconciliation however long it takes, and a ledger it cannot -read fails the start with +A strict app forgets nobody whose intention the ledger did not take: `forget`, `forgetAgain` and +a policy's `forget` fail with [`FORGET_LEDGER_UNAVAILABLE`](/reference/errors/#codes), the +ledger's error as its `cause`, and nothing is forgotten, so trying again is safe; a policy's run +fails and is retried as any other, and the retention holds the person and tries again later. It +waits at start for the reconciliation however long it takes, each call within the `timeout`, and a +ledger it cannot read fails the start with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes), the ledger's error as its `cause`; -starting again tries again. The CLI starts the app too, so its commands need the ledger then. -Availability now depends on the ledger's: give it a store at least as available as the database. +starting again tries again. The CLI's commands that start the app need the ledger then; +`bounda forgotten` does not, since it reads the event store alone. Availability now depends on the +ledger's: give it a store at least as available as the database. -A restore loses only what the ledger never received: a forgetting committed in the moment before -the process stopped, before its append, and that the subscriber had not copied yet, or one made -while the ledger was down whose copy was still waiting when the database was lost. Keep the +A strict app loses no forgetting to a restore: each is in the ledger before it commits. One that is +not strict loses only a forgetting whose intention the ledger did not take, while it was down or +slow, and that the subscriber had not copied yet when the database was lost. Keep the `forget-ledger` subscriber's lag near zero, as for any other. ### Keeping the ledger short @@ -952,8 +987,22 @@ The instance that takes a periodic round prunes after it, through the ledger's ` forgetting older than `keepFor` is not appended at all. KV expires each key on its own instead, `keepFor` after its forgetting, since it cannot list its oldest keys first. A backup older than `keepFor` brings back whom the removed entries named, so when you keep backups longer, raise it -first. Without `keepFor` entries are kept for good: ids and times only, but still the ids of -people who asked to be forgotten. +first. Without `keepFor` entries are kept for good: ids, times and fingerprints only, but still the +ids of people who asked to be forgotten. + +A KV key gets its expiration when it is written, so the keys written before `keepFor` was set, or +while it was longer, keep theirs. Give them the one `keepFor` sets now, from the project's root, +with the binding and the `keepFor` of the configuration: + +```bash +bounda forget-ledger expire --binding FORGET_LEDGER --keep-for 35d +``` + +It goes through the project's wrangler, logged in to the account, to the deployed namespace, +`--local` to the state of `wrangler dev`, with `--env` and `--wrangler-config` as wrangler takes +them; it writes every object's keys again with their expiration and removes those already past +it. `expireKvForgetLedger({ namespace, keepForMs })` from `@bounda-dev/cloudflare` does the same +from a Worker, within the KV operations one invocation may make. ### Without a ledger @@ -971,12 +1020,14 @@ Then, once the copy is back: bounda forget --from forgotten.jsonl ``` -`bounda forgotten` prints one line of JSON per forgetting, the entry a ledger would hold; -`bounda forget` forgets again, with `reason: "restored"`, each person listed who still holds a key, -unless the database holds the forgetting a line names. A line may also be a bare person's id, for -a list kept some other way, which is forgotten whenever the person holds a key: list only those -forgotten after the copy was taken. `--actor ` names who does it on each `PersonalDataForgotten` -it appends. `app.forgetAgain(entries, { actor })` is the same in code. +`bounda forgotten` prints one line of JSON per forgetting, the entry a ledger would hold, reading +the event store without starting the app; `bounda forget` commits again each forgetting listed, +under its own id, time and reason, as a reconciliation would: unless the database holds it, and +only while the person holds a key it names. With a ledger configured, it records each in the +ledger first. A line may also be a bare person's id, for a list kept some other way, which is +forgotten, as a request, whenever the person holds a key: list only those forgotten after the copy +was taken. `--actor ` names who does it on each `PersonalDataForgotten` it appends. +`app.forgetAgain(entries, { actor })` is the same in code. Without either, a restored backup brings back the data of everyone forgotten after it was taken, until they are forgotten again. diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index f4f4f9a5..e98fa06d 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -245,8 +245,9 @@ process letter it also says when one of the parked events failed the process aga ## `bounda forgotten` and `bounda forget` Apply again, by hand, the forgettings a restored backup undid, for an app without a -[forget ledger](/guides/personal-data/#backups). Both boot the project as `bounda dead-letters` -does. +[forget ledger](/guides/personal-data/#backups). `bounda forget` boots the project as +`bounda dead-letters` does; `bounda forgotten` only opens the project's storage and reads its +event store, without starting the app, so a strict forget ledger that is down does not stop it. ```bash bounda forgotten --since 2026-11-01T03:00:00Z > forgotten.jsonl @@ -257,18 +258,20 @@ bounda forget --from forgotten.jsonl `bounda forgotten` reads the stream and prints, one line of JSON each, the `PersonalDataForgotten` of every person forgotten since `--since`: a date, compared with each event's time, which reads the whole stream, or a position of the stream to read after. Each line is -`{"personId":…,"eventId":…,"forgottenAt":…}`, the entry a ledger would hold; the count goes to +`{"personId":…,"eventId":…,"forgottenAt":…,"reason":…,"fingerprints":[…]}`, the entry a ledger +would hold, without `fingerprints` for a forgetting recorded before Bounda 0.4.0; the count goes to standard error, so standard output is the list alone. Run it against the database before it is restored, or regularly next to the backups. -`bounda forget --from ` forgets again, with `reason: "restored"`, each person listed who -still holds a key, unless the database holds the forgetting the line names by `eventId`. A line -may also be a bare person's id, forgotten whenever they hold a key; blank lines are skipped. It -prints each person it forgot and how many of those listed: +`bounda forget --from ` commits again each forgetting listed, under its own id, time and +reason, unless the database holds it, and only while the person holds a key the line names, as a +reconciliation would; with a forget ledger configured, it records each in the ledger first. A +line may also be a bare person's id, forgotten as a request whenever they hold a key; blank lines +are skipped. It prints each person it forgot and how many of those listed: ``` forgot again c-42 -forgot again 1 of 3 listed; the rest hold no key, or this store holds their forgetting +forgot again 1 of 3 listed; the rest hold no key the line names, or this store holds their forgetting ``` | Option | Applies to | Meaning | @@ -281,6 +284,37 @@ forgot again 1 of 3 listed; the rest hold no key, or this store holds their forg Exit codes: `0` done, `1` bad arguments, `2` the project could not be loaded, or the file could not be read or has a line that is neither a person's id nor an entry, in which case nobody is forgotten. +## `bounda forget-ledger expire` + +Gives every entry of a Workers KV [forget ledger](/guides/personal-data/#keeping-the-ledger-short) +the expiration `keepFor` sets: KV expires a key only as it was written, so the keys written before +`keepFor` was set, or while it was longer, keep the expiration they had, none at first. Run it once +after setting or shortening `keepFor`, from the project's root. + +```bash +bounda forget-ledger expire --binding FORGET_LEDGER --keep-for 35d +``` + +It runs the project's own wrangler, logged in to the account: it lists every object's keys under +the ledger, writes again with their expiration those that expire later than `keepFor` after their +forgetting or never, and removes those already past it, through `wrangler kv bulk`. A key that is +not an entry of the ledger is never touched. It prints what it changed: + +``` +gave 1204 entries the expiration keepFor sets; removed 37 already past it +``` + +| Option | Meaning | +| --- | --- | +| `--binding ` | The KV namespace's binding in `wrangler.jsonc`, as `kvForgetLedger({ binding })` names it | +| `--keep-for ` | The configuration's `keepFor`, such as `35d` | +| `--local` | The local state of `wrangler dev` instead of the deployed namespace | +| `--env `, `--wrangler-config `, `--persist-to ` | As wrangler takes `--env`, `--config` and `--persist-to` | +| `--root ` | The project's root, where wrangler is installed | + +Exit codes: `0` done, `1` bad arguments, `2` wrangler failed, is not installed, or +`@bounda-dev/cloudflare` is not; what wrangler printed of its failure is shown. + ## Programmatic use `bounda generate` is exported from `@bounda-dev/cli` as `generate`: diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 4c777dec..63a1222c 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -123,29 +123,41 @@ a stopped one left: see [delivery guarantees](/concepts/delivery-guarantees/#the warns at start when the registry marks personal fields. It is app-wide. See [where the keys live](/guides/personal-data/#where-the-keys-live). - **`personalData.forgetLedger`**, none by default. Where every forgetting is recorded outside the - database, by person's id, event id and time, and how the app reconciles with it. Without it, a + database, before it commits, by person's id, event id, time, reason and the fingerprints of the + keys it destroys, and how the app reconciles with it. Without it, a restored backup brings back everyone forgotten after it was taken. See [backups](/guides/personal-data/#backups). It holds: - **`store`**, required. The ledger: an adapter pointed at another database than `storage`, such as `sqlite({ path: "./ledger/forget-ledger.db" })` or `postgresql({ url })`, which creates its `bounda_forget_ledger` table there on first use; `kvForgetLedger({ binding })` from `@bounda-dev/cloudflare` on Cloudflare; or a ledger of your own through `defineForgetLedger`. - The runtime appends each forgetting to it as the forgetting commits, and copies every + Every forgetting writes its intention to it before it commits, and the runtime copies every `PersonalDataForgotten` to it again from the stream in the background. An adapter pointing at the storage's own database fails the start. - **`reconcileEvery`**, `5m` by default (`DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS` in `@bounda-dev/core/config`). The app reads the ledger when it starts and, while it runs, again - this often, one instance a round, and forgets again, with `reason: "restored"`, whoever a - restored backup brought back. A Durable Object only runs a round on the passes it wakes for - anyway; it reconciles on every start. - - **`strict`**, `false` by default. When `false`, a ledger that cannot be read at start, or does - not answer within ten seconds, does not fail it: the app logs an error and reconciles in the - background, and `getLag()` reports `forgetLedger: { reconciled: false }` meanwhile. When - `true`, the start waits for the reconciliation however long it takes, and a ledger it cannot - read fails it with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes). + this often, one instance a round, and commits again, under its own id, time and reason, each + forgetting the database does not hold whose person still holds a key it names: one a restored + backup rolled back, or one whose commit never happened. A Durable Object only runs a round on + the passes it wakes for anyway; it reconciles on every start. + - **`strict`**, `false` by default. When `false`, a forgetting whose intention the ledger does + not take is logged and commits all the same, for the stream to copy later, and a ledger that + cannot be read at start, or a reconciliation longer than `timeout`, does not fail it: the app + logs an error and reconciles in the background, and `getLag()` reports + `forgetLedger: { reconciled: false }` meanwhile. When `true`, nothing is forgotten that the + ledger has not recorded: a forgetting whose intention it does not take fails with + [`FORGET_LEDGER_UNAVAILABLE`](/reference/errors/#codes) and forgets nobody, so trying again is + safe; and the start waits for the reconciliation however long it takes, and a ledger it cannot + read fails it with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes). `bounda forgotten` + needs neither, since it reads the event store alone. - **`keepFor`**, none by default. How long an entry is kept after the forgetting; set it longer than the oldest backup you can restore. The instance that runs a periodic round prunes the - older entries after it; KV expires each key on its own. Without it entries are kept for good. + older entries after it; KV expires each key on its own, and `bounda forget-ledger expire` gives + the keys written before `keepFor` was set theirs. Without it entries are kept for good. + - **`timeout`**, `10s` by default (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS` in + `@bounda-dev/core/config`). How long each call to the ledger may take before it is given up + with [`FORGET_LEDGER_TIMEOUT`](/reference/errors/#codes), and how long a start that is not + strict waits for the reconciliation. ### Dead letters diff --git a/docs/src/content/docs/reference/errors.md b/docs/src/content/docs/reference/errors.md index 1e952a08..70dc284b 100644 --- a/docs/src/content/docs/reference/errors.md +++ b/docs/src/content/docs/reference/errors.md @@ -75,6 +75,16 @@ dead letter with `errorType: "retriable_exhausted"`. is malformed: an unknown key, a port with no implementation chosen, a module missing an export. Thrown at boot, or when a handler reads a port a test left out. Also thrown at start by a database whose Bounda tables a version before 0.2.0 created. +- **`FORGET_LEDGER_UNAVAILABLE`** (`ForgetLedgerUnavailableError`; a forgetting's answer). Thrown + by `forget`, `forgetAgain`, `bounda forget` and a policy's or process's `forget` when + `runtime.personalData.forgetLedger.strict` is set and the + [forget ledger](/guides/personal-data/#what-the-runtime-does-with-it) did not take the + forgetting's intention: nothing was forgotten, so trying again is safe. `cause` is the ledger's + error, a `ForgetLedgerTimeoutError` when it did not answer in time. +- **`FORGET_LEDGER_TIMEOUT`** (`ForgetLedgerTimeoutError`, with `operation` and `timeoutMs`). A call + to the forget ledger did not settle within `runtime.personalData.forgetLedger.timeout`: the + `cause` of `FORGET_LEDGER_UNAVAILABLE` and `FORGET_LEDGER_UNRECONCILED`, and what an app that is + not strict logs. The call may still land after it. - **`FORGET_LEDGER_UNRECONCILED`** (`ForgetLedgerUnreconciledError`; a start's answer). Thrown by `createApp`, and so by `boot` and the CLI, when `runtime.personalData.forgetLedger.strict` is set and the [forget ledger](/guides/personal-data/#the-forget-ledger) could not be read through: diff --git a/packages/cli/package.json b/packages/cli/package.json index 8290c946..8ddeb500 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,21 +41,27 @@ "commander": "catalog:" }, "peerDependencies": { + "@bounda-dev/cloudflare": "workspace:*", "@bounda-dev/core": "workspace:*", "typescript": ">=7.0.0" }, "peerDependenciesMeta": { + "@bounda-dev/cloudflare": { + "optional": true + }, "typescript": { "optional": true } }, "devDependencies": { + "@bounda-dev/cloudflare": "workspace:*", "@bounda-dev/core": "workspace:*", "@stryker-mutator/core": "catalog:", "@stryker-mutator/vitest-runner": "catalog:", "@types/node": "catalog:", "tsdown": "catalog:", "typescript": "catalog:", - "vitest": "catalog:" + "vitest": "catalog:", + "wrangler": "catalog:" } } diff --git a/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts b/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts new file mode 100644 index 00000000..55d74300 --- /dev/null +++ b/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts @@ -0,0 +1,20 @@ +import { defineConfig, defineForgetLedger } from "@bounda-dev/core/config"; +import { memory } from "@bounda-dev/core/memory"; + +const down = async (): Promise => { + throw new Error("ledger is down"); +}; + +// A strict forget ledger that is down: the app does not start, and forgets nobody. +export default defineConfig({ + storage: memory(), + runtime: { + role: "all", + personalData: { + forgetLedger: { + store: defineForgetLedger(() => ({ append: down, read: down })), + strict: true, + }, + }, + }, +}); diff --git a/packages/cli/src/forget-ledger-expire.test.ts b/packages/cli/src/forget-ledger-expire.test.ts new file mode 100644 index 00000000..628489de --- /dev/null +++ b/packages/cli/src/forget-ledger-expire.test.ts @@ -0,0 +1,210 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import * as ledger from "@bounda-dev/cloudflare/forget-ledger"; +import { afterAll, describe, expect, it } from "vitest"; +import { expireKvLedger, type Wrangler, wranglerIn } from "./forget-ledger-expire.ts"; + +const DAY = 86_400_000; +const NOW = Date.UTC(2026, 10, 9); +const ROOT = ledger.KV_FORGET_LEDGER_ROOT; + +const temporary: string[] = []; + +afterAll(async () => { + for (const directory of temporary) await rm(directory, { recursive: true, force: true }); +}); + +const metadata = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(NOW - daysAgo * DAY).toISOString(), + reason: "requested", +}); + +// Answers the listing with `listed`, and keeps what each bulk command was handed. +const fakeWrangler = (listed: unknown) => { + const calls: (readonly string[])[] = []; + const files: unknown[] = []; + const wrangler: Wrangler = async (args) => { + calls.push(args); + if (args[1] === "bulk") files.push(JSON.parse(await readFile(args[3] ?? "", "utf8"))); + return args[1] === "key" ? JSON.stringify(listed) : ""; + }; + return { wrangler, calls, files }; +}; + +describe("expiring a KV forget ledger through wrangler", () => { + it("writes again, with an expiration, the entries kept for good or too long, and removes those past keepFor", async () => { + const { wrangler, calls, files } = fakeWrangler([ + { name: `${ROOT}t/1:e-1`, metadata: metadata(1) }, + { name: `${ROOT}t/2:e-2`, metadata: metadata(40) }, + { name: `${ROOT}t/3:e-3`, metadata: metadata(2), expiration: NOW / 1000 + 400 * 86_400 }, + { name: `${ROOT}t/4:e-4`, metadata: metadata(3), expiration: NOW / 1000 + 86_400 }, + { name: `${ROOT}t/5:e-5` }, + ]); + const expired = await expireKvLedger({ + wrangler, + ledger, + binding: "FORGET_LEDGER", + keepForMs: 35 * DAY, + target: { local: false }, + now: NOW, + }); + expect(expired).toEqual({ expired: 2, removed: 1 }); + expect(calls.map((args) => args.slice(0, 3))).toEqual([ + ["kv", "key", "list"], + ["kv", "bulk", "put"], + ["kv", "bulk", "delete"], + ]); + expect(calls[0]).toEqual([ + "kv", + "key", + "list", + "--binding", + "FORGET_LEDGER", + "--prefix", + ROOT, + "--remote", + ]); + expect(calls[1]?.slice(4)).toEqual(["--binding", "FORGET_LEDGER", "--remote"]); + expect(calls[2]?.slice(4)).toEqual(["--binding", "FORGET_LEDGER", "--force", "--remote"]); + expect(files).toEqual([ + [ + { + key: `${ROOT}t/1:e-1`, + value: "", + expiration: (NOW + 34 * DAY) / 1000, + metadata: metadata(1), + }, + { + key: `${ROOT}t/3:e-3`, + value: "", + expiration: (NOW + 33 * DAY) / 1000, + metadata: metadata(2), + }, + ], + [`${ROOT}t/2:e-2`], + ]); + }); + + it("calls no bulk command when nothing changes, and names the target it was given", async () => { + const { wrangler, calls } = fakeWrangler([]); + expect( + await expireKvLedger({ + wrangler, + ledger, + binding: "LEDGER", + keepForMs: DAY, + target: { local: true, env: "staging", config: "w.jsonc", persistTo: ".state" }, + now: NOW, + }), + ).toEqual({ expired: 0, removed: 0 }); + expect(calls).toEqual([ + [ + "kv", + "key", + "list", + "--binding", + "LEDGER", + "--prefix", + ROOT, + "--local", + "--env", + "staging", + "--config", + "w.jsonc", + "--persist-to", + ".state", + ], + ]); + }); + + it("refuses a listing that is not a list of keys, changing nothing", async () => { + for (const listed of [{}, [{ name: 3 }], [{ name: "k", expiration: "soon" }], [null]]) { + const { wrangler, calls } = fakeWrangler(listed); + await expect( + expireKvLedger({ + wrangler, + ledger, + binding: "LEDGER", + keepForMs: DAY, + target: { local: true }, + now: NOW, + }), + ).rejects.toThrow("wrangler kv key list printed something else than a list of keys"); + expect(calls).toHaveLength(1); + } + }); + + it("says wrangler is missing from a project without it, or installed without its executable", async () => { + const root = await mkdtemp(join(tmpdir(), "bounda-no-wrangler-")); + temporary.push(root); + await expect(wranglerIn(root)(["--version"])).rejects.toThrow( + `wrangler is not installed in ${root}: add it to the project`, + ); + const nested = join(root, "app", "nested"); + await mkdir(join(root, "node_modules", "wrangler"), { recursive: true }); + await mkdir(nested, { recursive: true }); + const packageJson = join(root, "node_modules", "wrangler", "package.json"); + await writeFile(packageJson, JSON.stringify({ name: "wrangler" })); + await expect(wranglerIn(nested)(["--version"])).rejects.toThrow( + `${packageJson} names no wrangler executable`, + ); + await writeFile(packageJson, JSON.stringify({ name: "wrangler", bin: "cli.js" })); + await writeFile(join(root, "node_modules", "wrangler", "cli.js"), 'console.log("4.0.0");'); + expect(await wranglerIn(nested)(["--version"])).toBe("4.0.0\n"); + await writeFile( + packageJson, + JSON.stringify({ name: "wrangler", bin: { wrangler: "cli.js", wrangler2: "other.js" } }), + ); + expect(await wranglerIn(root)(["--version"])).toBe("4.0.0\n"); + }); + + it("gives the entries in wrangler's local state their expiration, as the deployed namespace would get them", async () => { + const directory = await mkdtemp(join(tmpdir(), "bounda-kv-expire-")); + temporary.push(directory); + const config = join(directory, "wrangler.jsonc"); + await writeFile( + config, + JSON.stringify({ + name: "kv-expire", + compatibility_date: "2025-10-08", + kv_namespaces: [{ binding: "FORGET_LEDGER", id: "forget-ledger" }], + }), + ); + const where = ["--local", "--config", config, "--persist-to", join(directory, "state")]; + const wrangler = wranglerIn(resolve(import.meta.dirname, "..")); + const now = Date.now(); + const seeded = join(directory, "seeded.json"); + const at = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(now - daysAgo * DAY).toISOString(), + reason: "requested", + }); + await writeFile( + seeded, + JSON.stringify([ + { key: `${ROOT}t/1:e-1`, value: "", metadata: at(1) }, + { key: `${ROOT}t/2:e-2`, value: "", metadata: at(40) }, + { key: "elsewhere", value: "kept" }, + ]), + ); + await wrangler(["kv", "bulk", "put", seeded, "--binding", "FORGET_LEDGER", ...where]); + const expired = await expireKvLedger({ + wrangler, + ledger, + binding: "FORGET_LEDGER", + keepForMs: 35 * DAY, + target: { local: true, config, persistTo: join(directory, "state") }, + now, + }); + expect(expired).toEqual({ expired: 1, removed: 1 }); + const listed = JSON.parse( + await wrangler(["kv", "key", "list", "--binding", "FORGET_LEDGER", ...where]), + ) as readonly { readonly name: string; readonly expiration?: number }[]; + expect(listed.map(({ name, expiration }) => [name, expiration])).toEqual([ + [`${ROOT}t/1:e-1`, Math.ceil((now + 34 * DAY) / 1000)], + ["elsewhere", undefined], + ]); + }, 120_000); +}); diff --git a/packages/cli/src/forget-ledger-expire.ts b/packages/cli/src/forget-ledger-expire.ts new file mode 100644 index 00000000..ff01c178 --- /dev/null +++ b/packages/cli/src/forget-ledger-expire.ts @@ -0,0 +1,157 @@ +import { execFile } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import type { KvLedgerKey } from "@bounda-dev/cloudflare/forget-ledger"; + +const run = promisify(execFile); + +/** + * Runs wrangler with the arguments given and resolves what it printed to stdout. + */ +export interface Wrangler { + (args: readonly string[]): Promise; +} + +export interface WranglerInFunction { + (root: string): Wrangler; +} + +const binOf = (packageJson: string): string => { + const { bin } = JSON.parse(readFileSync(packageJson, "utf8")) as { + readonly bin?: string | Readonly>; + }; + const path = typeof bin === "string" ? bin : bin?.wrangler; + if (path === undefined) throw new Error(`${packageJson} names no wrangler executable`); + return resolve(dirname(packageJson), path); +}; + +// The package as the project installed it, in the `node_modules` of the root or a directory above. +const installedIn = (root: string, name: string): string | undefined => { + for (let directory = resolve(root); ; ) { + const packageJson = join(directory, "node_modules", name, "package.json"); + if (existsSync(packageJson)) return packageJson; + const parent = dirname(directory); + if (parent === directory) return undefined; + directory = parent; + } +}; + +/** + * The project's own wrangler, run through Node rather than a shell, so its `.cmd` shim on Windows + * is not needed. Wrangler's stderr, where it reports progress, is left out. + */ +export const wranglerIn: WranglerInFunction = (root) => { + let bin: string | undefined; + return async (args) => { + if (bin === undefined) { + const packageJson = installedIn(root, "wrangler"); + if (packageJson === undefined) { + throw new Error(`wrangler is not installed in ${root}: add it to the project`); + } + bin = binOf(packageJson); + } + const { stdout } = await run(process.execPath, [bin, ...args], { + cwd: root, + maxBuffer: 1024 * 1024 * 1024, + }); + return stdout; + }; +}; + +/** + * Where wrangler finds the namespace: the deployed one, or the local state `wrangler dev` keeps. + */ +export interface WranglerTarget { + readonly local: boolean; + readonly env?: string | undefined; + readonly config?: string | undefined; + readonly persistTo?: string | undefined; +} + +/** + * What `@bounda-dev/cloudflare/forget-ledger` gives this command, which owns the keys' layout. + */ +export type KvLedgerModule = Pick< + typeof import("@bounda-dev/cloudflare/forget-ledger"), + "KV_FORGET_LEDGER_ROOT" | "planKvForgetLedgerExpiry" +>; + +export interface ExpireKvLedgerArgs { + readonly wrangler: Wrangler; + readonly ledger: KvLedgerModule; + readonly binding: string; + readonly keepForMs: number; + readonly target: WranglerTarget; + readonly now: number; +} + +export interface KvLedgerExpired { + readonly expired: number; + readonly removed: number; +} + +export interface ExpireKvLedgerFunction { + (args: ExpireKvLedgerArgs): Promise; +} + +const isListedKey = (value: unknown): value is KvLedgerKey => + typeof value === "object" && + value !== null && + typeof Reflect.get(value, "name") === "string" && + ["number", "undefined"].includes(typeof Reflect.get(value, "expiration")); + +/** + * Lists every object's entries in the namespace, and writes again, through wrangler's bulk + * commands, those `planKvForgetLedgerExpiry` gives an expiration, removing those already past + * `keepFor`. + */ +export const expireKvLedger: ExpireKvLedgerFunction = async ({ + wrangler, + ledger, + binding, + keepForMs, + target, + now, +}) => { + const where = [ + target.local ? "--local" : "--remote", + ...(target.env === undefined ? [] : ["--env", target.env]), + ...(target.config === undefined ? [] : ["--config", target.config]), + ...(target.persistTo === undefined ? [] : ["--persist-to", target.persistTo]), + ]; + const listed: unknown = JSON.parse( + await wrangler([ + "kv", + "key", + "list", + "--binding", + binding, + "--prefix", + ledger.KV_FORGET_LEDGER_ROOT, + ...where, + ]), + ); + if (!Array.isArray(listed) || !listed.every(isListedKey)) { + throw new Error("wrangler kv key list printed something else than a list of keys"); + } + const { expire, remove } = ledger.planKvForgetLedgerExpiry({ keys: listed, keepForMs, now }); + const directory = await mkdtemp(join(tmpdir(), "bounda-forget-ledger-")); + try { + if (expire.length > 0) { + const file = join(directory, "expire.json"); + await writeFile(file, JSON.stringify(expire)); + await wrangler(["kv", "bulk", "put", file, "--binding", binding, ...where]); + } + if (remove.length > 0) { + const file = join(directory, "remove.json"); + await writeFile(file, JSON.stringify(remove)); + await wrangler(["kv", "bulk", "delete", file, "--binding", binding, "--force", ...where]); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } + return { expired: expire.length, removed: remove.length }; +}; diff --git a/packages/cli/src/forgotten.test.ts b/packages/cli/src/forgotten.test.ts index c415ed6f..370323f8 100644 --- a/packages/cli/src/forgotten.test.ts +++ b/packages/cli/src/forgotten.test.ts @@ -9,13 +9,14 @@ const event = ( aggregateId: string, type: string, timestamp = "2026-11-01T00:00:00.000Z", + payload: unknown = {}, ): StoredEvent => ({ id: `e-${position}`, aggregateType, aggregateId, version: 1, type, - payload: {}, + payload, timestamp, metadata: { correlationId: "c", causationId: "c", depth: 0, schemaVersion: 1, system: false }, person: null, @@ -59,16 +60,41 @@ describe("readForgotten", () => { events.push( event(1_004, "bounda", "c-1", "ProcessingRestricted"), event(1_005, "order", "o-x", "PersonalDataForgotten"), - event(1_006, "bounda", "c-1", "PersonalDataForgotten", "2026-11-02T00:00:00.000Z"), + event(1_006, "bounda", "c-1", "PersonalDataForgotten", "2026-11-02T00:00:00.000Z", { + reason: "retention", + fingerprints: ["k-1"], + }), event(1_007, "bounda", "c-2", "PersonalDataForgotten", "1970-01-01T00:00:00.001Z"), + event(1_008, "bounda", "c-3", "PersonalDataForgotten", undefined, { + reason: "restored", + fingerprints: [1], + }), ); const { stream, reads } = streamOf([ event(0, "bounda", "c-0", "PersonalDataForgotten"), ...events, ]); - expect(await readForgotten({ events: stream, since: { kind: "position", value: 2 } })).toEqual([ - { personId: "c-1", eventId: "e-1006", forgottenAt: "2026-11-02T00:00:00.000Z" }, - { personId: "c-2", eventId: "e-1007", forgottenAt: "1970-01-01T00:00:00.001Z" }, + const lines = await readForgotten({ events: stream, since: { kind: "position", value: 2 } }); + expect(lines).toStrictEqual([ + { + personId: "c-1", + eventId: "e-1006", + forgottenAt: "2026-11-02T00:00:00.000Z", + reason: "retention", + fingerprints: ["k-1"], + }, + { + personId: "c-2", + eventId: "e-1007", + forgottenAt: "1970-01-01T00:00:00.001Z", + reason: "requested", + }, + { + personId: "c-3", + eventId: "e-1008", + forgottenAt: "2026-11-01T00:00:00.000Z", + reason: "requested", + }, ]); expect(reads).toEqual([ { after: 2, limit: 1_000 }, @@ -97,9 +123,43 @@ describe("parseForgetList", () => { '{"personId":"c-1","eventId":"e-1","forgottenAt":"x"}\r\n\n c-2 \n{"personId":"c-3"}\n', ), ).toStrictEqual([ - { personId: "c-1", eventId: "e-1" }, + { personId: "c-1", eventId: "e-1", forgottenAt: "x" }, { personId: "c-2" }, { personId: "c-3" }, ]); }); + + it("reads back what bounda forgotten prints: the reason and the keys a forgetting names", () => { + expect( + parseForgetList( + '{"personId":"c-1","eventId":"e-1","forgottenAt":"t","reason":"retention","fingerprints":["k-1"]}\n{"personId":"c-2","reason":"requested","fingerprints":[]}', + ), + ).toStrictEqual([ + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "t", + reason: "retention", + fingerprints: ["k-1"], + }, + { personId: "c-2", reason: "requested", fingerprints: [] }, + ]); + }); + + it("refuses a line whose fields are not what bounda forgotten prints, naming it", () => { + for (const line of [ + '{"personId":3}', + '{"personId":"c-1","eventId":3}', + '{"personId":"c-1","forgottenAt":3}', + '{"personId":"c-1","reason":"restored"}', + '{"personId":"c-1","reason":"toString"}', + '{"personId":"c-1","fingerprints":"k-1"}', + '{"personId":"c-1","fingerprints":["k-1",2]}', + ]) { + expect(() => parseForgetList(`c-0\n${line}`)).toThrow( + 'line 2 needs a "personId" string; "eventId" and "forgottenAt" only as strings, "reason" only as "requested" or "retention", and "fingerprints" only as a list of strings', + ); + } + expect(() => parseForgetList("{not json")).toThrow("line 1 is neither a person's id nor JSON"); + }); }); diff --git a/packages/cli/src/forgotten.ts b/packages/cli/src/forgotten.ts index 8f218ff0..5d8cf29a 100644 --- a/packages/cli/src/forgotten.ts +++ b/packages/cli/src/forgotten.ts @@ -3,7 +3,9 @@ import { type ForgetAgainEntry, type GlobalStream, PERSONAL_DATA_FORGOTTEN_EVENT, + type PersonalDataForgottenReason, } from "@bounda-dev/core"; +import type { ForgetLedgerEntry } from "@bounda-dev/core/adapter"; import { InvalidArgumentError } from "commander"; const PAGE = 1_000; @@ -42,11 +44,21 @@ export const parseSince: ParseSinceFunction = (value) => { * One forgetting, as a line of what `bounda forgotten` prints and `bounda forget --from` reads: * the entry a forget ledger keeps. */ -export interface ForgottenLine { - readonly personId: string; - readonly eventId: string; - readonly forgottenAt: string; -} +export type ForgottenLine = ForgetLedgerEntry; + +const fieldOf = (payload: unknown, name: string): unknown => + typeof payload === "object" && payload !== null ? Reflect.get(payload, name) : undefined; + +const isTextList = (value: unknown): value is readonly string[] => + Array.isArray(value) && value.every((item) => typeof item === "string"); + +const REASONS: Readonly> = { + requested: true, + retention: true, +}; + +const isReason = (value: unknown): value is PersonalDataForgottenReason => + typeof value === "string" && Object.hasOwn(REASONS, value); export interface ReadForgottenArgs { readonly events: GlobalStream; @@ -69,10 +81,14 @@ export const readForgotten: ReadForgottenFunction = async ({ events, since }) => if (event.aggregateType !== BOUNDA_AGGREGATE) continue; if (event.type !== PERSONAL_DATA_FORGOTTEN_EVENT) continue; if (since.kind === "date" && Date.parse(event.timestamp) < since.value) continue; + const reason = fieldOf(event.payload, "reason"); + const fingerprints = fieldOf(event.payload, "fingerprints"); lines.push({ personId: event.aggregateId, eventId: event.id, forgottenAt: event.timestamp, + reason: isReason(reason) ? reason : "requested", + ...(isTextList(fingerprints) ? { fingerprints } : {}), }); } const last = page.at(-1); @@ -89,12 +105,29 @@ const entryOf = (line: string, number: number): ForgetAgainEntry => { } catch { throw new Error(`line ${number} is neither a person's id nor JSON`); } - const personId: unknown = Reflect.get(parsed as object, "personId"); - const eventId: unknown = Reflect.get(parsed as object, "eventId"); - if (typeof personId !== "string" || (eventId !== undefined && typeof eventId !== "string")) { - throw new Error(`line ${number} needs a "personId" string, and "eventId" only as a string`); + const personId = fieldOf(parsed, "personId"); + const eventId = fieldOf(parsed, "eventId"); + const forgottenAt = fieldOf(parsed, "forgottenAt"); + const reason = fieldOf(parsed, "reason"); + const fingerprints = fieldOf(parsed, "fingerprints"); + if ( + typeof personId !== "string" || + (eventId !== undefined && typeof eventId !== "string") || + (forgottenAt !== undefined && typeof forgottenAt !== "string") || + (reason !== undefined && !isReason(reason)) || + (fingerprints !== undefined && !isTextList(fingerprints)) + ) { + throw new Error( + `line ${number} needs a "personId" string; "eventId" and "forgottenAt" only as strings, "reason" only as "requested" or "retention", and "fingerprints" only as a list of strings`, + ); } - return eventId === undefined ? { personId } : { personId, eventId }; + return { + personId, + ...(eventId === undefined ? {} : { eventId }), + ...(forgottenAt === undefined ? {} : { forgottenAt }), + ...(reason === undefined ? {} : { reason }), + ...(fingerprints === undefined ? {} : { fingerprints }), + }; }; export interface ParseForgetListFunction { diff --git a/packages/cli/src/run-without-cloudflare.test.ts b/packages/cli/src/run-without-cloudflare.test.ts new file mode 100644 index 00000000..fa66bb90 --- /dev/null +++ b/packages/cli/src/run-without-cloudflare.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { EXIT_FAILURE, runCli } from "./run.ts"; + +// A project that never installed the package the KV ledger comes from. +vi.mock("@bounda-dev/cloudflare/forget-ledger", () => { + throw new Error("Cannot find package '@bounda-dev/cloudflare'"); +}); + +describe("bounda forget-ledger expire without @bounda-dev/cloudflare", () => { + it("says the package is missing, and runs no wrangler", async () => { + let stderr = ""; + let runs = 0; + const code = await runCli({ + argv: ["forget-ledger", "expire", "--binding", "L", "--keep-for", "35d"], + cwd: "/project", + stdout: { write: () => undefined }, + stderr: { + write: (text) => { + stderr += text; + }, + }, + wrangler: () => async () => { + runs += 1; + return "[]"; + }, + }); + expect(code).toBe(EXIT_FAILURE); + expect(stderr).toBe( + "error: @bounda-dev/cloudflare is not installed, and the KV forget ledger comes from it\n", + ); + expect(runs).toBe(0); + }); +}); diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 5ba996b7..5eb312d7 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -665,7 +665,13 @@ describe("bounda forgotten and bounda forget", () => { expect(byPosition.code).toBe(EXIT_OK); expect(besidesLogs(byPosition.stderr)).toBe("1 person forgotten\n"); expect(byPosition.stdout).toBe( - `${JSON.stringify({ personId: forgotten, eventId: event?.id, forgottenAt: event?.timestamp })}\n`, + `${JSON.stringify({ + personId: forgotten, + eventId: event?.id, + forgottenAt: event?.timestamp, + reason: "requested", + fingerprints: Reflect.get(Object(event?.payload), "fingerprints"), + })}\n`, ); const byDate = await cli(["forgotten", "--since", before], fixture); expect(byDate.code).toBe(EXIT_OK); @@ -680,7 +686,7 @@ describe("bounda forgotten and bounda forget", () => { expect(besidesLogs(forget.stderr)).toBe(""); expect(forget.code).toBe(EXIT_OK); expect(forget.stdout).toBe( - `forgot again ${kept}\nforgot again 1 of 2 listed; the rest hold no key, or this store holds their forgetting\n`, + `forgot again ${kept}\nforgot again 1 of 2 listed; the rest hold no key the line names, or this store holds their forgetting\n`, ); const after = await open(); const reasons = (await after.events.read({ after: lastPosition, limit: 100 })) @@ -689,7 +695,7 @@ describe("bounda forgotten and bounda forget", () => { await after.stop(); expect(reasons).toEqual([ [forgotten, "requested"], - [kept, "restored"], + [kept, "requested"], ]); }); @@ -703,7 +709,7 @@ describe("bounda forgotten and bounda forget", () => { const forget = await cli(["forget", "--from", file, "--actor", "operator-1"], fixture); expect(forget.code).toBe(EXIT_OK); expect(logged(forget.stderr, "info", "personal data forgotten")).toEqual([ - expect.objectContaining({ personId: memberId, reason: "restored", actor: "operator-1" }), + expect.objectContaining({ personId: memberId, reason: "requested", actor: "operator-1" }), ]); const after = await open(); const forgetting = (await after.events.read({ limit: 1_000 })).find( @@ -713,6 +719,35 @@ describe("bounda forgotten and bounda forget", () => { expect(forgetting?.metadata.actor).toBe("operator-1"); }); + it("lists what the database holds without starting the app, so a strict ledger that is down does not stop it", async () => { + const strict = ["--config", "strict-ledger.config.ts"]; + const listed = await cli(["forgotten", "--since", "0", ...strict], fixture); + expect(listed).toMatchObject({ code: EXIT_OK, stdout: "" }); + expect(besidesLogs(listed.stderr)).toBe("0 persons forgotten\n"); + const booted = await cli(["dead-letters", "list", ...strict], fixture); + expect(booted.code).toBe(EXIT_FAILURE); + expect(booted.stderr).toContain( + "error: The forget ledger could not be read, and runtime.personalData.forgetLedger.strict keeps the app from starting until it is", + ); + }); + + it("says a storage without its factories cannot be listed", async () => { + const root = await list(); + await writeFile( + join(root, "bounda.config.ts"), + `export default { storage: { kind: "bounda-adapter", name: "elsewhere", options: {} } };\n`, + ); + await writeFile( + join(root, "registry.ts"), + "export const registry = { aggregates: {}, readModels: {} };\n", + ); + const refused = await cli(["forgotten", "--since", "0", "--registry", "registry.ts"], root); + expect(refused.code).toBe(EXIT_FAILURE); + expect(refused.stderr).toContain( + `error: storage "elsewhere" is a definition without factories. Import the adapter package's factory.`, + ); + }); + it("refuses a --since that is neither a position nor a date", async () => { for (const since of ["yesterday", "-3", "", "2026-13-45"]) { const refused = await cli(["forgotten", "--since", since], fixture); @@ -731,7 +766,7 @@ describe("bounda forgotten and bounda forget", () => { expect(missing.stderr).toMatch(/^error: nowhere\.jsonl: ENOENT/); for (const [content, message] of [ ["m-1\n{not json", "line 2 is neither a person's id nor JSON"], - ['{"eventId":"e-1"}', 'line 1 needs a "personId" string, and "eventId" only as a string'], + ['{"eventId":"e-1"}', 'line 1 needs a "personId" string; "eventId" and "forgottenAt"'], ['{"personId":"m-1","eventId":7}', 'line 1 needs a "personId" string'], ] as const) { await writeFile(join(root, "list.jsonl"), content); @@ -761,3 +796,131 @@ describe("bounda forgotten and bounda forget", () => { ); }); }); + +describe("bounda forget-ledger expire", () => { + const ROOT = "bounda/forget-ledger/"; + const expire = async (argv: readonly string[], listed: unknown = []) => { + const stdout = capture(); + const stderr = capture(); + const calls: { readonly root: string; readonly args: readonly string[] }[] = []; + const code = await runCli({ + argv: ["forget-ledger", "expire", ...argv], + cwd: "/project", + stdout, + stderr, + wrangler: (root) => async (args) => { + calls.push({ root, args }); + if (args[1] === "bulk" && args[2] === "put") throw new Error("wrangler: not authorized"); + return JSON.stringify(listed); + }, + }); + return { code, stdout: stdout.text(), stderr: stderr.text(), calls }; + }; + + it("gives the entries their expiration through the project's wrangler, and says how many", async () => { + const day = 86_400_000; + const old = { personId: "c-1", forgottenAt: new Date(Date.now() - 40 * day).toISOString() }; + const done = await expire( + [ + "--binding", + "FORGET_LEDGER", + "--keep-for", + " 35d ", + "--local", + "--env", + "staging", + "--wrangler-config", + "w.jsonc", + "--persist-to", + ".state", + "--root", + "app", + ], + [{ name: `${ROOT}t/1:e-1`, metadata: old }], + ); + expect(done).toMatchObject({ + code: EXIT_OK, + stdout: "gave 0 entries the expiration keepFor sets; removed 1 already past it\n", + stderr: "", + }); + expect(done.calls.map(({ root }) => root)).toEqual([ + resolve("/project", "app"), + resolve("/project", "app"), + ]); + expect(done.calls[0]?.args).toEqual([ + "kv", + "key", + "list", + "--binding", + "FORGET_LEDGER", + "--prefix", + ROOT, + "--local", + "--env", + "staging", + "--config", + "w.jsonc", + "--persist-to", + ".state", + ]); + const one = await expire( + ["--binding", "L", "--keep-for", "35d"], + [{ name: `${ROOT}t/1:e-1`, metadata: { ...old, forgottenAt: new Date().toISOString() } }], + ); + expect(one.code).toBe(EXIT_FAILURE); + expect(one.stderr).toBe("error: wrangler: not authorized\n"); + expect(one.calls.map(({ root, args }) => [root, args.at(-1)])).toEqual([ + ["/project", "--remote"], + ["/project", "--remote"], + ]); + }); + + it("says how many it gave an expiration, one or more", async () => { + const stdout = capture(); + const listed = [1, 2].map((index) => ({ + name: `${ROOT}t/${index}:e-${index}`, + metadata: { personId: "c-1", forgottenAt: new Date().toISOString() }, + })); + for (const keys of [listed.slice(0, 1), listed]) { + await runCli({ + argv: ["forget-ledger", "expire", "--binding", "L", "--keep-for", "35d"], + cwd: "/project", + stdout, + stderr: capture(), + wrangler: () => async (args) => (args[1] === "key" ? JSON.stringify(keys) : ""), + }); + } + expect(stdout.text()).toBe( + "gave 1 entry the expiration keepFor sets; removed 0 already past it\ngave 2 entries the expiration keepFor sets; removed 0 already past it\n", + ); + }); + + it("refuses a keep-for that is not a duration longer than 0, and a missing binding", async () => { + for (const keepFor of ["0s", "soon", "-1d", ""]) { + const refused = await expire(["--binding", "L", "--keep-for", keepFor]); + expect(refused.code).toBe(EXIT_CONVENTION); + expect(refused.stderr).toContain("expected a duration longer than 0, such as 35d"); + expect(refused.calls).toEqual([]); + } + expect((await expire(["--keep-for", "35d"])).code).toBe(EXIT_CONVENTION); + }); + + it("documents itself in its help", async () => { + const help = (await expire(["--help"])).stdout.replace(/\s+/g, " "); + for (const text of [ + "give every entry of a Workers KV forget ledger the expiration keepFor sets, those written before it was set included", + "--binding ", + "the KV namespace's binding in wrangler.jsonc", + "--keep-for ", + "runtime.personalData.forgetLedger.keepFor, such as 35d", + "--local", + "the local state of wrangler dev instead of the deployed namespace", + "--env ", + "--wrangler-config ", + "--persist-to ", + "--root ", + ]) { + expect(help).toContain(text); + } + }); +}); diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index aee44f57..f296a8d4 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -9,12 +9,20 @@ import { type ListDeadLettersArgs, type Logger, MAX_ACTOR_LENGTH, + parseDuration, rebuildReadModel, } from "@bounda-dev/core"; +import type { Adapter, AdapterDefinition, Storage } from "@bounda-dev/core/adapter"; import type { Config, ErrorText } from "@bounda-dev/core/config"; import { boot, createConsoleLogger, loadProject } from "@bounda-dev/core/node"; import { Command, CommanderError, InvalidArgumentError, Option } from "commander"; import { formatLetter, formatRetried } from "./dead-letter-output.ts"; +import { + expireKvLedger, + type KvLedgerModule, + type Wrangler, + wranglerIn, +} from "./forget-ledger-expire.ts"; import { parseForgetList, parseSince, readForgotten, type Since } from "./forgotten.ts"; import { generateProject } from "./generate/generate.ts"; import { ConventionError } from "./generate/problems.ts"; @@ -34,6 +42,8 @@ export interface RunCliArgs { readonly stdout: Output; readonly stderr: Output; readonly signal?: AbortSignal; + // How `forget-ledger expire` runs wrangler in a project's root; the project's own by default. + readonly wrangler?: (root: string) => Wrangler; } export interface RunCliFunction { @@ -107,7 +117,8 @@ const withErrorText = (config: Config, options: ProjectOptions): Config => } : config; -const rootOf = (options: ProjectOptions, cwd: string): string => resolve(cwd, options.root ?? "."); +const rootOf = (options: Pick, cwd: string): string => + resolve(cwd, options.root ?? "."); // What fails while the project's modules load tells of files and exports, never of data a handler // read, so it is printed in full. @@ -308,18 +319,47 @@ interface ForgottenOptions extends ProjectOptions { readonly since: Since; } -const runForgotten = ( +const hasFactories = (definition: AdapterDefinition): definition is Adapter => + typeof Reflect.get(definition, "createStorage") === "function"; + +// Reads the event store alone, without starting the app: no ports, read models or forget ledger, +// so a strict ledger that is down does not keep it from listing what the database holds. +const runForgotten = async ( options: ForgottenOptions, cwd: string, stdout: Output, stderr: Output, -): Promise => - withApp(options, cwd, stderr, async (app) => { - const lines = await readForgotten({ events: app.events, since: options.since }); +): Promise => { + const project = await load(options, cwd, logTo(stderr), stderr); + if (project === undefined) return EXIT_FAILURE; + const definition: AdapterDefinition = project.config.storage; + if (!hasFactories(definition)) { + line( + stderr, + `error: storage "${definition.name}" is a definition without factories. Import the adapter package's factory.`, + ); + return EXIT_FAILURE; + } + let storage: Storage | undefined; + try { + storage = await definition.createStorage(); + const { eventStore } = storage; + const lines = await readForgotten({ + events: { + read: ({ after = 0, limit = 1_000 } = {}) => + eventStore.readAll({ afterPosition: after, limit }), + }, + since: options.since, + }); for (const forgotten of lines) line(stdout, JSON.stringify(forgotten)); line(stderr, `${lines.length} person${lines.length === 1 ? "" : "s"} forgotten`); return EXIT_OK; - }); + } catch (error) { + return failed(stderr, error, errorTextOf(options, project.config)); + } finally { + await storage?.close(); + } +}; interface ForgetOptions extends ActingOptions { readonly from: string; @@ -350,12 +390,73 @@ const runForget = async ( for (const personId of forgotten) line(stdout, `forgot again ${personId}`); line( stdout, - `forgot again ${forgotten.length} of ${entries.length} listed; the rest hold no key, or this store holds their forgetting`, + `forgot again ${forgotten.length} of ${entries.length} listed; the rest hold no key the line names, or this store holds their forgetting`, ); return EXIT_OK; }); }; +interface ExpireOptions { + readonly root?: string; + readonly binding: string; + readonly keepFor: number; + readonly local: boolean; + readonly env?: string; + readonly wranglerConfig?: string; + readonly persistTo?: string; +} + +const parseKeepFor = (value: string): number => { + try { + const keepForMs = parseDuration(value.trim()); + if (keepForMs > 0) return keepForMs; + } catch {} + throw new InvalidArgumentError("expected a duration longer than 0, such as 35d"); +}; + +const runExpire = async ( + options: ExpireOptions, + cwd: string, + stdout: Output, + stderr: Output, + wrangler: (root: string) => Wrangler, +): Promise => { + let ledger: KvLedgerModule; + try { + ledger = await import("@bounda-dev/cloudflare/forget-ledger"); + } catch { + line( + stderr, + "error: @bounda-dev/cloudflare is not installed, and the KV forget ledger comes from it", + ); + return EXIT_FAILURE; + } + try { + const { expired, removed } = await expireKvLedger({ + wrangler: wrangler(rootOf(options, cwd)), + ledger, + binding: options.binding, + keepForMs: options.keepFor, + target: { + local: options.local, + env: options.env, + config: options.wranglerConfig, + persistTo: options.persistTo, + }, + now: Date.now(), + }); + line( + stdout, + `gave ${expired} entr${expired === 1 ? "y" : "ies"} the expiration keepFor sets; removed ${removed} already past it`, + ); + return EXIT_OK; + } catch (error) { + // Wrangler's own failure and the namespace's keys: ids and times, never a person's data. + line(stderr, `error: ${error instanceof Error ? error.message : String(error)}`); + return EXIT_FAILURE; + } +}; + // Records, so a kind or status added to core fails to compile here until the CLI accepts it. const DEAD_LETTER_KINDS = Object.keys({ policy: true, @@ -386,7 +487,14 @@ const actingOptions = (command: T): T => parseActor, ); -export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal }) => { +export const runCli: RunCliFunction = async ({ + argv, + cwd, + stdout, + stderr, + signal, + wrangler = wranglerIn, +}) => { let exitCode = EXIT_OK; const program = new Command("bounda") .description("Bounda: event sourcing and CQRS for TypeScript") @@ -514,6 +622,28 @@ export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal exitCode = await runForget(options, cwd, stdout, stderr); }); + program + .command("forget-ledger") + .description("look after a forget ledger outside the app") + .command("expire") + .description( + "give every entry of a Workers KV forget ledger the expiration keepFor sets, those written before it was set included", + ) + .requiredOption("--binding ", "the KV namespace's binding in wrangler.jsonc") + .requiredOption( + "--keep-for ", + "runtime.personalData.forgetLedger.keepFor, such as 35d", + parseKeepFor, + ) + .option("--local", "the local state of wrangler dev instead of the deployed namespace", false) + .option("--env ", "the wrangler environment") + .option("--wrangler-config ", "the wrangler configuration file") + .option("--persist-to ", "with --local, where wrangler dev keeps its state") + .option("--root ", "the project's root directory, where wrangler is installed") + .action(async (options: ExpireOptions) => { + exitCode = await runExpire(options, cwd, stdout, stderr, wrangler); + }); + try { await program.parseAsync([...argv], { from: "user" }); return exitCode; diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 31d37575..684560e4 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -19,6 +19,10 @@ "types": "./dist/index.d.ts", "default": "./dist/index.js" }, + "./forget-ledger": { + "types": "./dist/forget-ledger.d.ts", + "default": "./dist/forget-ledger.js" + }, "./package.json": "./package.json" }, "publishConfig": { diff --git a/packages/cloudflare/src/forget-ledger.ts b/packages/cloudflare/src/forget-ledger.ts index 98bbacfd..7f719a2b 100644 --- a/packages/cloudflare/src/forget-ledger.ts +++ b/packages/cloudflare/src/forget-ledger.ts @@ -21,10 +21,37 @@ const KV_MIN_TTL_S = 60; interface EntryMetadata { readonly personId: string; readonly forgottenAt: string; + // Absent from a key written before entries kept them. + readonly reason?: string; + readonly fingerprints?: readonly string[]; } const timeKey = (forgottenAt: string): string => String(LATEST_MS - Date.parse(forgottenAt)); +/** + * Where every object's entries live, each under its own name. + */ +export const KV_FORGET_LEDGER_ROOT = "bounda/forget-ledger/"; + +// When an entry kept `keepForMs` expires, in seconds as KV takes it; `null` once it is past. +const expirationOf = (forgottenAt: string, keepForMs: number, now: number): number | null => { + const expiresAt = Date.parse(forgottenAt) + keepForMs; + if (!(expiresAt > now)) return null; + return Math.max(Math.ceil(expiresAt / 1000), Math.floor(now / 1000) + KV_MIN_TTL_S); +}; + +const entryOf = (metadata: EntryMetadata, eventId: string): ForgetLedgerEntry => { + const entry: ForgetLedgerEntry = { + personId: metadata.personId, + eventId, + forgottenAt: metadata.forgottenAt, + reason: metadata.reason === "retention" ? "retention" : "requested", + }; + return Array.isArray(metadata.fingerprints) + ? { ...entry, fingerprints: [...metadata.fingerprints] } + : entry; +}; + export interface CreateKvForgetLedgerArgs { readonly namespace: KVNamespace; /** @@ -58,17 +85,17 @@ export const createKvForgetLedger: CreateKvForgetLedgerFunction = ({ append: async (entries) => { const now = Date.now(); await Promise.all( - entries.flatMap(({ personId, eventId, forgottenAt }) => { - const metadata: EntryMetadata = { personId, forgottenAt }; + entries.flatMap(({ personId, eventId, forgottenAt, reason, fingerprints }) => { + const metadata: EntryMetadata = { + personId, + forgottenAt, + reason, + ...(fingerprints === undefined ? {} : { fingerprints }), + }; const key = `${prefix}${timeKey(forgottenAt)}:${eventId}`; if (keepForMs === undefined) return [namespace.put(key, "", { metadata })]; - const expiresAt = Date.parse(forgottenAt) + keepForMs; - if (!(expiresAt > now)) return []; - const expiration = Math.max( - Math.ceil(expiresAt / 1000), - Math.floor(now / 1000) + KV_MIN_TTL_S, - ); - return [namespace.put(key, "", { metadata, expiration })]; + const expiration = expirationOf(forgottenAt, keepForMs, now); + return expiration === null ? [] : [namespace.put(key, "", { metadata, expiration })]; }), ); }, @@ -91,11 +118,7 @@ export const createKvForgetLedger: CreateKvForgetLedgerFunction = ({ break; } if (metadata === null || metadata === undefined) continue; - found.push({ - personId: metadata.personId, - eventId: rest.slice(separator + 1), - forgottenAt: metadata.forgottenAt, - }); + found.push(entryOf(metadata, rest.slice(separator + 1))); } if (past || page.list_complete) break; cursor = page.cursor; @@ -149,7 +172,119 @@ export const kvForgetLedger: KvForgetLedgerFunction = ({ binding }) => } return createKvForgetLedger({ namespace, - prefix: `bounda/forget-ledger/${encodeURIComponent(tenant)}/`, + prefix: `${KV_FORGET_LEDGER_ROOT}${encodeURIComponent(tenant)}/`, keepForMs, }); }); + +/** + * A key of the ledger's namespace as KV lists it. + */ +export interface KvLedgerKey { + readonly name: string; + /** + * When KV expires it, in seconds since 1970; absent for a key kept for good. + */ + readonly expiration?: number | undefined; + readonly metadata?: unknown; +} + +/** + * A key written again, as it was, to expire at `expiration`. + */ +export interface KvLedgerPut { + readonly key: string; + readonly value: ""; + readonly expiration: number; + readonly metadata: unknown; +} + +export interface KvForgetLedgerExpiry { + readonly expire: readonly KvLedgerPut[]; + /** + * The keys already older than `keepFor`, which KV would not take an expiration for. + */ + readonly remove: readonly string[]; +} + +export interface PlanKvForgetLedgerExpiryArgs { + readonly keys: readonly KvLedgerKey[]; + readonly keepForMs: number; + readonly now: number; +} + +export interface PlanKvForgetLedgerExpiryFunction { + (args: PlanKvForgetLedgerExpiryArgs): KvForgetLedgerExpiry; +} + +/** + * What gives every entry the expiration `keepFor` sets, for keys written before `keepFor` was set, + * or while it was longer: a key that expires no later is left alone, and a key that is not an + * entry of the ledger is never touched. + */ +export const planKvForgetLedgerExpiry: PlanKvForgetLedgerExpiryFunction = ({ + keys, + keepForMs, + now, +}) => { + const expire: KvLedgerPut[] = []; + const remove: string[] = []; + for (const { name, expiration: current, metadata } of keys) { + const forgottenAt: unknown = + typeof metadata === "object" && metadata !== null + ? Reflect.get(metadata, "forgottenAt") + : undefined; + if (typeof forgottenAt !== "string" || Number.isNaN(Date.parse(forgottenAt))) continue; + const expiration = expirationOf(forgottenAt, keepForMs, now); + if (expiration === null) remove.push(name); + else if (current === undefined || current > expiration) { + expire.push({ key: name, value: "", expiration, metadata }); + } + } + return { expire, remove }; +}; + +export interface ExpireKvForgetLedgerArgs { + readonly namespace: KVNamespace; + readonly keepForMs: number; + /** + * Defaults to now. + */ + readonly now?: number; +} + +export interface ExpireKvForgetLedgerFunction { + (args: ExpireKvForgetLedgerArgs): Promise; +} + +/** + * Gives every object's entries in the namespace the expiration `keepFor` sets, as + * `planKvForgetLedgerExpiry` decides, from a Worker, as `bounda forget-ledger expire` does from a + * terminal. Resolves to what it changed. Run it once after setting or shortening `keepFor`: the + * entries written from then on expire on their own. + */ +export const expireKvForgetLedger: ExpireKvForgetLedgerFunction = async ({ + namespace, + keepForMs, + now = Date.now(), +}) => { + const keys: KvLedgerKey[] = []; + for (let cursor: string | undefined; ; ) { + const page = await namespace.list({ + prefix: KV_FORGET_LEDGER_ROOT, + limit: LIST_LIMIT, + ...(cursor === undefined ? {} : { cursor }), + }); + keys.push(...page.keys); + if (page.list_complete) break; + cursor = page.cursor; + } + const expiry = planKvForgetLedgerExpiry({ keys, keepForMs, now }); + await Promise.all([ + ...expiry.expire.map(({ key, value, expiration, metadata }) => + namespace.put(key, value, { expiration, metadata }), + ), + ...expiry.remove.map((key) => namespace.delete(key)), + ]); + return expiry; +}; diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 4fabf849..fd2643b2 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -16,8 +16,24 @@ export type { IsCloudflareDefinitionFunction, } from "./definition.ts"; export { cloudflare, isCloudflareDefinition } from "./definition.ts"; -export type { KvForgetLedgerFunction, KvForgetLedgerOptions } from "./forget-ledger.ts"; -export { KV_LIST_DELAY_MS, kvForgetLedger } from "./forget-ledger.ts"; +export type { + ExpireKvForgetLedgerArgs, + ExpireKvForgetLedgerFunction, + KvForgetLedgerExpiry, + KvForgetLedgerFunction, + KvForgetLedgerOptions, + KvLedgerKey, + KvLedgerPut, + PlanKvForgetLedgerExpiryArgs, + PlanKvForgetLedgerExpiryFunction, +} from "./forget-ledger.ts"; +export { + expireKvForgetLedger, + KV_FORGET_LEDGER_ROOT, + KV_LIST_DELAY_MS, + kvForgetLedger, + planKvForgetLedgerExpiry, +} from "./forget-ledger.ts"; export type { RpcOutcome, RpcRefusal } from "./outcome.ts"; export type { DurableSqlStorage } from "./sql-database.ts"; export type { diff --git a/packages/cloudflare/test/forget-ledger.test.ts b/packages/cloudflare/test/forget-ledger.test.ts index 2163e4a7..2ad35292 100644 --- a/packages/cloudflare/test/forget-ledger.test.ts +++ b/packages/cloudflare/test/forget-ledger.test.ts @@ -1,12 +1,27 @@ import { runInDurableObject } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import { ConfigurationError, createApp, type PersonalDataForgottenPayload } from "@bounda-dev/core"; +import { + ConfigurationError, + createApp, + FORGET_LEDGER_GRACE_MS, + ForgetLedgerUnavailableError, + type PersonalDataForgottenPayload, +} from "@bounda-dev/core"; +import type { Adapter } from "@bounda-dev/core/adapter"; import { forgetLedgerContract } from "@bounda-dev/core/adapter/testing"; import { describe, expect, it } from "vitest"; +import { durableObjectAdapter } from "../src/adapter.ts"; import { configForObject } from "../src/bounda-object.ts"; import { connect } from "../src/client.ts"; import { cloudflare } from "../src/definition.ts"; -import { createKvForgetLedger, KV_LIST_DELAY_MS, kvForgetLedger } from "../src/forget-ledger.ts"; +import { + createKvForgetLedger, + expireKvForgetLedger, + KV_FORGET_LEDGER_ROOT, + KV_LIST_DELAY_MS, + kvForgetLedger, + planKvForgetLedgerExpiry, +} from "../src/forget-ledger.ts"; import { quietRegistry } from "./app.ts"; import { clock } from "./clock.ts"; @@ -30,7 +45,13 @@ describe("the forget ledger in Workers KV", () => { kvForgetLedger({ binding: "FORGET_LEDGER" }).create({ env, tenant }); const ada = await ledgerOf("named-ada"); const bob = await ledgerOf("named-bob"); - const entry = { personId: "c-1", eventId: "e-1", forgottenAt: "2026-11-07T12:00:00.000Z" }; + const entry = { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-11-07T12:00:00.000Z", + reason: "requested" as const, + fingerprints: ["k-1"], + }; await ada.append([entry]); expect(await ada.read({ limit: 10 })).toEqual([entry]); expect(await bob.read({ limit: 10 })).toEqual([]); @@ -97,6 +118,7 @@ describe("the KV forget ledger's listing", () => { personId: `c-${index}`, eventId: `e-${String(index).padStart(4, "0")}`, forgottenAt: minute(index), + reason: "requested" as const, })); await ledger.append(entries); expect(await ledger.read({ limit: 2_000 })).toEqual(entries); @@ -110,6 +132,7 @@ describe("the KV forget ledger's listing", () => { personId: `c-${index}`, eventId: `e-${String(index).padStart(4, "0")}`, forgottenAt: minute(index), + reason: "requested" as const, })); await ledger.append(entries); const after = entries[1_002] as (typeof entries)[number]; @@ -128,9 +151,14 @@ describe("the KV forget ledger's listing", () => { const now = Date.now(); const at = (milliseconds: number): string => new Date(now + milliseconds).toISOString(); await ledger.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs - 1_000) }, - { personId: "c-2", eventId: "e-2", forgottenAt: at(-keepForMs + 30_000) }, - { personId: "c-3", eventId: "e-3", forgottenAt: at(0) }, + { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs - 1_000), reason: "requested" }, + { + personId: "c-2", + eventId: "e-2", + forgottenAt: at(-keepForMs + 30_000), + reason: "requested", + }, + { personId: "c-3", eventId: "e-3", forgottenAt: at(0), reason: "requested" }, ]); expect( [...keys.values()].map((metadata) => (metadata as { personId: string }).personId), @@ -142,7 +170,7 @@ describe("the KV forget ledger's listing", () => { expect(later).toBe(Math.ceil((Date.parse(at(0)) + keepForMs) / 1000)); const forever = countingNamespace(); await createKvForgetLedger({ namespace: forever.namespace, prefix: "p/" }).append([ - { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs * 1_000) }, + { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs * 1_000), reason: "requested" }, ]); expect([...forever.expirations.values()]).toEqual([undefined]); }); @@ -154,6 +182,19 @@ describe("the KV forget ledger's listing", () => { expect(await ledger.read({ limit: 10 })).toEqual([]); }); + it("reads a key written before entries kept their reason and keys as a request naming none", async () => { + const { namespace, keys } = countingNamespace(); + const ledger = createKvForgetLedger({ namespace, prefix: "p/" }); + keys.set("p/1:e-1", { personId: "c-1", forgottenAt: minute(0) }); + keys.set("p/2:e-2", { personId: "c-2", forgottenAt: minute(0), reason: "retention" }); + const read = await ledger.read({ limit: 10 }); + expect(read).toEqual([ + { personId: "c-1", eventId: "e-1", forgottenAt: minute(0), reason: "requested" }, + { personId: "c-2", eventId: "e-2", forgottenAt: minute(0), reason: "retention" }, + ]); + expect(read[0]).not.toHaveProperty("fingerprints"); + }); + it("refuses an object without a name, whose entries it could not keep apart", async () => { const { namespace } = countingNamespace(); expect(() => @@ -173,11 +214,24 @@ describe("the KV forget ledger's listing", () => { env: { ...env, LEDGER: namespace } as typeof env, tenant: "acme/eu", }); - await ledger.append([{ personId: "c-1", eventId: "e-1", forgottenAt: minute(0) }]); + await ledger.append([ + { + personId: "c-1", + eventId: "e-1", + forgottenAt: minute(0), + reason: "retention", + fingerprints: ["k-1"], + }, + { personId: "c-2", eventId: "e-2", forgottenAt: minute(0), reason: "requested" }, + ]); expect([...keys.keys()]).toEqual([ - `bounda/forget-ledger/acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-1`, + `${KV_FORGET_LEDGER_ROOT}acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-1`, + `${KV_FORGET_LEDGER_ROOT}acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-2`, + ]); + expect([...keys.values()]).toEqual([ + { personId: "c-1", forgottenAt: minute(0), reason: "retention", fingerprints: ["k-1"] }, + { personId: "c-2", forgottenAt: minute(0), reason: "requested" }, ]); - expect([...keys.values()]).toEqual([{ personId: "c-1", forgottenAt: minute(0) }]); }); }); @@ -219,6 +273,7 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { await store.forget("ada"); await runInDurableObject(stub, (instance) => instance.alarm()); await runInDurableObject(stub, (_instance, state) => copyTables(state.storage, true)); + clock.advance(FORGET_LEDGER_GRACE_MS); // What the object's constructor runs when it starts again. const outcome = await runInDurableObject(stub, async (_instance, state) => { @@ -246,7 +301,7 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { } }); expect(outcome).toMatchObject({ - forgotten: ["restored"], + forgotten: ["requested"], ada: { note: null }, bob: { note: "door" }, lag: { reconciled: true }, @@ -254,3 +309,195 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { expect(outcome.nextDueAt).toBe(outcome.now + KV_LIST_DELAY_MS); }); }); + +describe("expiring a KV forget ledger", () => { + const DAY = 86_400_000; + const NOW = Date.UTC(2026, 10, 9); + const at = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(NOW - daysAgo * DAY).toISOString(), + }); + + it("plans an expiration for every entry kept for good or longer than keepFor, and the removal of those past it", () => { + // A second short of keepFor: KV expires a key a minute from now at the soonest. + const soonest = { ...at(0), forgottenAt: new Date(NOW - 35 * DAY + 1_000).toISOString() }; + expect( + planKvForgetLedgerExpiry({ + keys: [ + { name: "kept", metadata: at(1) }, + { name: "past", metadata: at(36) }, + { name: "just-past", metadata: at(35) }, + { name: "longer", metadata: at(2), expiration: NOW / 1000 + 100 * 86_400 }, + { name: "sooner", metadata: at(2), expiration: NOW / 1000 + 86_400 }, + { name: "due", metadata: at(2), expiration: (NOW + 33 * DAY) / 1000 }, + { name: "soonest", metadata: soonest }, + { name: "stray" }, + { name: "unreadable", metadata: { forgottenAt: "not a time" } }, + { name: "untimed", metadata: { forgottenAt: 3 } }, + { name: "text", metadata: "kept" }, + ], + keepForMs: 35 * DAY, + now: NOW, + }), + ).toEqual({ + expire: [ + { key: "kept", value: "", expiration: (NOW + 34 * DAY) / 1000, metadata: at(1) }, + { key: "longer", value: "", expiration: (NOW + 33 * DAY) / 1000, metadata: at(2) }, + { + key: "soonest", + value: "", + expiration: NOW / 1000 + 60, + metadata: soonest, + }, + ], + remove: ["past", "just-past"], + }); + }); + + it("writes the plan to the namespace, every object's entries and nothing else", async () => { + const namespace = env.FORGET_LEDGER; + const now = Date.now(); + const written = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(now - daysAgo * DAY).toISOString(), + reason: "requested", + }); + await namespace.put(`${KV_FORGET_LEDGER_ROOT}expire-a/1:e-1`, "", { metadata: written(1) }); + await namespace.put(`${KV_FORGET_LEDGER_ROOT}expire-b/2:e-2`, "", { metadata: written(40) }); + await namespace.put("expire-elsewhere", "kept", { metadata: written(1) }); + const expiry = await expireKvForgetLedger({ namespace, keepForMs: 35 * DAY, now }); + expect(expiry.expire.map(({ key }) => key)).toContain(`${KV_FORGET_LEDGER_ROOT}expire-a/1:e-1`); + expect(expiry.remove).toContain(`${KV_FORGET_LEDGER_ROOT}expire-b/2:e-2`); + const listed = await namespace.list({ prefix: KV_FORGET_LEDGER_ROOT }); + expect(listed.keys.find(({ name }) => name.endsWith("expire-a/1:e-1"))).toMatchObject({ + expiration: Math.ceil((now + 34 * DAY) / 1000), + metadata: written(1), + }); + expect(listed.keys.some(({ name }) => name.endsWith("expire-b/2:e-2"))).toBe(false); + expect((await namespace.list({ prefix: "expire-elsewhere" })).keys).toEqual([ + { name: "expire-elsewhere", metadata: written(1) }, + ]); + expect(await expireKvForgetLedger({ namespace, keepForMs: 35 * DAY })).toMatchObject({ + remove: [], + }); + }); + + it("lists every page of a namespace with more keys than a page", async () => { + const { namespace, keys, pages } = countingNamespace(); + for (let index = 0; index < 1_001; index += 1) { + keys.set(`${KV_FORGET_LEDGER_ROOT}t/${index}:e-${index}`, at(0)); + } + const expiry = await expireKvForgetLedger({ namespace, keepForMs: DAY, now: NOW }); + expect(expiry.expire).toHaveLength(1_001); + expect(pages()).toBe(2); + }); +}); + +describe("a Durable Object's forgetting and the KV ledger", () => { + it("records the intention in KV before it commits, and the next start commits one whose commit never happened", async () => { + const stub = env.LEDGER_STORE.get(env.LEDGER_STORE.idFromName("intended-tenant")); + const store = connect(stub); + await store.commands.placeOrder({ orderId: "o-1", total: 7, customer: "ada", note: "bell" }); + const intention = await runInDurableObject(stub, async (_instance, state) => { + const opened = durableObjectAdapter({ storage: state.storage, options: {} }); + const evicted: Adapter = { + ...opened, + createStorage: async () => { + const storage = await opened.createStorage(); + return { + ...storage, + transact: () => Promise.reject(new Error("the object was evicted")), + }; + }, + }; + const app = await createApp({ + registry: quietRegistry, + config: { ...configForObject(config, state.storage), storage: evicted }, + env, + tenant: "intended-tenant", + clock, + }); + try { + await expect(app.forget("ada")).rejects.toThrow("the object was evicted"); + } finally { + await app.stop(); + } + const ledger = await kvForgetLedger({ binding: "FORGET_LEDGER" }).create({ + env, + tenant: "intended-tenant", + }); + const [entry] = await ledger.read({ limit: 10 }); + return entry; + }); + expect(intention).toMatchObject({ + personId: "ada", + reason: "requested", + fingerprints: [expect.any(String)], + }); + clock.advance(FORGET_LEDGER_GRACE_MS); + const outcome = await runInDurableObject(stub, async (_instance, state) => { + const app = await createApp({ + registry: quietRegistry, + config: configForObject(config, state.storage), + env, + tenant: "intended-tenant", + clock, + }); + try { + return { + forgotten: (await app.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => event.id), + ada: (await app.exportPersonalData("ada")).events[0]?.data, + }; + } finally { + await app.stop(); + } + }); + expect(outcome).toEqual({ + forgotten: [intention?.eventId], + ada: expect.objectContaining({ note: null }), + }); + }); + + it("forgets nobody, when strict, while KV does not take the intention", async () => { + const stub = env.LEDGER_STORE.get(env.LEDGER_STORE.idFromName("strict-tenant")); + const store = connect(stub); + await store.commands.placeOrder({ orderId: "o-1", total: 7, customer: "ada", note: "bell" }); + const refusal = await runInDurableObject(stub, async (_instance, state) => { + const down = { + list: (options?: KVNamespaceListOptions) => env.FORGET_LEDGER.list(options), + put: () => Promise.reject(new Error("KV is down")), + }; + const app = await createApp({ + registry: quietRegistry, + config: { + ...configForObject(config, state.storage), + runtime: { + personalData: { + forgetLedger: { store: kvForgetLedger({ binding: "DOWN" }), strict: true }, + }, + }, + }, + env: { ...env, DOWN: down } as typeof env, + tenant: "strict-tenant", + clock, + }); + try { + const error = await app.forget("ada").catch((thrown: unknown) => thrown); + return { + unavailable: error instanceof ForgetLedgerUnavailableError, + code: (error as { code?: string }).code, + ada: (await app.exportPersonalData("ada")).events[0]?.data, + }; + } finally { + await app.stop(); + } + }); + expect(refusal).toEqual({ + unavailable: true, + code: "FORGET_LEDGER_UNAVAILABLE", + ada: expect.objectContaining({ note: "bell" }), + }); + }); +}); diff --git a/packages/cloudflare/tsdown.config.ts b/packages/cloudflare/tsdown.config.ts index f560e8cb..f99e6f61 100644 --- a/packages/cloudflare/tsdown.config.ts +++ b/packages/cloudflare/tsdown.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsdown"; export default defineConfig({ - entry: ["src/index.ts"], + entry: ["src/index.ts", "src/forget-ledger.ts"], tsconfig: "tsconfig.build.json", unbundle: true, dts: true, diff --git a/packages/core/src/adapter/sql/forget-ledger.test.ts b/packages/core/src/adapter/sql/forget-ledger.test.ts index 75f77dd4..d24c3e45 100644 --- a/packages/core/src/adapter/sql/forget-ledger.test.ts +++ b/packages/core/src/adapter/sql/forget-ledger.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; +import type { ForgetLedgerEntry } from "../storage/forget-ledger.ts"; import { postgresqlDialect, sqliteDialect } from "./dialect.ts"; -import { createSqlForgetLedger, forgetLedgerStatements } from "./forget-ledger.ts"; +import { + createSqlForgetLedger, + forgetLedgerReasonStatements, + forgetLedgerStatements, +} from "./forget-ledger.ts"; const recording = () => { const statements: (readonly [string, readonly unknown[]])[] = []; @@ -16,10 +21,12 @@ const recording = () => { return { db, statements }; }; -const entry = (index: number) => ({ +const entry = (index: number): ForgetLedgerEntry => ({ personId: `c-${index}`, eventId: `e-${index}`, forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + fingerprints: [`k-${index}`], }); describe("the SQL forget ledger", () => { @@ -34,7 +41,14 @@ describe("the SQL forget ledger", () => { ); }); - it("inserts at most 25 entries a statement, each bound to its own numbered parameters", async () => { + it("adds the reason, a request for the entries before it, and the keys, none recorded", () => { + expect(forgetLedgerReasonStatements(`"t"`)).toEqual([ + `ALTER TABLE "t" ADD COLUMN "reason" TEXT NOT NULL DEFAULT 'requested'`, + `ALTER TABLE "t" ADD COLUMN "fingerprints" TEXT`, + ]); + }); + + it("inserts at most 16 entries a statement, each bound to its own numbered parameters", async () => { const { db, statements } = recording(); const ledger = createSqlForgetLedger({ db, @@ -44,15 +58,35 @@ describe("the SQL forget ledger", () => { ensureSchema: async () => {}, close: async () => {}, }); - await ledger.append(Array.from({ length: 26 }, (_, index) => entry(index))); + const { fingerprints: _, ...unrecorded } = entry(16); + await ledger.append([ + ...Array.from({ length: 16 }, (_, index) => entry(index)), + { ...unrecorded, reason: "retention" }, + ]); expect(statements).toHaveLength(2); const [first, second] = statements; - expect(first?.[0]).toContain("($1, $2, $3, $4), ($5, $6, $7, $8)"); - expect(first?.[0]).toContain("($97, $98, $99, $100) ON CONFLICT"); - expect(first?.[1]).toHaveLength(100); - expect(first?.[1].slice(0, 4)).toEqual(["", "e-0", "c-0", "2026-01-01T00:00:00.000Z"]); - expect(second?.[0]).toContain(`VALUES ($1, $2, $3, $4) ON CONFLICT`); - expect(second?.[1]).toEqual(["", "e-25", "c-25", "2026-01-01T00:00:00.000Z"]); + expect(first?.[0]).toContain( + `INSERT INTO "t" ("tenant", "event_id", "person", "forgotten_at", "reason", "fingerprints") VALUES ($1, $2, $3, $4, $5, $6), ($7, $8, $9, $10, $11, $12)`, + ); + expect(first?.[0]).toContain("($91, $92, $93, $94, $95, $96) ON CONFLICT"); + expect(first?.[1]).toHaveLength(96); + expect(first?.[1].slice(0, 6)).toEqual([ + "", + "e-0", + "c-0", + "2026-01-01T00:00:00.000Z", + "requested", + `["k-0"]`, + ]); + expect(second?.[0]).toContain(`VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT`); + expect(second?.[1]).toEqual([ + "", + "e-16", + "c-16", + "2026-01-01T00:00:00.000Z", + "retention", + null, + ]); }); it("creates its table before the first statement, and again after that failed", async () => { diff --git a/packages/core/src/adapter/sql/forget-ledger.ts b/packages/core/src/adapter/sql/forget-ledger.ts index f8346d91..a3a31bc3 100644 --- a/packages/core/src/adapter/sql/forget-ledger.ts +++ b/packages/core/src/adapter/sql/forget-ledger.ts @@ -3,16 +3,17 @@ import type { SqlDialect } from "./dialect.ts"; import { quoteIdentifier } from "./identifiers.ts"; import type { SqlExecutor } from "./sql-table.ts"; -// Four parameters a row, within the hundred a Durable Object's SQLite binds to one statement. -const ENTRIES_PER_INSERT = 25; +const COLUMNS = ["tenant", "event_id", "person", "forgotten_at", "reason", "fingerprints"] as const; +// Within the hundred parameters a Durable Object's SQLite binds to one statement. +const ENTRIES_PER_INSERT = Math.floor(100 / COLUMNS.length); export interface ForgetLedgerStatementsFunction { (table: string, dialect: Pick): readonly string[]; } /** - * The ledger's table, `table` already quoted. PostgreSQL compares the ordering columns by code - * unit, as SQLite does, rather than by the database's collation. + * The ledger's table as its first version created it, `table` already quoted. PostgreSQL compares + * the ordering columns by code unit, as SQLite does, rather than by the database's collation. */ export const forgetLedgerStatements: ForgetLedgerStatementsFunction = (table, { name }) => { const ordered = name === "postgresql" ? ` COLLATE "C"` : ""; @@ -28,6 +29,19 @@ export const forgetLedgerStatements: ForgetLedgerStatementsFunction = (table, { ]; }; +export interface ForgetLedgerReasonStatementsFunction { + (table: string): readonly string[]; +} + +/** + * The second version: each entry keeps its reason and the keys it names. An entry written before + * reads as a request that names none, which any key matches. + */ +export const forgetLedgerReasonStatements: ForgetLedgerReasonStatementsFunction = (table) => [ + `ALTER TABLE ${table} ADD COLUMN "reason" TEXT NOT NULL DEFAULT 'requested'`, + `ALTER TABLE ${table} ADD COLUMN "fingerprints" TEXT`, +]; + export interface CreateSqlForgetLedgerArgs { readonly db: SqlExecutor; /** @@ -47,11 +61,19 @@ export interface CreateSqlForgetLedgerFunction { (args: CreateSqlForgetLedgerArgs): ForgetLedger; } -const entryOf = (row: Record): ForgetLedgerEntry => ({ - personId: String(row.person), - eventId: String(row.event_id), - forgottenAt: String(row.forgotten_at), -}); +const entryOf = (row: Record): ForgetLedgerEntry => { + const entry: ForgetLedgerEntry = { + personId: String(row.person), + eventId: String(row.event_id), + forgottenAt: String(row.forgotten_at), + reason: row.reason === "retention" ? "retention" : "requested", + }; + const fingerprints: unknown = + typeof row.fingerprints === "string" ? JSON.parse(row.fingerprints) : null; + return Array.isArray(fingerprints) ? { ...entry, fingerprints: fingerprints.map(String) } : entry; +}; + +const SELECTED = `"event_id", "person", "forgotten_at", "reason", "fingerprints"`; /** * The forget ledger of SQLite and PostgreSQL alike, in a table of its own that holds every @@ -80,16 +102,18 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ for (let start = 0; start < entries.length; start += ENTRIES_PER_INSERT) { const chunk = entries.slice(start, start + ENTRIES_PER_INSERT); const rows = chunk.map( - (_, index) => - `(${p(index * 4 + 1)}, ${p(index * 4 + 2)}, ${p(index * 4 + 3)}, ${p(index * 4 + 4)})`, + (_, row) => + `(${COLUMNS.map((_, column) => p(row * COLUMNS.length + column + 1)).join(", ")})`, ); await db.run( - `INSERT INTO ${table} ("tenant", "event_id", "person", "forgotten_at") VALUES ${rows.join(", ")} ON CONFLICT ("tenant", "event_id") DO NOTHING`, - chunk.flatMap(({ eventId, personId, forgottenAt }) => [ + `INSERT INTO ${table} (${COLUMNS.map((column) => `"${column}"`).join(", ")}) VALUES ${rows.join(", ")} ON CONFLICT ("tenant", "event_id") DO NOTHING`, + chunk.flatMap(({ eventId, personId, forgottenAt, reason, fingerprints }) => [ tenant, eventId, personId, forgottenAt, + reason, + fingerprints === undefined ? null : JSON.stringify(fingerprints), ]), ); } @@ -99,11 +123,11 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ const rows = after === undefined ? await db.all( - `SELECT "event_id", "person", "forgotten_at" FROM ${table} WHERE "tenant" = ${p(1)} ORDER BY "forgotten_at", "event_id" LIMIT ${p(2)}`, + `SELECT ${SELECTED} FROM ${table} WHERE "tenant" = ${p(1)} ORDER BY "forgotten_at", "event_id" LIMIT ${p(2)}`, [tenant, limit], ) : await db.all( - `SELECT "event_id", "person", "forgotten_at" FROM ${table} WHERE "tenant" = ${p(1)} AND ("forgotten_at", "event_id") > (${p(2)}, ${p(3)}) ORDER BY "forgotten_at", "event_id" LIMIT ${p(4)}`, + `SELECT ${SELECTED} FROM ${table} WHERE "tenant" = ${p(1)} AND ("forgotten_at", "event_id") > (${p(2)}, ${p(3)}) ORDER BY "forgotten_at", "event_id" LIMIT ${p(4)}`, [tenant, after.forgottenAt, after.eventId, limit], ); return rows.map(entryOf); diff --git a/packages/core/src/adapter/sql/index.ts b/packages/core/src/adapter/sql/index.ts index 222d51c0..abb87cf3 100644 --- a/packages/core/src/adapter/sql/index.ts +++ b/packages/core/src/adapter/sql/index.ts @@ -20,7 +20,11 @@ export type { CreateSqlForgetLedgerFunction, ForgetLedgerStatementsFunction, } from "./forget-ledger.ts"; -export { createSqlForgetLedger, forgetLedgerStatements } from "./forget-ledger.ts"; +export { + createSqlForgetLedger, + forgetLedgerReasonStatements, + forgetLedgerStatements, +} from "./forget-ledger.ts"; export { DEFAULT_TABLE_PREFIX, quoteIdentifier, diff --git a/packages/core/src/adapter/sqlite/migrations.ts b/packages/core/src/adapter/sqlite/migrations.ts index 6e748c4f..90112510 100644 --- a/packages/core/src/adapter/sqlite/migrations.ts +++ b/packages/core/src/adapter/sqlite/migrations.ts @@ -2,7 +2,7 @@ import { ConfigurationError } from "../../contracts/errors.ts"; import type { SqlDatabase, SqlTransaction } from "../sql/database.ts"; import { resolvedAtOfEarlierLetters } from "../sql/dead-letters.ts"; import { sqliteDialect } from "../sql/dialect.ts"; -import { forgetLedgerStatements } from "../sql/forget-ledger.ts"; +import { forgetLedgerReasonStatements, forgetLedgerStatements } from "../sql/forget-ledger.ts"; import { quoteIdentifier } from "../sql/identifiers.ts"; import { migrateSchemas, @@ -145,6 +145,12 @@ export const sqliteForgetLedgerScope: SqliteForgetLedgerScopeFunction = (table) } }, }, + { + version: 2, + migrate: async (tx) => { + for (const statement of forgetLedgerReasonStatements(table)) await tx.run(statement, []); + }, + }, ], }); diff --git a/packages/core/src/adapter/storage/forget-ledger.ts b/packages/core/src/adapter/storage/forget-ledger.ts index a2d1f39c..90ee783a 100644 --- a/packages/core/src/adapter/storage/forget-ledger.ts +++ b/packages/core/src/adapter/storage/forget-ledger.ts @@ -1,7 +1,10 @@ +import type { PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; + /** - * One forgetting as the ledger keeps it: whom, which `PersonalDataForgotten` said so, and when. - * Ids and a time, nothing else: the ledger is a suppression list, the one thing about a forgotten - * person that has to outlive their data. + * One forgetting as the ledger keeps it: whom, which `PersonalDataForgotten` says so, when, why, + * and which keys it destroys. Ids, a time and fingerprints, nothing else: the ledger is a + * suppression list, the one thing about a forgotten person that has to outlive their data. The + * runtime writes it before the forgetting commits, so an entry may name an event no store holds. */ export interface ForgetLedgerEntry { readonly personId: string; @@ -14,6 +17,14 @@ export interface ForgetLedgerEntry { * The event's timestamp, as `Date.prototype.toISOString` writes it. */ readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + /** + * The fingerprints of the keys the person held when the forgetting was asked: the reconciliation + * commits it again only while the person holds one of them, since any other key was given to + * them after. Absent from an entry written by a Bounda that did not record them, which any key + * matches. + */ + readonly fingerprints?: readonly string[]; } /** @@ -44,14 +55,14 @@ export interface PruneForgetLedgerArgs { /** * The record of every person forgotten, kept outside the database, so restoring a copy of the * database taken before someone was forgotten can forget them again. The runtime appends each - * forgetting to it as soon as it commits, and again from the stream, and reads it when the app - * starts and every `reconcileEvery` while it runs. One ledger holds one store: a ledger built for - * a tenant sees only that tenant's entries. + * forgetting to it before the forgetting commits, and again from the stream once it has, and + * reads it when the app starts and every `reconcileEvery` while it runs. One ledger holds one + * store: a ledger built for a tenant sees only that tenant's entries. */ export interface ForgetLedger { /** - * Adds the entries, one per `eventId`: an event names one person at one time, so appending the - * same entries again leaves the ledger as it was. + * Adds the entries, one per `eventId`, each with every field it has: an event names one person + * at one time, so appending the same entries again leaves the ledger as it was. */ append(entries: readonly ForgetLedgerEntry[]): Promise; /** diff --git a/packages/core/src/adapter/testing/forget-ledger.contract.ts b/packages/core/src/adapter/testing/forget-ledger.contract.ts index 95b47796..89ea702c 100644 --- a/packages/core/src/adapter/testing/forget-ledger.contract.ts +++ b/packages/core/src/adapter/testing/forget-ledger.contract.ts @@ -16,6 +16,8 @@ const entry = (personId: string, eventId: string, minute: number): ForgetLedgerE personId, eventId, forgottenAt: new Date(Date.UTC(2026, 10, 7, 12, minute)).toISOString(), + reason: "requested", + fingerprints: [`k-${personId}`], }); const readAll = async (ledger: ForgetLedger, limit = 2): Promise => { @@ -89,6 +91,20 @@ export const forgetLedgerContract: ForgetLedgerContractFunction = ({ create }) = ]); }); + it("keeps each entry's reason and the keys it names, none, or none recorded", async () => { + const retained: ForgetLedgerEntry = { + ...entry("c-1", "e-1", 1), + reason: "retention", + fingerprints: ["k-1", "k-2"], + }; + const keyless: ForgetLedgerEntry = { ...entry("c-2", "e-2", 2), fingerprints: [] }; + const { fingerprints: _, ...unrecorded } = entry("c-3", "e-3", 3); + await ledger.append([retained, keyless, unrecorded]); + const read = await ledger.read({ limit: 10 }); + expect(read).toEqual([retained, keyless, unrecorded]); + expect(read[2]).not.toHaveProperty("fingerprints"); + }); + it("keeps one person's forgettings apart, each under its event", async () => { await ledger.append([entry("c-1", "e-1", 1), entry("c-1", "e-2", 2)]); expect(await ledger.read({ limit: 10 })).toEqual([ diff --git a/packages/core/src/adapter/testing/storage-schema.contract.ts b/packages/core/src/adapter/testing/storage-schema.contract.ts index 3b36b98f..18345b62 100644 --- a/packages/core/src/adapter/testing/storage-schema.contract.ts +++ b/packages/core/src/adapter/testing/storage-schema.contract.ts @@ -255,13 +255,19 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ const ledger = await target.adapter.createForgetLedger({ tenant: "t-1" }); try { await ledger.append([ - { personId: "p-1", eventId: "e-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "p-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ]); } finally { await ledger.close?.(); } expect((await versionsOf(target)).map(({ scope, version }) => ({ scope, version }))).toEqual([ { scope: "forget-ledger", version: 1 }, + { scope: "forget-ledger", version: 2 }, ]); }); diff --git a/packages/core/src/config/defaults.ts b/packages/core/src/config/defaults.ts index 0ea29566..1b654e11 100644 --- a/packages/core/src/config/defaults.ts +++ b/packages/core/src/config/defaults.ts @@ -42,6 +42,10 @@ export const DEFAULT_CATCH_UP_POLL_MS: number = 15; * The `reconcileEvery` of `runtime.personalData.forgetLedger` when it is left out. */ export const DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS: number = 5 * 60_000; +/** + * The `timeout` of `runtime.personalData.forgetLedger` when it is left out. + */ +export const DEFAULT_FORGET_LEDGER_TIMEOUT_MS: number = 10_000; /** * The share of the loads from a snapshot checked against the whole stream when `snapshot.verify` * is left out. diff --git a/packages/core/src/config/index.ts b/packages/core/src/config/index.ts index 3ee7ed4a..6cd53eb6 100644 --- a/packages/core/src/config/index.ts +++ b/packages/core/src/config/index.ts @@ -17,6 +17,7 @@ export type { MasterKeyFunction, MasterKeyOptions } from "../kernel/personal-dat export { masterKey } from "../kernel/personal-data/master-key.ts"; export { DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, DEFAULT_SNAPSHOT_VERIFY, DEFAULT_SNAPSHOT_VERSION, } from "./defaults.ts"; diff --git a/packages/core/src/config/schema.test.ts b/packages/core/src/config/schema.test.ts index 4888d160..0acbabcf 100644 --- a/packages/core/src/config/schema.test.ts +++ b/packages/core/src/config/schema.test.ts @@ -4,7 +4,10 @@ import { ConfigurationError } from "../contracts/errors.ts"; import { defineForgetLedger } from "../kernel/personal-data/forget-ledger.ts"; import { masterKey } from "../kernel/personal-data/master-key.ts"; import { memory } from "../memory/index.ts"; -import { DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS } from "./defaults.ts"; +import { + DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, +} from "./defaults.ts"; import { resolveConfig } from "./schema.ts"; const sqlite: AdapterDefinition<"sqlite", { path: string }> = { @@ -86,7 +89,13 @@ describe("resolveConfig", () => { personalData: { retention: "30d", keyWrapper, - forgetLedger: { store: forgetLedger, reconcileEvery: "1m", strict: true, keepFor: "90d" }, + forgetLedger: { + store: forgetLedger, + reconcileEvery: "1m", + strict: true, + keepFor: "90d", + timeout: "3s", + }, }, deadLetters: { retention: "7d" }, }, @@ -123,6 +132,7 @@ describe("resolveConfig", () => { reconcileEveryMs: 60_000, strict: true, keepForMs: 7_776_000_000, + timeoutMs: 3_000, }, }); expect(resolved.runtime.deadLetters).toEqual({ retentionMs: 604_800_000 }); @@ -136,6 +146,7 @@ describe("resolveConfig", () => { reconcileEveryMs: DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, strict: false, keepForMs: null, + timeoutMs: DEFAULT_FORGET_LEDGER_TIMEOUT_MS, }); expect(defaulted?.store).toBe(ledgerDatabase); expect(resolved.readModels["users-directory"]).toBe(sqlite); @@ -279,6 +290,7 @@ describe("resolveConfig", () => { for (const [option, value] of [ ["reconcileEvery", 0], ["keepFor", "0s"], + ["timeout", 0], ] as const) { expect( message({ diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.ts index 2e30afa7..1219b61d 100644 --- a/packages/core/src/config/schema.ts +++ b/packages/core/src/config/schema.ts @@ -19,6 +19,7 @@ import { DEFAULT_COMMANDS, DEFAULT_CONCURRENCY_RETRIES, DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, DEFAULT_IDEMPOTENCY_KEY_LIFETIME_MS, DEFAULT_IDLE_INTERVAL_MS, DEFAULT_POLICIES, @@ -102,6 +103,7 @@ const forgetLedger = z reconcileEvery: timeout.optional(), strict: z.boolean().optional(), keepFor: timeout.optional(), + timeout: timeout.optional(), }) .refine( ({ reconcileEvery = DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, keepFor }) => @@ -226,6 +228,7 @@ const resolveForgetLedger = ( reconcileEveryMs: parsed.reconcileEvery ?? DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, strict: parsed.strict ?? false, keepForMs: parsed.keepFor ?? null, + timeoutMs: parsed.timeout ?? DEFAULT_FORGET_LEDGER_TIMEOUT_MS, }; const formatIssues = (issues: readonly z.core.$ZodIssue[]): string => diff --git a/packages/core/src/config/types.ts b/packages/core/src/config/types.ts index be183c87..45184718 100644 --- a/packages/core/src/config/types.ts +++ b/packages/core/src/config/types.ts @@ -155,12 +155,11 @@ export interface PersonalDataConfig { } /** - * The `runtime.personalData.forgetLedger` of the configuration. The runtime appends each - * forgetting to the ledger as soon as it commits, and copies it again from the stream in the - * background, in case the process stops in between. When the app starts, and every - * `reconcileEvery` while it runs, it reads the ledger and forgets again, with `reason: - * "restored"`, whoever it names whose forgetting the database does not hold and who still holds - * a key. + * The `runtime.personalData.forgetLedger` of the configuration. Every forgetting writes its + * intention to the ledger before it commits, and the stream copies each committed one again in + * the background. When the app starts, and every `reconcileEvery` while it runs, it reads the + * ledger and commits again each forgetting the database does not hold whose person still holds a + * key it names: one a restored backup rolled back, or one whose commit never happened. */ export interface ForgetLedgerConfig { /** @@ -179,10 +178,14 @@ export interface ForgetLedgerConfig { */ readonly reconcileEvery?: DurationInput; /** - * When `true`, the app does not start until it has read the ledger through: a ledger that cannot - * be read fails the start with `ForgetLedgerUnreconciledError`, and the start waits for the - * reconciliation however long it takes. When `false`, the default, the app starts anyway, waits - * for the ledger at most a few seconds, and reconciles in the background. + * When `true`, nothing is forgotten that the ledger has not recorded, and the app does not + * start until it has read the ledger through: a forgetting whose intention the ledger does not + * take fails with `ForgetLedgerUnavailableError` and forgets nobody, so calling again is safe; + * a ledger that cannot be read fails the start with `ForgetLedgerUnreconciledError`, and the + * start waits for the reconciliation however long it takes. When `false`, the default, a + * forgetting the ledger does not take is logged and commits all the same, for the stream to copy + * later, and the app starts after waiting for the ledger at most `timeout`, reconciling in the + * background. */ readonly strict?: boolean; /** @@ -191,6 +194,12 @@ export interface ForgetLedgerConfig { * default: entries are kept for good. */ readonly keepFor?: DurationInput; + /** + * How long each call to the ledger may take before it is given up with + * `ForgetLedgerTimeoutError`, and how long a start that is not strict waits for the + * reconciliation. Defaults to `DEFAULT_FORGET_LEDGER_TIMEOUT_MS` (ten seconds). + */ + readonly timeout?: DurationInput; } export interface ResolvedForgetLedgerConfig { @@ -201,6 +210,7 @@ export interface ResolvedForgetLedgerConfig { * `null` when entries are kept for good. */ readonly keepForMs: number | null; + readonly timeoutMs: number; } /** diff --git a/packages/core/src/contracts/duration.ts b/packages/core/src/contracts/duration.ts index e0644c55..b7315bee 100644 --- a/packages/core/src/contracts/duration.ts +++ b/packages/core/src/contracts/duration.ts @@ -33,6 +33,10 @@ export interface ParseDurationFunction { (input: LooseDurationInput): number; } +/** + * The milliseconds of a duration written as the configuration takes it, `"35d"` or a number of + * milliseconds. Throws `ValidationError` for anything else, or a negative number. + */ export const parseDuration: ParseDurationFunction = (input) => { if (typeof input === "number") { if (!Number.isFinite(input) || input < 0) { diff --git a/packages/core/src/contracts/errors.test.ts b/packages/core/src/contracts/errors.test.ts index 2f31b2dc..da6942ab 100644 --- a/packages/core/src/contracts/errors.test.ts +++ b/packages/core/src/contracts/errors.test.ts @@ -7,6 +7,8 @@ import { DeadLetterNotRetriableError, DeadLetterSettledError, DomainError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, NotFoundError, RebuildSupersededError, @@ -69,6 +71,24 @@ describe("errors", () => { expect(error.message).toContain("runtime.personalData.forgetLedger.strict"); }); + it("say a strict forget ledger refused the forgetting, with why", () => { + const cause = new Error("down"); + const error = new ForgetLedgerUnavailableError(cause); + expect(error.code).toBe("FORGET_LEDGER_UNAVAILABLE"); + expect(error.name).toBe("ForgetLedgerUnavailableError"); + expect(error.cause).toBe(cause); + expect(error.message).toContain("nothing was forgotten"); + }); + + it("name the forget ledger call that ran out of time, and the time", () => { + const error = new ForgetLedgerTimeoutError("append", 2500); + expect(error.code).toBe("FORGET_LEDGER_TIMEOUT"); + expect(error.name).toBe("ForgetLedgerTimeoutError"); + expect(error.operation).toBe("append"); + expect(error.timeoutMs).toBe(2500); + expect(error.message).toBe("The forget ledger did not answer append within 2500 ms"); + }); + it("name the read model whose rebuild was taken over", () => { const error = new RebuildSupersededError("orderSummary"); expect(error.code).toBe("REBUILD_SUPERSEDED"); diff --git a/packages/core/src/contracts/errors.ts b/packages/core/src/contracts/errors.ts index 66e99c51..718fd8cd 100644 --- a/packages/core/src/contracts/errors.ts +++ b/packages/core/src/contracts/errors.ts @@ -273,6 +273,41 @@ export class ForgetLedgerUnreconciledError extends BoundaError { } } +/** + * Thrown by `forget`, and by every other forgetting, when `runtime.personalData.forgetLedger.strict` + * is set and the forget ledger did not take the forgetting's intention: nothing was forgotten, and + * calling again tries again. `cause` is why the ledger failed, a `ForgetLedgerTimeoutError` when + * it did not answer in time. + */ +export class ForgetLedgerUnavailableError extends BoundaError { + constructor(cause: unknown) { + super( + "FORGET_LEDGER_UNAVAILABLE", + "The forget ledger did not take the forgetting, and runtime.personalData.forgetLedger.strict forgets nobody it has not recorded: nothing was forgotten, try again", + { cause }, + ); + } +} + +/** + * A call to the forget ledger did not settle within `runtime.personalData.forgetLedger.timeout`: + * the `cause` of `ForgetLedgerUnavailableError` and `ForgetLedgerUnreconciledError`, and what the + * app logs when the ledger is not strict. The call may still land after it. + */ +export class ForgetLedgerTimeoutError extends BoundaError { + readonly operation: string; + readonly timeoutMs: number; + + constructor(operation: string, timeoutMs: number) { + super( + "FORGET_LEDGER_TIMEOUT", + `The forget ledger did not answer ${operation} within ${timeoutMs} ms`, + ); + this.operation = operation; + this.timeoutMs = timeoutMs; + } +} + export interface StorageSchemaTooNewErrorArgs { readonly scope: string; readonly version: number; diff --git a/packages/core/src/contracts/index.ts b/packages/core/src/contracts/index.ts index f85a08c8..b6028c3c 100644 --- a/packages/core/src/contracts/index.ts +++ b/packages/core/src/contracts/index.ts @@ -19,8 +19,9 @@ export type { DurationString, DurationUnit, LooseDurationInput, + ParseDurationFunction, } from "./duration.ts"; -export { asDuration } from "./duration.ts"; +export { asDuration, parseDuration } from "./duration.ts"; export { BoundaError, ChainDepthExceededError, @@ -33,6 +34,8 @@ export { DeadLetterSettledError, type DeadLetterSettledErrorArgs, DomainError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, IdempotencyKeyConflictError, type IdempotencyKeyConflictErrorArgs, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5e0a6bc8..7e2c7de2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { PersonalDataExport, } from "./kernel/personal-data/export.ts"; export type { ForgetAgainEntry } from "./kernel/personal-data/reconciliation.ts"; +export { FORGET_LEDGER_GRACE_MS } from "./kernel/personal-data/reconciliation.ts"; export type { ProcessStatus } from "./kernel/process/lifecycle.ts"; export type { RebuildReadModelArgs, diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 007bec50..7ab9b2af 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -30,21 +30,26 @@ import { createPersonalDataBackfill } from "./personal-data/backfill.ts"; import { exportPersonalData, type PersonalDataExport } from "./personal-data/export.ts"; import { forgetPersonalData, logForgotten } from "./personal-data/forget.ts"; import { - FORGET_LEDGER_WAIT_MS, + boundForgetLedger, openForgetLedger, waitForLedger, } from "./personal-data/forget-ledger.ts"; -import { withLedgerAppend } from "./personal-data/ledger-append.ts"; +import { createForgetIntentions } from "./personal-data/intention.ts"; import { createLedgerOutbox } from "./personal-data/ledger-outbox.ts"; import { marksPersonalFields } from "./personal-data/marks.ts"; import { createForgetLedgerReconciliation, type ForgetAgainEntry, forgetAgain, + NEVER_DUE, } from "./personal-data/reconciliation.ts"; import { marksFingerprint } from "./personal-data/resealing.ts"; import { liftProcessingRestriction, restrictProcessing } from "./personal-data/restriction.ts"; -import { createRetentionSubscriber, createRetentionSweep } from "./personal-data/retention.ts"; +import { + createRetentionSubscriber, + createRetentionSweep, + retentionDueOf, +} from "./personal-data/retention.ts"; import { createKeyRotation } from "./personal-data/rotation.ts"; import { checkPersonId } from "./personal-data/transact.ts"; import { type PersonalDataWitness, witnessedWrapping } from "./personal-data/witness.ts"; @@ -113,7 +118,10 @@ export interface BoundaClient { * the stream `bounda:`, which every read model that projects personal fields handles. * Both happen together or not at all. Resolves like a command, so * `catchUpReadModels({ through })` waits for those read models. Rejects with - * `ValidationError` an empty id. + * `ValidationError` an empty id. With `runtime.personalData.forgetLedger`, the forgetting is in + * the ledger before it commits: one that rejects after that still forgets the person at the next + * reconciliation, and calling again is safe; a strict ledger that does not take it rejects with + * `ForgetLedgerUnavailableError`, forgetting nobody. */ forget(personId: string): Promise; /** @@ -231,13 +239,13 @@ export interface BoundaApp extends BoundaClien */ readonly deadLetters: DeadLetters; /** - * Forgets again, as `forget` does with `reason: "restored"`, each person an entry names who - * still holds a key, unless the store holds the `PersonalDataForgotten` the entry names by - * `eventId`: a store that holds it applied that forgetting, and a key the person holds was given - * to them after. What `bounda forget --from` runs, for a backup restored without - * `runtime.personalData.forgetLedger`; resolves to the ids of the persons it forgot, in order. - * Rejects with `ValidationError` an entry without a person's id, or an `actor` that `as` would - * refuse, before forgetting anyone. + * Commits again each forgetting an entry names, as a reconciliation does: unless the store holds + * the `PersonalDataForgotten` the entry names by `eventId`, and only while the person holds a key + * it names, any key when it names none; a whole entry under its own id, time and reason, a bare + * person's id as a request. What `bounda forget --from` runs, for a backup restored without + * `runtime.personalData.forgetLedger`; with one, each is recorded in it first, as `forget` does. + * Resolves to the ids of the persons it forgot, in order. Rejects with `ValidationError` an entry + * without a person's id, or an `actor` that `as` would refuse, before forgetting anyone. */ forgetAgain( entries: readonly ForgetAgainEntry[], @@ -464,10 +472,18 @@ export const assembleApp: AssembleAppFunction = async ({ const close = ledger.close.bind(ledger); opened.unshift(close); } - const written = - ledger === null - ? adapted - : withLedgerAppend({ storage: adapted, ledger, keepForMs, clock, logger }); + const bounded = + ledger === null || ledgerConfig === null + ? null + : boundForgetLedger({ ledger, timeoutMs: ledgerConfig.timeoutMs, clock }); + const intentions = createForgetIntentions({ + ledger: bounded, + strict: ledgerConfig?.strict ?? false, + ids, + clock, + logger, + }); + const written = adapted; const aggregates = buildAggregates({ registry, ports: ports.byAggregate, config }); const processDefinitions = buildProcesses({ registry, aggregates, config }); const storage = appStorage({ @@ -530,6 +546,7 @@ export const assembleApp: AssembleAppFunction = async ({ clock, pendingRetries, logger, + intentions, }); const policies = buildPolicies({ registry, aggregates }); const policyExecutor = createPolicyExecutor({ @@ -539,6 +556,7 @@ export const assembleApp: AssembleAppFunction = async ({ ids, clock, logger, + intentions, }); const reactive = [ { @@ -579,6 +597,7 @@ export const assembleApp: AssembleAppFunction = async ({ ? undefined : createRetentionSweep({ storage: written, + intentions, retentionMs, ids, clock, @@ -597,11 +616,21 @@ export const assembleApp: AssembleAppFunction = async ({ logger, }); const { concurrencyRetries } = config.runtime.commands; + const retentionDue = + retentionMs === null + ? NEVER_DUE + : retentionDueOf({ + retentionMs, + batchSize: config.runtime.dispatcher.batchSize, + checkpoints: written.checkpointStore, + }); const reconciliation = - ledger === null || ledgerConfig === null + bounded === null || ledgerConfig === null ? undefined : createForgetLedgerReconciliation({ - ledger, + ledger: bounded, + intentions, + retentionDue, storage: written, ids, clock, @@ -666,7 +695,7 @@ export const assembleApp: AssembleAppFunction = async ({ ...(retention === undefined ? [] : [createRetentionSubscriber({ persons: storage.persons })]), - ...(ledger === null ? [] : [createLedgerOutbox({ ledger, keepForMs, clock })]), + ...(bounded === null ? [] : [createLedgerOutbox({ ledger: bounded, keepForMs, clock })]), ], batchSize: config.runtime.dispatcher.batchSize, pollIntervalMs: config.runtime.dispatcher.pollIntervalMs, @@ -764,13 +793,14 @@ export const assembleApp: AssembleAppFunction = async ({ // unless the ledger is strict, whose start waits and fails with it. if (reconciliation !== undefined) { const reconciling = reconciliation.reconcile(clock.now()); + const timeoutMs = ledgerConfig?.timeoutMs ?? 0; const forgotten = ledgerConfig?.strict ? await reconciling - : await waitForLedger(clock, reconciling); + : await waitForLedger(clock, timeoutMs, reconciling); if (forgotten === "late") { logger.error( "forget ledger did not answer in time; the app starts and reconciles in the background", - { timeoutMs: FORGET_LEDGER_WAIT_MS }, + { timeoutMs }, ); } else if (typeof forgotten !== "number") { if (ledgerConfig?.strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); @@ -855,6 +885,7 @@ export const assembleApp: AssembleAppFunction = async ({ run: () => forgetPersonalData({ storage, + intentions, ids, clock, logger, @@ -971,10 +1002,13 @@ export const assembleApp: AssembleAppFunction = async ({ for (const entry of entries) { const again = await forgetAgain({ storage: written, + intentions, ids, clock, concurrencyRetries, entry, + retentionDue, + record: true, actor, }); if (again === null) continue; diff --git a/packages/core/src/kernel/command/reaction-commands.test.ts b/packages/core/src/kernel/command/reaction-commands.test.ts index 56662937..ae0e9db1 100644 --- a/packages/core/src/kernel/command/reaction-commands.test.ts +++ b/packages/core/src/kernel/command/reaction-commands.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import { pendingEvent } from "../../adapter/testing/fixtures.ts"; import { ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import type { Registry } from "../../modules/registry.ts"; +import { boundForgetLedger } from "../personal-data/forget-ledger.ts"; +import { createForgetIntentions } from "../personal-data/intention.ts"; import { createReactionCommandIds } from "../shared/idempotency-key.ts"; import { createKernelHarness, @@ -10,6 +14,7 @@ import { drained, orderRegistry, slowJob, + withoutLedger, } from "../test-support.ts"; import { createUnitOfWork } from "../unit-of-work/unit-of-work.ts"; import { scheduledCommandKey } from "./pipeline.ts"; @@ -25,6 +30,7 @@ const setUp = async (registry: Registry = orderRegistry) => { const harness = await createKernelHarness({ registry }); const unit = createUnitOfWork({ storage: harness.storage }); const { logger, entries } = createRecordingLogger(); + const ids = createSequentialIdGenerator(); const reaction = createReactionCommands({ aggregates: harness.aggregates, pipeline: harness.pipeline, @@ -32,8 +38,9 @@ const setUp = async (registry: Registry = orderRegistry) => { idempotencyKey: "key-1", within: unit, logger, - ids: createSequentialIdGenerator(), + ids, clock: harness.clock, + intentions: withoutLedger({ ids, clock: harness.clock }), maxChainDepth: 5, }); return { ...harness, reaction, unit, entries }; @@ -44,6 +51,44 @@ const forgottenEvents = async (storage: Storage) => (event) => event.type === "PersonalDataForgotten", ); +const withLedger = async () => { + const harness = await createKernelHarness({ registry: orderRegistry }); + const unit = createUnitOfWork({ storage: harness.storage }); + const recorded: ForgetLedgerEntry[] = []; + const keysAtRecord: unknown[] = []; + const ids = createSequentialIdGenerator(); + const { logger, entries } = createRecordingLogger(); + const reaction = createReactionCommands({ + aggregates: harness.aggregates, + pipeline: harness.pipeline, + context, + idempotencyKey: "key-1", + within: unit, + logger, + ids, + clock: harness.clock, + intentions: createForgetIntentions({ + ledger: boundForgetLedger({ + ledger: { + append: async (appended) => { + recorded.push(...appended); + keysAtRecord.push(await harness.storage.personalKeys.get("c-1")); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock: harness.clock, + }), + strict: true, + ids, + clock: harness.clock, + logger, + }), + maxChainDepth: 5, + }); + return { harness, unit, reaction, recorded, keysAtRecord, entries }; +}; + describe("createReactionCommands", () => { it("dispatches one causal hop deeper, with ids derived from the idempotency key", async () => { const { reaction, storage, unit } = await setUp(); @@ -360,6 +405,62 @@ describe("createReactionCommands", () => { }); }); + it("records the forgetting's intention in the ledger just before the commit's transaction", async () => { + const { harness, unit, reaction, recorded, keysAtRecord } = await withLedger(); + await harness.storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-1", kid: null }); + await reaction.forget("c-1"); + expect(recorded).toEqual([]); + await unit.commit(); + expect(keysAtRecord).toEqual([{ key: "k", fingerprint: "f-1", kid: null }]); + const [forgotten] = await forgottenEvents(harness.storage); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: forgotten?.id, + forgottenAt: forgotten?.timestamp, + reason: "requested", + fingerprints: ["f-1"], + }, + ]); + }); + + it("forgets a key given to the person after the intention too, in the same commit", async () => { + const { harness, unit, reaction, recorded } = await withLedger(); + await reaction.forget("c-1"); + unit.beforeCommit(async () => { + await harness.storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-2", kid: null }); + }); + await unit.commit(); + expect(recorded.map(({ fingerprints }) => fingerprints)).toEqual([[]]); + expect(await harness.storage.personalKeys.get("c-1")).toBeNull(); + const [forgotten] = await forgottenEvents(harness.storage); + expect(forgotten?.payload).toMatchObject({ fingerprints: ["f-2"] }); + }); + + it("answers nothing more for a forgetting a reconciliation committed under the intention's id", async () => { + const { harness, unit, reaction, entries } = await withLedger(); + await reaction.forget("c-1"); + unit.beforeCommit(async () => { + await harness.storage.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + pendingEvent({ + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "id-1", + }), + ], + }); + }); + await unit.commit(); + expect((await forgottenEvents(harness.storage)).map(({ id }) => id)).toEqual(["id-1"]); + expect(entries.map(({ message }) => message)).not.toContain("personal data forgotten"); + }); + it("forgets nobody for a unit that never commits", async () => { const { reaction, storage } = await setUp(); await storage.personalKeys.add("c-1", { key: "k", fingerprint: "f", kid: null }); @@ -383,6 +484,7 @@ describe("createReactionCommands", () => { }); const harness = await setUp(); + const deepIds = createSequentialIdGenerator(); const deep = createReactionCommands({ aggregates: harness.aggregates, pipeline: harness.pipeline, @@ -390,8 +492,9 @@ describe("createReactionCommands", () => { idempotencyKey: "key-2", within: harness.unit, logger: createRecordingLogger().logger, - ids: createSequentialIdGenerator(), + ids: deepIds, clock: harness.clock, + intentions: withoutLedger({ ids: deepIds, clock: harness.clock }), maxChainDepth: 5, }); await expect(deep.forget("c-1")).rejects.toMatchObject({ diff --git a/packages/core/src/kernel/command/reaction-commands.ts b/packages/core/src/kernel/command/reaction-commands.ts index 05ef9c05..836d4a3b 100644 --- a/packages/core/src/kernel/command/reaction-commands.ts +++ b/packages/core/src/kernel/command/reaction-commands.ts @@ -9,7 +9,8 @@ import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; import type { CausationContext } from "../../contracts/metadata.ts"; import type { AggregatesRuntime } from "../aggregate/runtime.ts"; -import { forgetWithin, logForgotten } from "../personal-data/forget.ts"; +import { forgetWithin, logForgotten, STALE_INTENTION } from "../personal-data/forget.ts"; +import type { ForgetIntention, ForgetIntentions } from "../personal-data/intention.ts"; import { checkPersonId } from "../personal-data/transact.ts"; import { createReactionCommandIds } from "../shared/idempotency-key.ts"; import { errorDetails } from "../shared/retry.ts"; @@ -58,7 +59,9 @@ export interface ReactionCommands { readonly signal: AbortSignal; /** * Forgets the person when the run's attempt commits, in its transaction: once however often the - * attempt runs, never for one that does not commit, and once for a person named twice. + * attempt runs, and once for a person named twice. Its intention goes to the forget ledger just + * before the transaction opens, so an attempt whose transaction then fails leaves it for the + * next reconciliation, which commits it. */ forget(personId: string): Promise; /** @@ -94,6 +97,7 @@ export interface CreateReactionCommandsArgs { readonly logger: Logger; readonly ids: IdGenerator; readonly clock: Clock; + readonly intentions: ForgetIntentions; /** * How deep a `forget` may go, which no pipeline checks for it as it does for a command. */ @@ -120,6 +124,7 @@ export const createReactionCommands: CreateReactionCommandsFunction = ({ logger, ids, clock, + intentions, maxChainDepth, }) => { const commandIds = createReactionCommandIds(idempotencyKey); @@ -211,15 +216,32 @@ export const createReactionCommands: CreateReactionCommandsFunction = ({ if (depth > maxChainDepth) throw new ChainDepthExceededError(depth, maxChainDepth); if (forgetting.has(personId)) return; forgetting.add(personId); - within.atCommit(async (transaction) => { - const forgotten = await forgetWithin({ - transaction, - ids, - clock, + let intention: ForgetIntention | undefined; + within.beforeCommit(async (storage) => { + intention = await intentions.intend({ personId, reason: "requested", - context: { ...context, depth }, + keys: storage.personalKeys, }); + }); + within.atCommit(async (transaction) => { + const intended = intention as ForgetIntention; + const forget = (named: ForgetIntention) => + forgetWithin({ + transaction, + ids, + clock, + personId, + intention: named, + context: { ...context, depth }, + }); + // A key given after the intention goes too: the commit is one, and a run whose commit + // fails is delivered again, with an intention that names it. + let outcome = await forget(intended); + if (outcome === STALE_INTENTION) + outcome = await forget({ ...intended, fingerprints: undefined }); + if (outcome === STALE_INTENTION || "held" in outcome) return; + const forgotten = outcome; within.afterCommit(() => logForgotten(logger, forgotten)); }); }, diff --git a/packages/core/src/kernel/personal-data/commands.test.ts b/packages/core/src/kernel/personal-data/commands.test.ts index 73eda185..e61d86af 100644 --- a/packages/core/src/kernel/personal-data/commands.test.ts +++ b/packages/core/src/kernel/personal-data/commands.test.ts @@ -373,6 +373,7 @@ describe("personal data in delayed commands", () => { expect(await forgottenPayload()).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [expect.any(String)], cancelledCommands: named.map((entry) => ({ commandId: entry.dedupeKey.replace(/^command:/, ""), type: entry.command.type, @@ -417,6 +418,7 @@ describe("personal data in delayed commands", () => { expect(await forgottenPayload()).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [key?.fingerprint], cancelledCommands: [], removedDeadLetters: [{ id: stored?.id, commandType: "RemindCustomer" }], }); diff --git a/packages/core/src/kernel/personal-data/forget-ledger.ts b/packages/core/src/kernel/personal-data/forget-ledger.ts index c1f76cc5..2b796707 100644 --- a/packages/core/src/kernel/personal-data/forget-ledger.ts +++ b/packages/core/src/kernel/personal-data/forget-ledger.ts @@ -2,7 +2,7 @@ import type { Adapter } from "../../adapter/adapter.ts"; import type { AdapterDefinition } from "../../adapter/adapter-definition.ts"; import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; import type { Clock } from "../../contracts/clock.ts"; -import { ConfigurationError } from "../../contracts/errors.ts"; +import { ConfigurationError, ForgetLedgerTimeoutError } from "../../contracts/errors.ts"; import type { AppEnv } from "../../register/index.ts"; export interface CreateForgetLedgerFromEnvArgs { @@ -104,21 +104,74 @@ export const openForgetLedger: OpenForgetLedgerFunction = async ({ : source.createForgetLedger({ tenant, keepForMs }); }; -// How long the app waits for the ledger, at start or after a forgetting, before it goes on and -// lets the ledger finish behind. -export const FORGET_LEDGER_WAIT_MS = 10_000; +export interface BoundedForgetLedger extends ForgetLedger { + /** + * Whether a call given up for its timeout has not settled yet. + */ + late(): boolean; +} + +export interface BoundForgetLedgerArgs { + readonly ledger: ForgetLedger; + readonly timeoutMs: number; + readonly clock: Clock; +} + +export interface BoundForgetLedgerFunction { + (args: BoundForgetLedgerArgs): BoundedForgetLedger; +} + +/** + * The ledger whose calls reject with `ForgetLedgerTimeoutError` once `timeoutMs` passes without + * them settling. A call given up that way goes on behind, and may still land. + */ +export const boundForgetLedger: BoundForgetLedgerFunction = ({ ledger, timeoutMs, clock }) => { + let late = 0; + const bound = async (operation: string, call: () => Promise): Promise => { + const work = call(); + let cancel: () => void = () => undefined; + const timeout = new Promise((_, reject) => { + cancel = clock.after(timeoutMs, () => { + late += 1; + const settle = (): void => { + late -= 1; + }; + work.then(settle, settle); + reject(new ForgetLedgerTimeoutError(operation, timeoutMs)); + }); + }); + try { + return await Promise.race([work, timeout]); + } finally { + cancel(); + } + }; + const { prune, close } = ledger; + return { + append: (entries) => bound("append", () => ledger.append(entries)), + read: (args) => bound("read", () => ledger.read(args)), + ...(ledger.visibilityDelayMs === undefined + ? {} + : { visibilityDelayMs: ledger.visibilityDelayMs }), + ...(prune === undefined + ? {} + : { prune: (args) => bound("prune", () => prune.call(ledger, args)) }), + ...(close === undefined ? {} : { close: () => close.call(ledger) }), + late: () => late > 0, + }; +}; export interface WaitForLedgerFunction { - (clock: Clock, work: Promise): Promise; + (clock: Clock, timeoutMs: number, work: Promise): Promise; } /** - * `work`, or `"late"` once `FORGET_LEDGER_WAIT_MS` has passed without it settling. + * `work`, or `"late"` once `timeoutMs` has passed without it settling. */ -export const waitForLedger: WaitForLedgerFunction = async (clock, work) => { +export const waitForLedger: WaitForLedgerFunction = async (clock, timeoutMs, work) => { let cancel: () => void = () => undefined; const late = new Promise<"late">((resolve) => { - cancel = clock.after(FORGET_LEDGER_WAIT_MS, () => resolve("late")); + cancel = clock.after(timeoutMs, () => resolve("late")); }); try { return await Promise.race([work, late]); diff --git a/packages/core/src/kernel/personal-data/forget.test.ts b/packages/core/src/kernel/personal-data/forget.test.ts index 80f16fd0..1fd7e136 100644 --- a/packages/core/src/kernel/personal-data/forget.test.ts +++ b/packages/core/src/kernel/personal-data/forget.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { KeyedPerson } from "../../adapter/storage/personal-key-store.ts"; import { + pendingEvent, testCommand, testContext, testDeadLetter, @@ -12,17 +14,22 @@ import { createFixedClock } from "../../contracts/clock.ts"; import { ConcurrencyError, ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { memory } from "../../memory/index.ts"; -import { createRecordingLogger } from "../test-support.ts"; -import { forgetPersonalData } from "./forget.ts"; +import { createRecordingLogger, withoutLedger } from "../test-support.ts"; +import { forgetIntended, forgetPersonalData } from "./forget.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; +import { createForgetIntentions } from "./intention.ts"; const setup = async (storage?: Storage) => { const live = storage ?? (await memory().createStorage()); const { logger, entries: logs } = createRecordingLogger(); + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(); const forget = (personId: string) => forgetPersonalData({ storage: live, - ids: createSequentialIdGenerator(), - clock: createFixedClock(), + intentions: withoutLedger({ ids, clock }), + ids, + clock, logger, concurrencyRetries: 2, personId, @@ -55,6 +62,7 @@ describe("forgetPersonalData", () => { payload: { personId: "c-1", reason: "requested", + fingerprints: ["f-1"], cancelledCommands: [], removedDeadLetters: [], }, @@ -131,6 +139,7 @@ describe("forgetPersonalData", () => { expect(event?.payload).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [], cancelledCommands: [ { commandId: "cmd-1", type: "RemindCustomer" }, { commandId: "deadline:x", type: "RemindCustomer" }, @@ -187,3 +196,133 @@ describe("forgetPersonalData", () => { await expect(forget("c-1")).rejects.toBeInstanceOf(ConcurrencyError); }); }); + +describe("forgetIntended", () => { + const recordingIntentions = (ids = createSequentialIdGenerator()) => { + const recorded: ForgetLedgerEntry[] = []; + const clock = createFixedClock(); + const intentions = createForgetIntentions({ + ledger: boundForgetLedger({ + ledger: { + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + ids, + clock, + logger: createRecordingLogger().logger, + }); + return { intentions, recorded, ids, clock }; + }; + + it("takes a new intention that names a key given to the person between the intention and the commit", async () => { + const base = await memory().createStorage(); + await base.personalKeys.add("c-1", testPersonalKey("1")); + let raced = false; + const racing: Storage = { + ...base, + transact: async (work) => { + if (!raced) { + raced = true; + // Another forgetting committed meanwhile, and the person came back with a new key. + await base.personalKeys.remove("c-1"); + await base.personalKeys.add("c-1", testPersonalKey("2")); + } + return base.transact(work); + }, + }; + const { intentions, recorded, ids, clock } = recordingIntentions(); + const forgotten = await forgetIntended({ + storage: racing, + intentions, + ids, + clock, + concurrencyRetries: 0, + personId: "c-1", + reason: "requested", + context: testContext, + }); + expect(recorded.map(({ eventId, fingerprints }) => [eventId, fingerprints])).toEqual([ + ["id-1", ["f-1"]], + ["id-2", ["f-2"]], + ]); + expect(forgotten).toMatchObject({ + hadKey: true, + event: { id: "id-2", payload: { fingerprints: ["f-2"] } }, + }); + expect(await base.personalKeys.get("c-1")).toBeNull(); + }); + + it("forgets a person whose key went since the intention, naming none", async () => { + const base = await memory().createStorage(); + await base.personalKeys.add("c-1", testPersonalKey("1")); + let raced = false; + const racing: Storage = { + ...base, + transact: async (work) => { + if (!raced) { + raced = true; + await base.personalKeys.remove("c-1"); + } + return base.transact(work); + }, + }; + const { intentions, recorded, ids, clock } = recordingIntentions(); + const forgotten = await forgetIntended({ + storage: racing, + intentions, + ids, + clock, + concurrencyRetries: 0, + personId: "c-1", + reason: "requested", + context: testContext, + }); + expect(recorded).toHaveLength(1); + expect(forgotten).toMatchObject({ hadKey: false, payload: { fingerprints: [] } }); + }); + + it("answers with the forgetting a reconciliation committed under the intention's id meanwhile", async () => { + const { storage, logs } = await setup(); + const intended = { + eventId: "id-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested" as const, + fingerprints: [], + }; + await storage.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + { + ...pendingEvent({ + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "id-1", + }), + person: "c-1", + }, + ], + }); + const dispatch = await forgetPersonalData({ + storage, + intentions: { intend: async () => intended, record: async () => undefined }, + ids: createSequentialIdGenerator({ prefix: "other" }), + clock: createFixedClock(), + logger: createRecordingLogger().logger, + concurrencyRetries: 0, + personId: "c-1", + }); + expect(dispatch).toMatchObject({ eventIds: ["id-1"], version: 1, position: 1 }); + expect(await storage.eventStore.lastPosition()).toBe(1); + expect(logs).toEqual([]); + }); +}); diff --git a/packages/core/src/kernel/personal-data/forget.ts b/packages/core/src/kernel/personal-data/forget.ts index 53a4a8c0..400b5959 100644 --- a/packages/core/src/kernel/personal-data/forget.ts +++ b/packages/core/src/kernel/personal-data/forget.ts @@ -14,6 +14,7 @@ import { import { scheduledCommandId } from "../command/pipeline.ts"; import { actedBy, startedBy } from "../shared/causation.ts"; import { appendSystemEvent } from "../system-events.ts"; +import type { ForgetIntention, ForgetIntentions } from "./intention.ts"; import { checkPersonId, transactRetrying } from "./transact.ts"; export interface ForgetWithinArgs { @@ -21,7 +22,7 @@ export interface ForgetWithinArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly personId: string; - readonly reason: PersonalDataForgottenReason; + readonly intention: ForgetIntention; readonly context: CausationContext; } @@ -31,25 +32,53 @@ export interface Forgotten { readonly event: StoredEvent; } +/** + * The forgetting the intention names is in the store already, committed by a reconciliation + * meanwhile. + */ +export interface HeldForgetting { + readonly held: StoredEvent; +} + +/** + * The person holds a key the intention does not name: given to them after it was written. + */ +export const STALE_INTENTION: "stale" = "stale"; + +export type ForgetOutcome = Forgotten | HeldForgetting | typeof STALE_INTENTION; + export interface ForgetWithinFunction { - (args: ForgetWithinArgs): Promise; + (args: ForgetWithinArgs): Promise; } /** - * Removes the person's key and appends `PersonalDataForgotten` to their stream, on the - * transaction given: whoever reads the event finds the fields gone already. The scheduled - * commands and dead letters that hold the person's data go in the same transaction, and the event - * names them. A person without a key is forgotten all the same, since read models may hold what - * their events carried in the clear. + * Removes the person's key and appends `PersonalDataForgotten` to their stream, with the id, time + * and reason of the intention, on the transaction given: whoever reads the event finds the fields + * gone already. The scheduled commands and dead letters that hold the person's data go in the same + * transaction, and the event names them. A person without a key is forgotten all the same, since + * read models may hold what their events carried in the clear. */ export const forgetWithin: ForgetWithinFunction = async ({ transaction, ids, clock, personId, - reason, + intention, context, }) => { + const { fingerprints } = intention; + if (fingerprints !== undefined) { + const held = await transaction.eventStore.readOne({ + aggregateType: BOUNDA_AGGREGATE, + aggregateId: personId, + eventId: intention.eventId, + }); + if (held !== null) return { held }; + } + const key = await transaction.personalKeys.get(personId); + if (key !== null && fingerprints !== undefined && !fingerprints.includes(key.fingerprint)) { + return STALE_INTENTION; + } // The key goes first: a snapshot written beside the transaction then either waits for it and // is forgotten with the rest, or finds no key and is never written. The scheduled commands go // before the dead letters: a worker giving one up writes its letter and settles its claim @@ -62,7 +91,8 @@ export const forgetWithin: ForgetWithinFunction = async ({ const letters = await transaction.deadLetterStore.forget(personId); const payload: PersonalDataForgottenPayload = { personId, - reason, + reason: intention.reason, + fingerprints: key === null ? [] : [key.fingerprint], cancelledCommands: cancelled.map(({ dedupeKey, command }) => ({ commandId: scheduledCommandId(dedupeKey) ?? dedupeKey, type: command.type, @@ -79,10 +109,53 @@ export const forgetWithin: ForgetWithinFunction = async ({ payload, context, person: personId, + identity: { id: intention.eventId, timestamp: intention.forgottenAt }, }); return { hadKey, payload, event }; }; +export interface ForgetIntendedArgs { + readonly storage: Pick; + readonly intentions: ForgetIntentions; + readonly ids: IdGenerator; + readonly clock: Clock; + readonly concurrencyRetries: number; + readonly personId: string; + readonly reason: PersonalDataForgottenReason; + readonly context: CausationContext; +} + +export interface ForgetIntendedFunction { + (args: ForgetIntendedArgs): Promise; +} + +/** + * Records the intention, then forgets in a transaction of its own. A key given to the person + * between the two takes a new intention that names it: each round needs a forgetting of the same + * person to commit in between, so the rounds end when those do. + */ +export const forgetIntended: ForgetIntendedFunction = async ({ + storage, + intentions, + ids, + clock, + concurrencyRetries, + personId, + reason, + context, +}) => { + for (;;) { + const intention = await intentions.intend({ personId, reason, keys: storage.personalKeys }); + const outcome = await transactRetrying({ + storage, + concurrencyRetries, + work: (transaction) => + forgetWithin({ transaction, ids, clock, personId, intention, context }), + }); + if (outcome !== STALE_INTENTION) return outcome; + } +}; + export interface LogForgottenFunction { (logger: Logger, forgotten: Forgotten): void; } @@ -114,6 +187,7 @@ export const forgottenDispatch: ForgottenDispatchFunction = (event) => ({ export interface ForgetPersonalDataArgs { readonly storage: Storage; + readonly intentions: ForgetIntentions; readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; @@ -127,10 +201,11 @@ export interface ForgetPersonalDataFunction { } /** - * `forget` of a client: `forgetWithin` in a transaction of its own, as a request of the person. + * `forget` of a client: `forgetIntended`, as a request of the person. */ export const forgetPersonalData: ForgetPersonalDataFunction = async ({ storage, + intentions, ids, clock, logger, @@ -140,19 +215,17 @@ export const forgetPersonalData: ForgetPersonalDataFunction = async ({ }) => { checkPersonId(personId, "forget"); const correlationId = ids.next(); - const forgotten = await transactRetrying({ + const forgotten = await forgetIntended({ storage, + intentions, + ids, + clock, concurrencyRetries, - work: (transaction) => - forgetWithin({ - transaction, - ids, - clock, - personId, - reason: "requested", - context: startedBy(correlationId, actor), - }), + personId, + reason: "requested", + context: startedBy(correlationId, actor), }); + if ("held" in forgotten) return forgottenDispatch(forgotten.held); logForgotten(logger, forgotten); return forgottenDispatch(forgotten.event); }; diff --git a/packages/core/src/kernel/personal-data/intention.test.ts b/packages/core/src/kernel/personal-data/intention.test.ts new file mode 100644 index 00000000..249ffb33 --- /dev/null +++ b/packages/core/src/kernel/personal-data/intention.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from "vitest"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import { testPersonalKey } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { ForgetLedgerUnavailableError } from "../../contracts/errors.ts"; +import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import { memory } from "../../memory/index.ts"; +import { createRecordingLogger } from "../test-support.ts"; +import { type BoundedForgetLedger, boundForgetLedger } from "./forget-ledger.ts"; +import { createForgetIntentions } from "./intention.ts"; + +const NOW = new Date("2026-03-01T00:00:00.000Z"); + +const setUp = async ({ + ledger, + strict = false, +}: { + readonly ledger: BoundedForgetLedger | null; + readonly strict?: boolean; +}) => { + const storage = await memory().createStorage(); + const { logger, entries: logs } = createRecordingLogger(); + const intentions = createForgetIntentions({ + ledger, + strict, + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + logger, + }); + return { storage, intentions, logs }; +}; + +const recordingLedger = (append: (entries: readonly ForgetLedgerEntry[]) => Promise) => + boundForgetLedger({ + ledger: { append, read: async () => [] }, + timeoutMs: 1_000, + clock: createFixedClock(NOW), + }); + +describe("the intentions of forgetting", () => { + it("record each in the ledger before they resolve, naming the key the person holds now", async () => { + const appended: ForgetLedgerEntry[] = []; + const { storage, intentions } = await setUp({ + ledger: recordingLedger(async (entries) => { + appended.push(...entries); + }), + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + expect( + await intentions.intend({ personId: "c-1", reason: "retention", keys: storage.personalKeys }), + ).toEqual({ + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "retention", + fingerprints: ["f-1"], + }); + expect( + await intentions.intend({ personId: "c-2", reason: "requested", keys: storage.personalKeys }), + ).toMatchObject({ eventId: "id-2", fingerprints: [] }); + expect(appended).toEqual([ + { + personId: "c-1", + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "retention", + fingerprints: ["f-1"], + }, + { + personId: "c-2", + eventId: "id-2", + forgottenAt: NOW.toISOString(), + reason: "requested", + fingerprints: [], + }, + ]); + }); + + it("read no key and record nothing without a ledger, leaving any key to the forgetting", async () => { + const { storage, intentions } = await setUp({ ledger: null }); + let reads = 0; + const keys = { + get: async (person: string) => { + reads += 1; + return storage.personalKeys.get(person); + }, + }; + expect(await intentions.intend({ personId: "c-1", reason: "requested", keys })).toEqual({ + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "requested", + }); + await intentions.record({ + personId: "c-1", + eventId: "e-1", + forgottenAt: NOW.toISOString(), + reason: "requested", + }); + expect(reads).toBe(0); + }); + + it("refuse, when strict, a forgetting the ledger does not take, with why", async () => { + const down = new Error("ledger is down"); + const { storage, intentions } = await setUp({ + ledger: recordingLedger(async () => { + throw down; + }), + strict: true, + }); + const refused = intentions.intend({ + personId: "c-1", + reason: "requested", + keys: storage.personalKeys, + }); + await expect(refused).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + await expect(refused).rejects.toMatchObject({ cause: down }); + }); + + it("log a forgetting the ledger does not take, when not strict, and let it go on", async () => { + const { storage, intentions, logs } = await setUp({ + ledger: recordingLedger(async () => { + throw new Error("ledger is down"); + }), + }); + expect( + await intentions.intend({ personId: "c-1", reason: "requested", keys: storage.personalKeys }), + ).toMatchObject({ eventId: "id-1", fingerprints: [] }); + expect(logs).toEqual([ + { + level: "warn", + message: "forget ledger did not take the forgetting; it commits, and the stream copies it", + fields: expect.objectContaining({ errorName: "Error", errorMessage: "ledger is down" }), + }, + ]); + }); + + it("skip, when not strict, a ledger whose call given up for its timeout is still pending", async () => { + let late = true; + let appends = 0; + const ledger: BoundedForgetLedger = { + append: async () => { + appends += 1; + }, + read: async () => [], + late: () => late, + }; + const loose = await setUp({ ledger }); + await loose.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: loose.storage.personalKeys, + }); + expect(appends).toBe(0); + const strict = await setUp({ ledger, strict: true }); + await strict.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: strict.storage.personalKeys, + }); + expect(appends).toBe(1); + late = false; + await loose.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: loose.storage.personalKeys, + }); + expect(appends).toBe(2); + }); +}); diff --git a/packages/core/src/kernel/personal-data/intention.ts b/packages/core/src/kernel/personal-data/intention.ts new file mode 100644 index 00000000..4b405a7f --- /dev/null +++ b/packages/core/src/kernel/personal-data/intention.ts @@ -0,0 +1,102 @@ +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import type { PersonalKeyStore } from "../../adapter/storage/personal-key-store.ts"; +import type { Clock } from "../../contracts/clock.ts"; +import { ForgetLedgerUnavailableError } from "../../contracts/errors.ts"; +import type { IdGenerator } from "../../contracts/ids.ts"; +import type { Logger } from "../../contracts/logger.ts"; +import type { PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; +import type { BoundedForgetLedger } from "./forget-ledger.ts"; + +/** + * The forgetting about to commit, as its `PersonalDataForgotten` will carry it. + */ +export interface ForgetIntention { + readonly eventId: string; + readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + /** + * The keys the intention names, which the forgetting may destroy: one the person holds that is + * not here was given to them after the intention was written. `undefined` without a ledger, and + * for an entry that does not record them, which any key matches. + */ + readonly fingerprints?: readonly string[] | undefined; +} + +export interface IntendArgs { + readonly personId: string; + readonly reason: PersonalDataForgottenReason; + /** + * Where the key the person holds now is read, outside any transaction. + */ + readonly keys: Pick; +} + +export interface ForgetIntentions { + /** + * The intention of forgetting the person now, naming the key they hold, recorded in the ledger + * before it resolves. + */ + intend(args: IntendArgs): Promise; + /** + * Records an entry in the ledger. A strict ledger that does not take it rejects with + * `ForgetLedgerUnavailableError`; any other failure is logged, and the stream copies the + * forgetting once it commits. + */ + record(entry: ForgetLedgerEntry): Promise; +} + +export interface CreateForgetIntentionsArgs { + /** + * `null` without a ledger: nothing is recorded, and no key is read for it. + */ + readonly ledger: BoundedForgetLedger | null; + readonly strict: boolean; + readonly ids: IdGenerator; + readonly clock: Clock; + readonly logger: Logger; +} + +export interface CreateForgetIntentionsFunction { + (args: CreateForgetIntentionsArgs): ForgetIntentions; +} + +/** + * Every forgetting writes its intention first, so the ledger holds it whether or not the commit + * happens: an intention whose commit does not happen is committed by the next reconciliation, + * since forgetting is what was asked. While a call given up for its timeout is still pending, a + * ledger that is not strict is skipped, rather than each forgetting waiting on one that hangs. + */ +export const createForgetIntentions: CreateForgetIntentionsFunction = ({ + ledger, + strict, + ids, + clock, + logger, +}) => { + const record = async (entry: ForgetLedgerEntry): Promise => { + if (ledger === null || (!strict && ledger.late())) return; + try { + await ledger.append([entry]); + } catch (error) { + if (strict) throw new ForgetLedgerUnavailableError(error); + logger.warn( + "forget ledger did not take the forgetting; it commits, and the stream copies it", + { + error, + }, + ); + } + }; + return { + intend: async ({ personId, reason, keys }) => { + const eventId = ids.next(); + const forgottenAt = clock.now().toISOString(); + if (ledger === null) return { eventId, forgottenAt, reason }; + const key = await keys.get(personId); + const fingerprints = key === null ? [] : [key.fingerprint]; + await record({ personId, eventId, forgottenAt, reason, fingerprints }); + return { eventId, forgottenAt, reason, fingerprints }; + }, + record, + }; +}; diff --git a/packages/core/src/kernel/personal-data/ledger-append.test.ts b/packages/core/src/kernel/personal-data/ledger-append.test.ts deleted file mode 100644 index 4722affd..00000000 --- a/packages/core/src/kernel/personal-data/ledger-append.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Storage } from "../../adapter/adapter.ts"; -import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; -import { pendingEvent } from "../../adapter/testing/fixtures.ts"; -import { createFixedClock } from "../../contracts/clock.ts"; -import { type Logger, silentLogger } from "../../contracts/logger.ts"; -import { memory } from "../../memory/index.ts"; -import { FORGET_LEDGER_WAIT_MS } from "./forget-ledger.ts"; -import { withLedgerAppend } from "./ledger-append.ts"; - -const NOW = new Date("2026-01-01T00:01:00.000Z"); - -const forgetting = (personId: string, version = 1) => - pendingEvent({ - aggregateType: "bounda", - aggregateId: personId, - version, - type: "PersonalDataForgotten", - id: `f-${personId}-${version}`, - }); - -const setUp = async ({ - ledger, - keepForMs = null, - logger = silentLogger, -}: { - readonly ledger?: Partial; - readonly keepForMs?: number | null; - readonly logger?: Logger; -} = {}) => { - const appended: (readonly ForgetLedgerEntry[])[] = []; - const inner = await memory().createStorage(); - const storage: Storage = withLedgerAppend({ - storage: inner, - ledger: { - append: async (entries) => { - appended.push(entries); - }, - read: async () => [], - ...ledger, - }, - keepForMs, - clock: createFixedClock(NOW), - logger, - }); - return { storage, appended }; -}; - -describe("the append to the forget ledger as a transaction commits", () => { - it("appends the forgettings a transaction committed, in one call, and nothing else", async () => { - const { storage, appended } = await setUp(); - const result = await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "order", - aggregateId: "o-1", - expectedVersion: 0, - events: [pendingEvent({ aggregateId: "o-1", version: 1 })], - }); - for (const person of ["c-1", "c-2"]) { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }); - } - return "done"; - }); - expect(result).toBe("done"); - expect(appended).toEqual([ - [ - { personId: "c-1", eventId: "f-c-1-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - { personId: "c-2", eventId: "f-c-2-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - ], - ]); - await storage.transact(async () => undefined); - expect(appended).toHaveLength(1); - await storage.transact((transaction) => - transaction.eventStore.reseal([{ position: 1, payload: { total: 9 }, person: "c-9" }]), - ); - expect( - await storage.eventStore.readByPerson({ person: "c-9", afterPosition: 0, limit: 1 }), - ).toMatchObject([{ payload: { total: 9 } }]); - expect(appended).toHaveLength(1); - }); - - it("appends nothing for a transaction that rolls back, nor for an earlier run of the work", async () => { - const { storage, appended } = await setUp(); - await expect( - storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }); - throw new Error("rolled back"); - }), - ).rejects.toThrow("rolled back"); - expect(appended).toEqual([]); - const inner = await memory().createStorage(); - let runs = 0; - const retrying = withLedgerAppend({ - storage: { - ...inner, - // A store that runs the work again on a fresh transaction, as a busy one may. - transact: async (work) => { - await inner.transact(work).catch(() => undefined); - return inner.transact(work); - }, - }, - ledger: { - append: async (entries) => { - appended.push(entries); - }, - read: async () => [], - }, - keepForMs: null, - clock: createFixedClock(NOW), - logger: silentLogger, - }); - await retrying.transact(async (transaction) => { - runs += 1; - const person = `c-${runs}`; - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }); - if (runs === 1) throw new Error("busy"); - }); - expect(appended.map((entries) => entries.map(({ personId }) => personId))).toEqual([["c-2"]]); - }); - - it("leaves out a forgetting older than keepFor", async () => { - const { storage, appended } = await setUp({ keepForMs: 60_000 }); - await storage.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }), - ); - expect(appended).toEqual([]); - const { storage: fresh, appended: kept } = await setUp({ keepForMs: 60_001 }); - await fresh.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }), - ); - expect(kept).toHaveLength(1); - }); - - it("logs a ledger that refuses the forgetting and resolves what the transaction did", async () => { - const warnings: unknown[] = []; - const { storage } = await setUp({ - ledger: { - append: async () => { - throw new Error("ledger is down"); - }, - }, - logger: { ...silentLogger, warn: (message, fields) => warnings.push([message, fields]) }, - }); - const result = await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }); - return "committed"; - }); - expect(result).toBe("committed"); - expect(warnings).toEqual([ - [ - "forget ledger did not take the forgetting; the stream copies it", - expect.objectContaining({ - forgotten: 1, - error: expect.objectContaining({ message: "ledger is down" }), - }), - ], - ]); - }); - - it("leaves forgettings to the stream while an append that did not answer in time is pending", async () => { - const inner = await memory().createStorage(); - const clock = createFixedClock(NOW); - let answer: () => void = () => undefined; - const appends: string[][] = []; - const storage = withLedgerAppend({ - storage: inner, - ledger: { - append: (entries) => { - appends.push(entries.map(({ personId }) => personId)); - return appends.length === 1 - ? new Promise((resolve) => { - answer = resolve; - }) - : Promise.resolve(); - }, - read: async () => [], - }, - keepForMs: null, - clock, - logger: silentLogger, - }); - const forget = (person: string) => - storage.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }), - ); - let done = false; - const first = forget("c-1").then(() => { - done = true; - }); - while (!done) { - clock.advance(FORGET_LEDGER_WAIT_MS); - await new Promise((resolve) => setTimeout(resolve, 1)); - } - await first; - await forget("c-2"); - expect(appends).toEqual([["c-1"]]); - answer(); - await new Promise((resolve) => setTimeout(resolve, 1)); - await forget("c-3"); - expect(appends).toEqual([["c-1"], ["c-3"]]); - }); - - it("reads through the transaction's event store as it is", async () => { - const { storage } = await setUp(); - await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "order", - aggregateId: "o-1", - expectedVersion: 0, - events: [pendingEvent({ aggregateId: "o-1", version: 1 })], - }); - const loaded = await transaction.eventStore.load({ - aggregateType: "order", - aggregateId: "o-1", - }); - expect(loaded.version).toBe(1); - }); - const { eventStore } = await storage.transact(async (transaction) => transaction); - expect( - await eventStore.readOne({ - aggregateType: "order", - aggregateId: "o-1", - eventId: "order-o-1-1", - }), - ).toMatchObject({ id: "order-o-1-1" }); - expect(await eventStore.lastPosition()).toBe(1); - expect(await eventStore.readAll({ afterPosition: 0, limit: 10 })).toHaveLength(1); - expect(await eventStore.readByPerson({ afterPosition: 0, limit: 10, person: "c-1" })).toEqual( - [], - ); - }); -}); diff --git a/packages/core/src/kernel/personal-data/ledger-append.ts b/packages/core/src/kernel/personal-data/ledger-append.ts deleted file mode 100644 index acb182ca..00000000 --- a/packages/core/src/kernel/personal-data/ledger-append.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { Storage } from "../../adapter/adapter.ts"; -import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; -import type { Clock } from "../../contracts/clock.ts"; -import type { StoredEvent } from "../../contracts/event.ts"; -import type { Logger } from "../../contracts/logger.ts"; -import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; -import { FORGET_LEDGER_WAIT_MS, waitForLedger } from "./forget-ledger.ts"; - -export interface IsForgettingFunction { - (event: StoredEvent): boolean; -} - -export const isForgetting: IsForgettingFunction = (event) => - event.aggregateType === BOUNDA_AGGREGATE && event.type === PERSONAL_DATA_FORGOTTEN_EVENT; - -export interface LedgerEntriesArgs { - readonly events: readonly StoredEvent[]; - /** - * `null` keeps every entry. - */ - readonly keepForMs: number | null; - readonly now: Date; -} - -export interface LedgerEntriesFunction { - (args: LedgerEntriesArgs): readonly ForgetLedgerEntry[]; -} - -/** - * The entries of the forgettings among `events`, but those already older than `keepFor`, which a - * prune would remove anyway. - */ -export const ledgerEntries: LedgerEntriesFunction = ({ events, keepForMs, now }) => - events - .filter(isForgetting) - .filter( - (event) => keepForMs === null || Date.parse(event.timestamp) + keepForMs > now.getTime(), - ) - .map((event) => ({ - personId: event.aggregateId, - eventId: event.id, - forgottenAt: event.timestamp, - })); - -export interface WithLedgerAppendArgs { - readonly storage: Storage; - readonly ledger: ForgetLedger; - readonly keepForMs: number | null; - readonly clock: Clock; - readonly logger: Logger; -} - -export interface WithLedgerAppendFunction { - (args: WithLedgerAppendArgs): Storage; -} - -/** - * The storage whose transactions append the forgettings they commit to the ledger before they - * resolve, so a forgetting reaches the ledger unless the process stops between the commit and the - * append. The outbox copies every one again from the stream, which the ledger takes as an entry - * it holds, so a ledger that fails or does not answer in time is only logged, and while an append - * that did not answer in time is still pending, the next forgettings are left to the outbox - * rather than each waiting for a ledger that hangs. - */ -export const withLedgerAppend: WithLedgerAppendFunction = ({ - storage, - ledger, - keepForMs, - clock, - logger, -}) => { - let stalled = false; - return { - ...storage, - transact: async (work) => { - const appended: StoredEvent[] = []; - const result = await storage.transact((transaction) => { - // A store may run the work again on a fresh transaction: only the last run commits. - appended.length = 0; - const { eventStore } = transaction; - return work({ - ...transaction, - eventStore: { - append: async (args) => { - const written = await eventStore.append(args); - appended.push(...written.events); - return written; - }, - load: (args) => eventStore.load(args), - readOne: (args) => eventStore.readOne(args), - readAll: (args) => eventStore.readAll(args), - readByPerson: (args) => eventStore.readByPerson(args), - lastPosition: () => eventStore.lastPosition(), - reseal: (events) => eventStore.reseal(events), - }, - }); - }); - const entries = ledgerEntries({ events: appended, keepForMs, now: clock.now() }); - if (entries.length > 0 && !stalled) { - const appending = ledger.append(entries); - const outcome = await waitForLedger(clock, appending).catch((error: unknown) => ({ - error, - })); - if (outcome === "late") { - stalled = true; - const resume = (): void => { - stalled = false; - }; - appending.then(resume, resume); - logger.warn("forget ledger did not take the forgetting in time; the stream copies it", { - forgotten: entries.length, - timeoutMs: FORGET_LEDGER_WAIT_MS, - }); - } else if (outcome !== undefined) { - logger.warn("forget ledger did not take the forgetting; the stream copies it", { - forgotten: entries.length, - error: outcome.error, - }); - } - } - return result; - }, - }; -}; diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts index 6c39319a..8d22ae0b 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts @@ -1,18 +1,19 @@ import { trace } from "@opentelemetry/api"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import { pendingEvent } from "../../adapter/testing/fixtures.ts"; import { createFixedClock } from "../../contracts/clock.ts"; +import { ForgetLedgerTimeoutError } from "../../contracts/errors.ts"; import type { StoredEvent } from "../../contracts/event.ts"; import { memory } from "../../memory/index.ts"; import { + boundForgetLedger, defineForgetLedger, - FORGET_LEDGER_WAIT_MS, isForgetLedgerSource, openForgetLedger, waitForLedger, } from "./forget-ledger.ts"; -import { createLedgerOutbox, FORGET_LEDGER_SUBSCRIBER } from "./ledger-outbox.ts"; +import { createLedgerOutbox, FORGET_LEDGER_SUBSCRIBER, forgettingEntry } from "./ledger-outbox.ts"; const span = trace.getTracer("test").startSpan("test"); @@ -63,8 +64,18 @@ describe("the forget ledger's outbox", () => { expect(await outbox.process(events, span)).toBe(5); expect(appended).toEqual([ [ - { personId: "c-1", eventId: "f-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - { personId: "c-2", eventId: "f-2", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "c-1", + eventId: "f-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, + { + personId: "c-2", + eventId: "f-2", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ], ]); expect(await outbox.process(events.slice(0, 3), span)).toBe(3); @@ -100,13 +111,63 @@ describe("the forget ledger's outbox", () => { ]; expect(await outbox.process(events, span)).toBe(3); expect(appended).toEqual([ - [{ personId: "c-3", eventId: "f-3", forgottenAt: "2026-01-01T00:01:00.001Z" }], + [ + { + personId: "c-3", + eventId: "f-3", + forgottenAt: "2026-01-01T00:01:00.001Z", + reason: "requested", + }, + ], ]); expect(await outbox.process(events.slice(0, 2), span)).toBe(2); expect(appended).toHaveLength(1); }); }); +describe("a forgetting's entry", () => { + const forgetting = (payload: unknown): StoredEvent => ({ + ...stored(4, { + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "f-1", + }), + payload, + }); + + it("carries the reason and the keys its event names, as its intention wrote them", () => { + expect( + forgettingEntry(forgetting({ personId: "c-1", reason: "retention", fingerprints: ["k-1"] })), + ).toEqual({ + personId: "c-1", + eventId: "f-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "retention", + fingerprints: ["k-1"], + }); + expect(forgettingEntry(forgetting({ reason: "requested", fingerprints: [] }))).toMatchObject({ + reason: "requested", + fingerprints: [], + }); + }); + + it("reads as a request naming no keys what an earlier event did not record", () => { + for (const payload of [ + {}, + null, + "text", + { reason: "restored", fingerprints: "k-1" }, + { reason: 3, fingerprints: ["k-1", 2] }, + ]) { + const entry = forgettingEntry(forgetting(payload)); + expect(entry.reason).toBe("requested"); + expect(entry).not.toHaveProperty("fingerprints"); + } + }); +}); + describe("a forget ledger's source", () => { it("is an adapter that opens ledgers or a ledger definition, nothing else", () => { const definition = defineForgetLedger(() => ({ append: async () => {}, read: async () => [] })); @@ -140,7 +201,12 @@ describe("a forget ledger's source", () => { tenant: "t-1", }); await ledger?.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ]); expect( await (await adapter.createForgetLedger({ tenant: "t-1" })).read({ limit: 1 }), @@ -181,13 +247,100 @@ describe("a forget ledger's source", () => { describe("the wait for the forget ledger", () => { it("resolves what the ledger answers in time and lets its timer go, or late once it passes", async () => { const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); - expect(await waitForLedger(clock, Promise.resolve(3))).toBe(3); + expect(await waitForLedger(clock, 500, Promise.resolve(3))).toBe(3); expect(clock.pending()).toBe(0); - await expect(waitForLedger(clock, Promise.reject(new Error("down")))).rejects.toThrow("down"); + await expect(waitForLedger(clock, 500, Promise.reject(new Error("down")))).rejects.toThrow( + "down", + ); + expect(clock.pending()).toBe(0); + let settled: unknown; + void waitForLedger(clock, 500, new Promise(() => undefined)).then((outcome) => { + settled = outcome; + }); + clock.advance(499); + await Promise.resolve(); + expect(settled).toBeUndefined(); + clock.advance(1); + await vi.waitFor(() => expect(settled).toBe("late")); + }); +}); + +describe("the forget ledger bound by its timeout", () => { + const entry = { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested" as const, + }; + + it("passes every call through, its delay and close included, and lets each timer go", async () => { + const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); + const calls: string[] = []; + const ledger = boundForgetLedger({ + ledger: { + visibilityDelayMs: 60_000, + append: async (entries) => { + calls.push(`append ${entries.length}`); + }, + read: async ({ limit }) => { + calls.push(`read ${limit}`); + return [entry]; + }, + prune: async ({ before }) => { + calls.push(`prune ${before}`); + }, + close: async () => { + calls.push("close"); + }, + }, + timeoutMs: 500, + clock, + }); + await ledger.append([entry]); + expect(await ledger.read({ limit: 3 })).toEqual([entry]); + await ledger.prune?.({ before: "t" }); + await ledger.close?.(); + expect(calls).toEqual(["append 1", "read 3", "prune t", "close"]); + expect(ledger.visibilityDelayMs).toBe(60_000); + expect(ledger.late()).toBe(false); expect(clock.pending()).toBe(0); - const waiting = waitForLedger(clock, new Promise(() => undefined)); - clock.advance(FORGET_LEDGER_WAIT_MS - 1); + const bare = boundForgetLedger({ + ledger: { append: async () => {}, read: async () => [] }, + timeoutMs: 500, + clock, + }); + expect(bare).not.toHaveProperty("visibilityDelayMs"); + expect(bare).not.toHaveProperty("prune"); + expect(bare).not.toHaveProperty("close"); + }); + + it("gives a call up at its timeout, and says it is late until that call settles", async () => { + const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); + let answer: () => void = () => undefined; + const ledger = boundForgetLedger({ + ledger: { + append: () => + new Promise((resolve) => { + answer = resolve; + }), + read: () => Promise.reject(new Error("down")), + prune: () => new Promise(() => undefined), + }, + timeoutMs: 500, + clock, + }); + const appending = ledger.append([entry]); + clock.advance(499); + expect(ledger.late()).toBe(false); clock.advance(1); - expect(await waiting).toBe("late"); + await expect(appending).rejects.toEqual(new ForgetLedgerTimeoutError("append", 500)); + expect(ledger.late()).toBe(true); + answer(); + await vi.waitFor(() => expect(ledger.late()).toBe(false)); + await expect(ledger.read({ limit: 1 })).rejects.toThrow("down"); + expect(ledger.late()).toBe(false); + const pruning = ledger.prune?.({ before: "t" }); + clock.advance(500); + await expect(pruning).rejects.toMatchObject({ operation: "prune", timeoutMs: 500 }); }); }); diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.ts b/packages/core/src/kernel/personal-data/ledger-outbox.ts index 27a4e144..ee1784b3 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.ts @@ -1,10 +1,69 @@ -import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; +import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { Clock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; +import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; import type { Subscriber } from "../dispatch/delivery.ts"; -import { ledgerEntries } from "./ledger-append.ts"; export const FORGET_LEDGER_SUBSCRIBER: "forget-ledger" = "forget-ledger"; +export interface IsForgettingFunction { + (event: StoredEvent): boolean; +} + +export const isForgetting: IsForgettingFunction = (event) => + event.aggregateType === BOUNDA_AGGREGATE && event.type === PERSONAL_DATA_FORGOTTEN_EVENT; + +export interface ForgettingEntryFunction { + (event: StoredEvent): ForgetLedgerEntry; +} + +/** + * The entry of a `PersonalDataForgotten`, as its intention wrote it. An event written before + * forgettings named their keys leaves `fingerprints` out, and one whose reason is no longer known + * reads as `requested`. + */ +export const forgettingEntry: ForgettingEntryFunction = (event) => { + const field = (name: string): unknown => + typeof event.payload === "object" && event.payload !== null + ? Reflect.get(event.payload, name) + : undefined; + const fingerprints = field("fingerprints"); + const entry: ForgetLedgerEntry = { + personId: event.aggregateId, + eventId: event.id, + forgottenAt: event.timestamp, + reason: field("reason") === "retention" ? "retention" : "requested", + }; + return Array.isArray(fingerprints) && fingerprints.every((value) => typeof value === "string") + ? { ...entry, fingerprints } + : entry; +}; + +export interface LedgerEntriesArgs { + readonly events: readonly StoredEvent[]; + /** + * `null` keeps every entry. + */ + readonly keepForMs: number | null; + readonly now: Date; +} + +export interface LedgerEntriesFunction { + (args: LedgerEntriesArgs): readonly ForgetLedgerEntry[]; +} + +/** + * The entries of the forgettings among `events`, but those already older than `keepFor`, which a + * prune would remove anyway. + */ +export const ledgerEntries: LedgerEntriesFunction = ({ events, keepForMs, now }) => + events + .filter(isForgetting) + .filter( + (event) => keepForMs === null || Date.parse(event.timestamp) + keepForMs > now.getTime(), + ) + .map(forgettingEntry); + export interface CreateLedgerOutboxArgs { readonly ledger: ForgetLedger; readonly keepForMs: number | null; @@ -17,10 +76,10 @@ export interface CreateLedgerOutboxFunction { /** * Follows the global stream from its first event and appends every `PersonalDataForgotten` to the - * ledger, a batch at a time: the event store is the outbox, behind the append each transaction - * makes as it commits. A batch the ledger refuses is delivered again, and so is every batch after - * a restored database rolls the checkpoint back, which the ledger takes as the entries it holds - * already. + * ledger, a batch at a time: the event store is the outbox, behind the intention each forgetting + * writes before it commits, for the forgettings whose intention a ledger that is not strict did + * not take. A batch the ledger refuses is delivered again, and so is every batch after a restored + * database rolls the checkpoint back, which the ledger takes as the entries it holds already. */ export const createLedgerOutbox: CreateLedgerOutboxFunction = ({ ledger, keepForMs, clock }) => ({ name: FORGET_LEDGER_SUBSCRIBER, diff --git a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts index a77a0555..837e5f2b 100644 --- a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts @@ -2,9 +2,14 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { Adapter } from "../../adapter/adapter.ts"; import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { Table } from "../../adapter/storage/table.ts"; +import { DEFAULT_FORGET_LEDGER_TIMEOUT_MS } from "../../config/defaults.ts"; import type { ForgetLedgerConfig } from "../../config/types.ts"; import { createFixedClock } from "../../contracts/clock.ts"; -import { ForgetLedgerUnreconciledError, ValidationError } from "../../contracts/errors.ts"; +import { + ForgetLedgerUnavailableError, + ForgetLedgerUnreconciledError, + ValidationError, +} from "../../contracts/errors.ts"; import type { StoredEvent } from "../../contracts/event.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { type Logger, silentLogger } from "../../contracts/logger.ts"; @@ -18,16 +23,13 @@ import type { Registry } from "../../modules/registry.ts"; import type { FieldsArgs } from "../../modules/view.ts"; import { createTestApp } from "../../testing/index.ts"; import { createApp } from "../app.ts"; -import { - defineForgetLedger, - FORGET_LEDGER_WAIT_MS, - type ForgetLedgerSource, -} from "./forget-ledger.ts"; +import { defineForgetLedger, type ForgetLedgerSource } from "./forget-ledger.ts"; import { HOLD_MS } from "./hold.ts"; import { FORGET_LEDGER_SUBSCRIBER } from "./ledger-outbox.ts"; import { FORGET_LEDGER_ATTEMPTS, FORGET_LEDGER_CURSOR, + FORGET_LEDGER_GRACE_MS, FORGET_LEDGER_HELD_UNTIL, FORGET_LEDGER_REREAD_MS, } from "./reconciliation.ts"; @@ -185,8 +187,12 @@ const entry = (personId: string, eventId: string, at: Date): ForgetLedgerEntry = personId, eventId, forgottenAt: at.toISOString(), + reason: "requested", }); +// A forgetting the store lost, old enough at `START` for a reconciliation to commit it again. +const LOST = new Date(START.getTime() - FORGET_LEDGER_GRACE_MS - 60_000); + const opened = async (ledgerAdapter: Adapter): Promise => ledgerAdapter.createForgetLedger({}); @@ -208,18 +214,21 @@ describe("the forget ledger", () => { (event) => event.type === "PersonalDataForgotten", ); expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual( - events.map((event) => entry(event.aggregateId, event.id, new Date(event.timestamp))), + events.map((event) => ({ + ...entry(event.aggregateId, event.id, new Date(event.timestamp)), + fingerprints: [], + })), ); expect(events.map((event) => event.aggregateId)).toEqual(["c-0", "c-1"]); }); - it("forgets again at start, with reason restored, a person the ledger names whose forgetting the store does not hold", async () => { + it("commits again at start, under its own id, time and reason, a forgetting the ledger names that the store does not hold", async () => { const storage = memory(); const ledgerAdapter = memory(); const { app: before, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await register("c-2"); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); await before.stop(); const { logs, logger } = recording(); const { app, forgotten, customer } = await start({ @@ -227,13 +236,13 @@ describe("the forget ledger", () => { forgetLedger: ledgerAdapter, logger, }); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await customer("c-1")).toBeNull(); expect(await customer("c-2")).toEqual({ customerId: "c-2", name: "name of c-2" }); expect(logs).toContainEqual([ "info", "personal data forgotten", - expect.objectContaining({ personId: "c-1", reason: "restored", hadKey: true }), + expect.objectContaining({ personId: "c-1", reason: "requested", hadKey: true }), ]); const exported = await app.exportPersonalData("c-1"); expect(exported.events.map(({ type, data }) => [type, data])).toEqual([ @@ -243,17 +252,20 @@ describe("the forget ledger", () => { expect((await app.exportPersonalData("c-2")).events[0]?.data).toEqual({ name: "name of c-2" }); expect(logs).toContainEqual([ "warn", - "forgot again persons a restored backup brought back", + "committed again forgettings the database did not hold", { forgotten: 1 }, ]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); await app.runUntilIdle(); const restored = (await app.events.read({ limit: 100 })).at(-1); - expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toContainEqual({ - personId: "c-1", - eventId: restored?.id, - forgottenAt: restored?.timestamp, + expect(restored).toMatchObject({ + id: "lost-forget", + timestamp: LOST.toISOString(), + payload: { reason: "requested", fingerprints: [expect.any(String)] }, }); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual([ + entry("c-1", "lost-forget", LOST), + ]); }); it("forgets nobody on a normal start, nor a person given a key again after the forgetting the store holds", async () => { @@ -278,7 +290,7 @@ describe("the forget ledger", () => { // A start that forgot nobody leaves the read models to the background. expect(await customer("c-2")).toBeNull(); expect(logs.map(([, message]) => message)).not.toContain( - "forgot again persons a restored backup brought back", + "committed again forgettings the database did not hold", ); expect((await app.exportPersonalData("c-1")).events.at(-1)?.data).toEqual({ name: "name of c-1", @@ -311,8 +323,12 @@ describe("the forget ledger", () => { entry("c-1", "too-old", new Date(cursor - FORGET_LEDGER_REREAD_MS - 1)), entry("c-2", "in-window", new Date(cursor - FORGET_LEDGER_REREAD_MS + 1)), ]); - const { forgotten } = await start({ storage, forgetLedger: ledgerAdapter }); - expect(await forgotten()).toEqual([["c-2", "restored"]]); + const { forgotten } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: new Date(cursor), + }); + expect(await forgotten()).toEqual([["c-2", "requested"]]); expect(await written.checkpointStore.get(FORGET_LEDGER_CURSOR)).toBe(cursor); }); @@ -321,13 +337,13 @@ describe("the forget ledger", () => { const ledgerAdapter = memory(); const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); await first.stop(); const [one] = await Promise.all([ start({ storage, forgetLedger: ledgerAdapter }), start({ storage, forgetLedger: ledgerAdapter }), ]); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); }); it("starts when the ledger cannot be read, says so, and reconciles in the worker after a while", async () => { @@ -336,7 +352,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let failures = 2; const flaky = defineForgetLedger(async () => { const inner = await opened(ledgerAdapter); @@ -369,7 +385,7 @@ describe("the forget ledger", () => { expect(await app.nextDueAt()).toEqual(new Date(START.getTime() + 3 * HOLD_MS)); clock.advance(2 * HOLD_MS); expect(await app.runUntilIdle({ maxPasses: 1 })).toMatchObject({ idle: false }); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); expect(await app.nextDueAt()).toBeNull(); }); @@ -380,7 +396,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let down = true; let reads = 0; const flaky = defineForgetLedger(async () => { @@ -419,13 +435,13 @@ describe("the forget ledger", () => { down = false; one.clock.set(new Date(heldUntil)); expect(await one.app.runUntilIdle({ maxPasses: 1 })).toMatchObject({ idle: false }); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); expect(await stored()).toEqual([0, 0]); expect(await one.app.nextDueAt()).toBeNull(); other.clock.set(new Date(heldUntil)); await other.app.runUntilIdle(); expect(await other.app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); - expect(await other.forgotten()).toEqual([["c-1", "restored"]]); + expect(await other.forgotten()).toEqual([["c-1", "requested"]]); }); it("reconciles at every start, and backs off from the failures before a restart", async () => { @@ -456,7 +472,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let answer: () => void = () => undefined; const answered = new Promise((resolve) => { answer = resolve; @@ -486,18 +502,79 @@ describe("the forget ledger", () => { }); // The start arms its timer once it reaches the ledger, which the test cannot see. while (started === undefined) { - clock.advance(FORGET_LEDGER_WAIT_MS); + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } const app = await starting; stops.push(() => app.stop()); + // Each call to the ledger is given up at the timeout, so the reconciliation fails then, and + // tries again once its hold ends. expect(logs).toContainEqual([ "error", - "forget ledger did not answer in time; the app starts and reconciles in the background", - { timeoutMs: FORGET_LEDGER_WAIT_MS }, + "forget ledger could not be reconciled; persons a restored backup brought back are not forgotten yet", + expect.objectContaining({ + errorName: "ForgetLedgerTimeoutError", + errorCode: "FORGET_LEDGER_TIMEOUT", + }), ]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: false } }); answer(); + clock.advance(HOLD_MS); + await app.runUntilIdle(); + expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); + }); + + it("starts after the timeout when the reconciliation takes longer, each call to the ledger within it", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); + for (const person of ["c-1", "c-2", "c-3"]) await register(person); + await first.stop(); + await (await opened(ledgerAdapter)).append( + ["c-1", "c-2", "c-3"].map((person) => entry(person, `lost-${person}`, LOST)), + ); + const clock = createFixedClock(START); + let reconciling = false; + // Each forgetting the reconciliation commits again takes most of the timeout. + const slow: Adapter = { + ...storage, + createStorage: async () => { + const opened = await storage.createStorage(); + return { + ...opened, + transact: async (work) => { + if (reconciling) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS / 2); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return opened.transact(work); + }, + }; + }, + }; + const watching = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + append: (entries) => inner.append(entries), + read: (args) => { + reconciling = true; + return inner.read(args); + }, + }; + }); + const { logs, logger } = recording(); + const app = await createApp({ + registry, + config: { storage: slow, runtime: { personalData: { forgetLedger: { store: watching } } } }, + clock, + logger, + }); + stops.push(() => app.stop()); + expect(logs).toContainEqual([ + "error", + "forget ledger did not answer in time; the app starts and reconciles in the background", + { timeoutMs: DEFAULT_FORGET_LEDGER_TIMEOUT_MS }, + ]); await vi.waitFor(async () => expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }), ); @@ -518,11 +595,11 @@ describe("the forget ledger", () => { await (await opened(ledgerAdapter)).append([entry("c-1", "shown-late", START)]); clock.advance(60_000); await app.runUntilIdle(); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await app.nextDueAt()).toBeNull(); }); - it("copies a forgetting from the stream once the ledger takes it, when it refused it at commit", async () => { + it("copies a forgetting from the stream once the ledger takes it, when it refused its intention", async () => { const storage = memory(); const ledgerAdapter = memory(); let refusals = 2; @@ -545,8 +622,8 @@ describe("the forget ledger", () => { expect(refusals).toBe(1); expect(logs).toContainEqual([ "warn", - "forget ledger did not take the forgetting; the stream copies it", - expect.objectContaining({ forgotten: 1, errorName: "Error" }), + "forget ledger did not take the forgetting; it commits, and the stream copies it", + expect.objectContaining({ errorName: "Error" }), ]); await app.runUntilIdle({ maxPasses: 1 }); expect(refusals).toBe(0); @@ -562,7 +639,7 @@ describe("the forget ledger", () => { ).toEqual(["c-1"]); }); - it("appends a forgetting to the ledger before forget resolves, and the stream copies it once", async () => { + it("records a forgetting in the ledger before it commits, and the stream copies it once", async () => { const storage = memory(); const ledgerAdapter = memory(); const { app, register } = await start({ storage, forgetLedger: ledgerAdapter }); @@ -571,7 +648,9 @@ describe("the forget ledger", () => { const [forgetting] = (await app.events.read({ limit: 100 })).filter( (event) => event.type === "PersonalDataForgotten", ); - const appended = [entry("c-1", forgetting?.id ?? "", START)]; + const appended = [ + { ...entry("c-1", forgetting?.id ?? "", START), fingerprints: [expect.any(String)] }, + ]; expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual(appended); await app.runUntilIdle(); expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual(appended); @@ -583,13 +662,13 @@ describe("the forget ledger", () => { const { app, register, forgotten } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await app.forgetAgain([{ personId: "c-1" }]); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect( (await (await opened(ledgerAdapter)).read({ limit: 10 })).map(({ personId }) => personId), ).toEqual(["c-1"]); }); - it("appends what the retention forgets as it commits, before the stream copies it", async () => { + it("records what the retention forgets before it commits, and the stream copies it", async () => { const ledgerAdapter = memory(); const appends: string[] = []; const watching = defineForgetLedger(async () => { @@ -625,11 +704,11 @@ describe("the forget ledger", () => { (event) => event.type === "PersonalDataForgotten", ), ).toHaveLength(1); - // Once as the forgetting committed, once more from the stream. + // Once as the forgetting's intention, once more from the stream. expect(appends).toEqual(["c-1", "c-1"]); }); - it("resolves forget once the ledger has not taken the forgetting in time, and says so", async () => { + it("commits a forgetting once the ledger has not taken its intention in time, says so, and waits for it no more while it hangs", async () => { const storage = memory(); const ledgerAdapter = memory(); const hanging = defineForgetLedger(async () => { @@ -645,17 +724,25 @@ describe("the forget ledger", () => { const forgetting = app.forget("c-1").then(() => { done = true; }); - // The append arms its timer once the forgetting commits, which the test cannot see. + // The append arms its timer once it is called, which the test cannot see. while (!done) { - clock.advance(FORGET_LEDGER_WAIT_MS); + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } await forgetting; expect(logs).toContainEqual([ "warn", - "forget ledger did not take the forgetting in time; the stream copies it", - { forgotten: 1, timeoutMs: FORGET_LEDGER_WAIT_MS }, + "forget ledger did not take the forgetting; it commits, and the stream copies it", + expect.objectContaining({ + errorName: "ForgetLedgerTimeoutError", + errorCode: "FORGET_LEDGER_TIMEOUT", + }), ]); + await app.forget("c-2"); + const forgotten = (await app.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ); + expect(forgotten.map((event) => event.aggregateId)).toEqual(["c-1", "c-2"]); }); it("reconciles again every reconcileEvery while the app runs, one instance a period", async () => { @@ -678,7 +765,7 @@ describe("the forget ledger", () => { await one.register("c-1"); expect(reads).toBe(2); // As if the database had been restored, while the app runs, to before c-1 was forgotten. - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); expect(await one.app.nextDueAt()).toBeNull(); one.clock.advance(59_999); other.clock.advance(59_999); @@ -689,7 +776,7 @@ describe("the forget ledger", () => { await one.app.runUntilIdle(); await other.app.runUntilIdle(); expect(reads).toBe(3); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); expect(await one.customer("c-1")).toBeNull(); other.clock.advance(60_000); await other.app.runUntilIdle(); @@ -799,7 +886,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); const down = new Error("ledger is down"); let closed = 0; const broken = defineForgetLedger(async () => ({ @@ -833,26 +920,44 @@ describe("the forget ledger", () => { }; }); const clock = createFixedClock(START); - let started = false; + let settled = false; const starting = createApp({ registry, config: { storage, - runtime: { personalData: { forgetLedger: { store: slow, strict: true } } }, + runtime: { + personalData: { forgetLedger: { store: slow, strict: true, timeout: "1m" } }, + }, }, clock, - }).then((app) => { - started = true; - stops.push(() => app.stop()); - return app; + }).finally(() => { + settled = true; }); - for (let round = 0; round < 3; round += 1) { - clock.advance(FORGET_LEDGER_WAIT_MS); + void starting.catch(() => undefined); + // Waits past the default timeout, then gives the read up at its own. + for (let round = 0; round < 5; round += 1) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } - expect(started).toBe(false); + expect(settled).toBe(false); + while (!settled) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await expect(starting).rejects.toMatchObject({ + code: "FORGET_LEDGER_UNRECONCILED", + cause: { code: "FORGET_LEDGER_TIMEOUT", operation: "read", timeoutMs: 60_000 }, + }); answer(); - const app = await starting; + const app = await createApp({ + registry, + config: { + storage, + runtime: { personalData: { forgetLedger: { store: slow, strict: true } } }, + }, + clock, + }); + stops.push(() => app.stop()); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); const events = await app.events.read({ limit: 100 }); expect(events.filter((event) => event.type === "PersonalDataForgotten")).toHaveLength(1); @@ -907,6 +1012,283 @@ describe("the forget ledger", () => { }); }); +// The memory adapter whose storage transactions fail while `failing` says so, as a process that +// stops between the intention and the commit leaves the store. +const crashing = (storage: Adapter) => { + const state = { failing: false }; + const adapter: Adapter = { + ...storage, + createStorage: async () => { + const opened = await storage.createStorage(); + return { + ...opened, + transact: async (work) => { + if (state.failing) throw new Error("the process stopped"); + return opened.transact(work); + }, + }; + }, + }; + return { adapter, state }; +}; + +describe("the intention a forgetting records first", () => { + it("is committed by the next start when the process stops before the commit, under its id, time and reason", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app, register } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + await register("c-1"); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + const [intention] = await (await opened(ledgerAdapter)).read({ limit: 10 }); + expect(intention).toEqual({ + personId: "c-1", + eventId: expect.any(String), + forgottenAt: START.toISOString(), + reason: "requested", + fingerprints: [expect.any(String)], + }); + expect(await app.exportPersonalData("c-1")).toMatchObject({ + events: [{ data: { name: "name of c-1" } }], + }); + await app.stop(); + state.failing = false; + const later = new Date(START.getTime() + FORGET_LEDGER_GRACE_MS); + const { app: next, customer } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: later, + }); + const forgetting = (await next.events.read({ limit: 100 })).at(-1); + expect(forgetting).toMatchObject({ + id: intention?.eventId, + timestamp: START.toISOString(), + payload: { personId: "c-1", reason: "requested", fingerprints: intention?.fingerprints }, + }); + expect(await customer("c-1")).toBeNull(); + }); + + it("waits for its grace before a reconciliation commits it, and is due once that ends", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app, register } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + await register("c-1"); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + await app.stop(); + state.failing = false; + const { + app: next, + clock, + forgotten, + } = await start({ + storage, + forgetLedger: ledgerAdapter, + }); + expect(await forgotten()).toEqual([]); + expect(await next.nextDueAt()).toEqual(new Date(START.getTime() + FORGET_LEDGER_GRACE_MS)); + clock.advance(FORGET_LEDGER_GRACE_MS); + await next.runUntilIdle(); + expect(await forgotten()).toEqual([["c-1", "requested"]]); + expect(await next.nextDueAt()).toBeNull(); + }); + + it("leaves a key given to the person after it, when its commit never happened", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toMatchObject([ + { personId: "c-1", fingerprints: [] }, + ]); + state.failing = false; + await app.stop(); + const later = new Date(START.getTime() + FORGET_LEDGER_GRACE_MS); + const { register } = await start({ storage, forgetLedger: ledgerAdapter, now: later }); + await register("c-1"); + const { forgotten, customer } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: later, + }); + expect(await forgotten()).toEqual([]); + expect(await customer("c-1")).toEqual({ customerId: "c-1", name: "name of c-1" }); + }); + + it("forgets nobody, when strict, while the ledger does not take it, so calling again is safe", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + let down = true; + const flaky = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + append: async (entries) => { + if (down) throw new Error("ledger is down"); + await inner.append(entries); + }, + }; + }); + const { app, register, forgotten, customer } = await start({ + storage, + forgetLedger: flaky, + options: { strict: true }, + }); + await register("c-1"); + const refused = app.forget("c-1"); + await expect(refused).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + await expect(refused).rejects.toMatchObject({ + code: "FORGET_LEDGER_UNAVAILABLE", + cause: new Error("ledger is down"), + }); + expect(await forgotten()).toEqual([]); + expect(await customer("c-1")).toEqual({ customerId: "c-1", name: "name of c-1" }); + expect(await app.forgetAgain([{ personId: "c-1" }]).catch((error: unknown) => error)).toEqual( + expect.any(ForgetLedgerUnavailableError), + ); + expect(await forgotten()).toEqual([]); + down = false; + await app.forget("c-1"); + expect(await forgotten()).toEqual([["c-1", "requested"]]); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toHaveLength(1); + }); + + it("forgets nobody, when strict, once the ledger has not taken it within its timeout", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const hanging = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + append: () => new Promise(() => undefined), + }; + }); + const { app, clock, register, forgotten } = await start({ + storage, + forgetLedger: hanging, + options: { strict: true, timeout: "2s" }, + }); + await register("c-1"); + let outcome: unknown; + const forgetting = app.forget("c-1").then( + () => "forgotten", + (error: unknown) => error, + ); + void forgetting.then((settled) => { + outcome = settled; + }); + while (outcome === undefined) { + clock.advance(1_000); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(outcome).toBeInstanceOf(ForgetLedgerUnavailableError); + expect(outcome).toMatchObject({ + cause: { code: "FORGET_LEDGER_TIMEOUT", operation: "append", timeoutMs: 2_000 }, + }); + expect(await forgotten()).toEqual([]); + }); + + it("keeps a person past the retention, when strict, while the ledger does not take it", async () => { + let down = true; + const ledgerAdapter = memory(); + const flaky = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + append: async (entries) => { + if (down) throw new Error("ledger is down"); + await inner.append(entries); + }, + }; + }); + const clock = createFixedClock(START); + const { logs, logger } = recording(); + const app = await createApp({ + registry, + config: { + storage: memory(), + runtime: { + personalData: { retention: "1h", forgetLedger: { store: flaky, strict: true } }, + }, + }, + clock, + logger, + }); + stops.push(() => app.stop()); + const commands = app.commands as unknown as Record< + string, + (payload: { customerId: string; name: string }) => Promise + >; + await commands.registerCustomer?.({ customerId: "c-1", name: "Ada" }); + await app.runUntilIdle(); + clock.advance(3_600_000); + await app.runUntilIdle(); + const forgettings = async () => + (await app.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ); + expect(await forgettings()).toEqual([]); + expect(logs).toContainEqual([ + "error", + "person past the retention could not be forgotten", + expect.objectContaining({ personId: "c-1", errorCode: "FORGET_LEDGER_UNAVAILABLE" }), + ]); + down = false; + clock.advance(HOLD_MS); + await app.runUntilIdle(); + expect((await forgettings()).map((event) => event.payload)).toMatchObject([ + { personId: "c-1", reason: "retention" }, + ]); + }); + + it("commits again a forgetting the retention made only while the retention would still forget the person", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const config = (retention: "1h" | "2h") => ({ + storage, + runtime: { personalData: { retention, forgetLedger: { store: ledgerAdapter } } }, + }); + const clock = createFixedClock(START); + const first = await createApp({ registry, config: config("1h"), clock }); + const commands = first.commands as unknown as Record< + string, + (payload: { customerId: string; name: string }) => Promise + >; + await commands.registerCustomer?.({ customerId: "c-1", name: "Ada" }); + await commands.registerCustomer?.({ customerId: "c-2", name: "Bob" }); + await first.runUntilIdle(); + await first.stop(); + const retained = (person: string): ForgetLedgerEntry => ({ + ...entry(person, `retained-${person}`, LOST), + reason: "retention", + }); + await (await opened(ledgerAdapter)).append([retained("c-1"), retained("c-2")]); + clock.advance(90 * 60_000); + const later = await createApp({ registry, config: config("2h"), clock }); + stops.push(() => later.stop()); + expect( + (await later.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ), + ).toEqual([]); + await later.stop(); + const due = await createApp({ registry, config: config("1h"), clock }); + stops.push(() => due.stop()); + expect( + (await due.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => [event.id, (event.payload as PersonalDataForgottenPayload).reason]), + ).toEqual([ + ["retained-c-1", "retention"], + ["retained-c-2", "retention"], + ]); + }); +}); + describe("forgetAgain", () => { it("forgets again whom an entry names unless the store holds its forgetting or they hold no key", async () => { const storage = memory(); @@ -937,8 +1319,8 @@ describe("forgetAgain", () => { expect(logs.filter(([, message]) => message === "personal data forgotten")).toHaveLength(3); expect(await forgotten()).toEqual([ ["c-4", "requested"], - ["c-1", "restored"], - ["c-2", "restored"], + ["c-1", "requested"], + ["c-2", "requested"], ]); }); @@ -958,7 +1340,7 @@ describe("forgetAgain", () => { expect(logs).toContainEqual([ "info", "personal data forgotten", - expect.objectContaining({ personId: "c-1", reason: "restored", actor: "operator-1" }), + expect.objectContaining({ personId: "c-1", reason: "requested", actor: "operator-1" }), ]); }); diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index c37298a8..d5ec8c65 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -7,20 +7,25 @@ import type { ReadForgetLedgerArgs, } from "../../adapter/storage/forget-ledger.ts"; import { createFixedClock } from "../../contracts/clock.ts"; -import { ValidationError } from "../../contracts/errors.ts"; +import { ForgetLedgerUnavailableError, ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { type LogFields, type Logger, silentLogger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; +import { withoutLedger } from "../test-support.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; import { HOLD_MS } from "./hold.ts"; +import { createForgetIntentions } from "./intention.ts"; import { createForgetLedgerReconciliation, FORGET_LEDGER_ATTEMPTS, FORGET_LEDGER_CURSOR, + FORGET_LEDGER_GRACE_MS, FORGET_LEDGER_HELD_UNTIL, FORGET_LEDGER_LAST_RUN, FORGET_LEDGER_REREAD_MS, type ForgetAgainEntry, forgetAgain, + NEVER_DUE, } from "./reconciliation.ts"; // Each error line as its message, marked when it carries no error. @@ -79,6 +84,8 @@ const setUp = async ({ await keyed(storage, persons); const { ledger, reads } = recordingLedger(entries, visibilityDelayMs); const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ ids: createSequentialIdGenerator(), clock: createFixedClock(NOW) }), + retentionDue: NEVER_DUE, ledger, storage, ids: createSequentialIdGenerator(), @@ -96,9 +103,12 @@ const entry = (personId: string, eventId: string, forgottenAt: string): ForgetLe personId, eventId, forgottenAt, + reason: "requested", }); -const at = (minute: number): string => new Date(NOW.getTime() + minute * 60_000).toISOString(); +// Minutes into the hour before `NOW`: every entry is past the grace a reconciliation gives. +const at = (minute: number): string => + new Date(NOW.getTime() - 60 * 60_000 + minute * 60_000).toISOString(); describe("the forget ledger's reconciliation", () => { it("reads every page of the ledger, and stops at the first short one", async () => { @@ -137,7 +147,7 @@ describe("the forget ledger's reconciliation", () => { }); await storage.checkpointStore.set( FORGET_LEDGER_CURSOR, - NOW.getTime() + FORGET_LEDGER_REREAD_MS, + Date.parse(at(0)) + FORGET_LEDGER_REREAD_MS, ); expect(await reconciliation.reconcile(NOW)).toBe(1); }); @@ -198,6 +208,11 @@ describe("the forget ledger's reconciliation", () => { const inner = await memory().createForgetLedger({}); await inner.append([entry("c-1", "e-1", at(0))]); const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { append: async () => {}, read: async (args) => { @@ -242,6 +257,11 @@ describe("the forget ledger's reconciliation", () => { const errors: unknown[] = []; let reads = 0; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { append: async () => {}, read: async () => { @@ -290,6 +310,11 @@ describe("the forget ledger's reconciliation", () => { const errors: unknown[] = []; const create = () => createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([]).ledger, storage: { ...storage, checkpointStore: checkpoints }, ids: createSequentialIdGenerator(), @@ -316,6 +341,11 @@ describe("the forget ledger's reconciliation", () => { const storage = await memory().createStorage(); const asked: string[] = []; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([], 1).ledger, storage: { ...storage, @@ -349,6 +379,11 @@ describe("the forget ledger's reconciliation", () => { await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 60_000); const told: string[] = []; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([entry("c-1", "e-1", at(0))]).ledger, storage, ids: createSequentialIdGenerator(), @@ -395,6 +430,11 @@ describe("the forget ledger's periodic reconciliation", () => { const { ledger, reads } = recordingLedger([]); let left = failures; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { ...ledger, read: async (args) => { @@ -568,15 +608,131 @@ describe("the forget ledger's periodic reconciliation", () => { }); }); +describe("what the reconciliation commits again", () => { + const named = (fingerprints: readonly string[] | undefined, minute = 0): ForgetLedgerEntry => ({ + ...entry("c-1", `e-${String(fingerprints)}`, at(minute)), + ...(fingerprints === undefined ? {} : { fingerprints }), + }); + + it("commits a forgetting while the person holds a key it names, or any key when it names none recorded", async () => { + for (const [fingerprints, forgotten] of [ + [["f-c-1"], 1], + [["f-other", "f-c-1"], 1], + [undefined, 1], + [["f-other"], 0], + [[], 0], + ] as const) { + const { reconciliation, storage } = await setUp({ + entries: [named(fingerprints)], + persons: ["c-1"], + }); + expect(await reconciliation.reconcile(NOW)).toBe(forgotten); + expect((await storage.personalKeys.get("c-1")) === null).toBe(forgotten === 1); + } + }); + + it("commits it under its own id, time and reason, naming the key it destroyed", async () => { + const retained = named(["f-c-1"], 7); + const { reconciliation, storage } = await setUp({ entries: [retained], persons: ["c-1"] }); + await reconciliation.reconcile(NOW); + expect(await storage.eventStore.readAll({ afterPosition: 0, limit: 10 })).toMatchObject([ + { + id: retained.eventId, + timestamp: retained.forgottenAt, + aggregateType: "bounda", + aggregateId: "c-1", + person: "c-1", + payload: { personId: "c-1", reason: "requested", fingerprints: ["f-c-1"] }, + }, + ]); + }); + + it("commits a forgetting the retention made only while the retention would forget the person now", async () => { + const due: string[] = []; + for (const [answer, forgotten] of [ + [false, 0], + [true, 1], + ] as const) { + const storage = await memory().createStorage(); + await keyed(storage, ["c-1"]); + const clock = createFixedClock(NOW); + const reconciliation = createForgetLedgerReconciliation({ + ledger: recordingLedger([{ ...named(["f-c-1"]), reason: "retention" }]).ledger, + intentions: withoutLedger({ ids: createSequentialIdGenerator(), clock }), + retentionDue: async (_, personId, now) => { + due.push(`${personId} ${now.toISOString()}`); + return answer; + }, + storage, + ids: createSequentialIdGenerator(), + clock, + logger: silentLogger, + concurrencyRetries: 3, + reconcileEveryMs: EVERY, + keepForMs: null, + }); + expect(await reconciliation.reconcile(NOW)).toBe(forgotten); + } + // Once before the transaction, and once more in it for the one it forgot. + expect(due).toEqual([ + `c-1 ${NOW.toISOString()}`, + `c-1 ${NOW.toISOString()}`, + `c-1 ${NOW.toISOString()}`, + ]); + }); + + it("leaves an entry younger than its grace for a reconciliation due once the youngest comes of age", async () => { + const young = new Date(NOW.getTime() - FORGET_LEDGER_GRACE_MS + 1).toISOString(); + const younger = new Date(NOW.getTime() - FORGET_LEDGER_GRACE_MS + 2).toISOString(); + const { reconciliation, storage } = await setUp({ + entries: [ + entry("c-1", "e-1", at(0)), + entry("c-2", "e-2", young), + entry("c-3", "e-3", younger), + ], + persons: ["c-1", "c-2", "c-3"], + }); + expect(await reconciliation.reconcile(NOW)).toBe(1); + expect(await storage.personalKeys.get("c-2")).not.toBeNull(); + expect(reconciliation.nextDueAt()).toEqual( + new Date(Date.parse(younger) + FORGET_LEDGER_GRACE_MS), + ); + expect(await storage.checkpointStore.get(FORGET_LEDGER_CURSOR)).toBe(Date.parse(younger)); + const due = new Date(Date.parse(younger) + FORGET_LEDGER_GRACE_MS); + expect(await reconciliation.sweep(due)).toBe(2); + expect(reconciliation.nextDueAt()).toBeNull(); + }); + + it("is due at the sooner of a ledger's delay and an entry's grace", async () => { + const young = new Date(NOW.getTime() - 1).toISOString(); + const late = await setUp({ entries: [entry("c-1", "e-1", young)], visibilityDelayMs: 1_000 }); + await late.reconciliation.reconcile(NOW); + expect(late.reconciliation.nextDueAt()).toEqual(new Date(NOW.getTime() + 1_000)); + const slow = await setUp({ + entries: [entry("c-1", "e-1", young)], + visibilityDelayMs: 10 * FORGET_LEDGER_GRACE_MS, + }); + await slow.reconciliation.reconcile(NOW); + expect(slow.reconciliation.nextDueAt()).toEqual( + new Date(Date.parse(young) + FORGET_LEDGER_GRACE_MS), + ); + }); +}); + describe("forgetAgain", () => { it("refuses an entry without a person's id", async () => { const storage = await memory().createStorage(); + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); const refused = forgetAgain({ storage, - ids: createSequentialIdGenerator(), - clock: createFixedClock(NOW), + intentions: withoutLedger({ ids, clock }), + ids, + clock, concurrencyRetries: 3, entry: { personId: "" }, + retentionDue: NEVER_DUE, + record: true, }); await expect(refused).rejects.toBeInstanceOf(ValidationError); await expect(refused).rejects.toThrow("forgetting again needs the id of the person"); @@ -592,13 +748,18 @@ describe("forgetAgain", () => { return storage.transact(work); }, }; + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); const run = (entry: ForgetAgainEntry) => forgetAgain({ storage: counted, - ids: createSequentialIdGenerator(), - clock: createFixedClock(NOW), + intentions: withoutLedger({ ids, clock }), + ids, + clock, concurrencyRetries: 3, entry, + retentionDue: NEVER_DUE, + record: true, }); expect(await run({ personId: "c-1", eventId: "e-1" })).toBeNull(); expect(transactions).toBe(0); @@ -606,4 +767,99 @@ describe("forgetAgain", () => { expect(await run({ personId: "c-1", eventId: "e-1" })).not.toBeNull(); expect(transactions).toBe(1); }); + + const ledgered = async () => { + const storage = await memory().createStorage(); + const recorded: ForgetLedgerEntry[] = []; + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); + let down = false; + const intentions = createForgetIntentions({ + ledger: boundForgetLedger({ + ledger: { + append: async (entries) => { + if (down) throw new Error("ledger is down"); + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + ids, + clock, + logger: silentLogger, + }); + const run = (entry: ForgetAgainEntry) => + forgetAgain({ + storage, + intentions, + ids, + clock, + concurrencyRetries: 3, + entry, + retentionDue: NEVER_DUE, + record: true, + }); + return { + storage, + recorded, + run, + fail: () => { + down = true; + }, + }; + }; + + it("records a whole entry from elsewhere in the ledger as it is, then commits it under its id", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1", "c-2"]); + const whole: ForgetLedgerEntry = { ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] }; + const unrecorded = entry("c-2", "e-2", at(4)); + expect(await run(whole)).toMatchObject({ event: { id: "e-1", timestamp: at(3) } }); + expect(await run(unrecorded)).toMatchObject({ event: { id: "e-2" } }); + expect(recorded).toEqual([whole, unrecorded]); + expect(recorded[1]).not.toHaveProperty("fingerprints"); + }); + + it("takes an intention of its own for a bare id, naming the key the person holds, and none for one who holds none", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1"]); + expect(await run({ personId: "c-2" })).toBeNull(); + expect(await run({ personId: "c-1", eventId: "elsewhere" })).toMatchObject({ + event: { id: "id-1", payload: { reason: "requested", fingerprints: ["f-c-1"] } }, + }); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "requested", + fingerprints: ["f-c-1"], + }, + ]); + }); + + it("forgets nobody, when strict, while the ledger does not take the entry", async () => { + const { storage, run, fail } = await ledgered(); + await keyed(storage, ["c-1"]); + fail(); + await expect(run({ ...entry("c-1", "e-1", at(3)) })).rejects.toBeInstanceOf( + ForgetLedgerUnavailableError, + ); + await expect(run({ personId: "c-1" })).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + expect(await storage.eventStore.lastPosition()).toBe(0); + }); + + it("leaves a key the entry does not name, and a person a store holds the forgetting of", async () => { + const { storage, run } = await ledgered(); + await keyed(storage, ["c-1"]); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-other"] })).toBeNull(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).not.toBeNull(); + await keyed(storage, ["c-1"]); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).toBeNull(); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); }); diff --git a/packages/core/src/kernel/personal-data/reconciliation.ts b/packages/core/src/kernel/personal-data/reconciliation.ts index 4d98c4ec..9a2a0473 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.ts @@ -1,4 +1,4 @@ -import type { Storage } from "../../adapter/adapter.ts"; +import type { Storage, StorageTransaction } from "../../adapter/adapter.ts"; import type { ForgetLedger, ForgetLedgerEntry, @@ -8,10 +8,11 @@ import { type Hold, isReleasedBy } from "../../adapter/storage/hold.ts"; import type { Clock } from "../../contracts/clock.ts"; import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; -import { BOUNDA_AGGREGATE } from "../../modules/bounda-events.ts"; +import { BOUNDA_AGGREGATE, type PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; import { startedBy } from "../shared/causation.ts"; -import { type Forgotten, forgetWithin, logForgotten } from "./forget.ts"; +import { type Forgotten, forgetWithin, logForgotten, STALE_INTENTION } from "./forget.ts"; import { holdAfter } from "./hold.ts"; +import type { ForgetIntention, ForgetIntentions } from "./intention.ts"; import { checkPersonId, transactRetrying } from "./transact.ts"; /** @@ -45,21 +46,61 @@ export const FORGET_LEDGER_LAST_RUN = "forget-ledger:last-run"; // KV does, lists again for every page. export const FORGET_LEDGER_PAGE = 1_000; +/** + * How old a forget ledger's entry must be before a reconciliation commits it again: a younger one + * may be the intention of a forgetting still committing. A reconciliation that leaves one is due + * again once it is that old. + */ +export const FORGET_LEDGER_GRACE_MS: number = 60_000; + +/** + * A forgetting to commit again: a ledger's entry, or a line of `bounda forgotten`, or a bare + * person's id. + */ export interface ForgetAgainEntry { readonly personId: string; /** * The `PersonalDataForgotten` the person was forgotten by: a store that holds it has applied the - * forgetting already, and a key the person holds now was given to them after. + * forgetting already. With `forgottenAt` and `reason`, it is committed again under the same id, + * time and reason. */ readonly eventId?: string; + readonly forgottenAt?: string; + readonly reason?: PersonalDataForgottenReason; + /** + * The keys the forgetting named: a key the person holds that is not here was given to them + * after, and is kept. Any key counts when absent. + */ + readonly fingerprints?: readonly string[]; } +/** + * Whether the retention would forget the person now, read on the stores given: a forgetting the + * retention made is committed again only then. + */ +export interface RetentionDueFunction { + ( + stores: Pick, + personId: string, + now: Date, + ): Promise; +} + +export const NEVER_DUE: RetentionDueFunction = async () => false; + export interface ForgetAgainArgs { - readonly storage: Pick; + readonly storage: Pick; + readonly intentions: ForgetIntentions; readonly ids: IdGenerator; readonly clock: Clock; readonly concurrencyRetries: number; readonly entry: ForgetAgainEntry; + readonly retentionDue: RetentionDueFunction; + /** + * Whether the entry is written to the ledger first, as it is when it comes from elsewhere than + * the ledger. + */ + readonly record: boolean; readonly actor?: string | undefined; } @@ -67,23 +108,31 @@ export interface ForgetAgainFunction { (args: ForgetAgainArgs): Promise; } +type DueStores = Pick; + /** - * Forgets the person again, with `reason: "restored"`, when the store does not hold the event the - * entry names and the person still holds a key, deciding both again in the transaction that - * forgets them, so of two instances doing it at once, the one whose append conflicts finds the key - * gone on its retry; an entry that reads as applied takes no transaction. Resolves `null` when it - * forgot nobody. `storage` is the store as written: nothing here reads a personal field. + * Commits the forgetting again when the store does not hold the event the entry names, the person + * still holds a key it names, and, for one the retention made, the retention would forget them + * now; deciding all of it again in the transaction that forgets them, so of two instances doing + * it at once, the one whose append conflicts finds it done on its retry. An entry that reads as + * applied takes no transaction. A whole entry is committed under its own id, time and reason; any + * other takes an intention of its own, as a request. Resolves `null` when it forgot nobody. + * `storage` is the store as written: nothing here reads a personal field. */ export const forgetAgain: ForgetAgainFunction = async ({ storage, + intentions, ids, clock, concurrencyRetries, - entry: { personId, eventId }, + entry, + retentionDue, + record, actor, }) => { + const { personId, eventId, forgottenAt, reason, fingerprints } = entry; checkPersonId(personId, "forgetting again"); - const isDue = async (stores: Pick): Promise => { + const isDue = async (stores: DueStores, named: readonly string[] | undefined) => { if (eventId !== undefined) { const applied = await stores.eventStore.readOne({ aggregateType: BOUNDA_AGGREGATE, @@ -92,23 +141,42 @@ export const forgetAgain: ForgetAgainFunction = async ({ }); if (applied !== null) return false; } - return (await stores.personalKeys.get(personId)) !== null; + const key = await stores.personalKeys.get(personId); + if (key === null || (named !== undefined && !named.includes(key.fingerprint))) return false; + return reason !== "retention" || retentionDue(stores, personId, clock.now()); }; - if (!(await isDue(storage))) return null; + if (!(await isDue(storage, fingerprints))) return null; + let intention: ForgetIntention; + if (eventId !== undefined && forgottenAt !== undefined && reason !== undefined) { + const recorded = { personId, eventId, forgottenAt, reason }; + if (record) { + await intentions.record( + fingerprints === undefined ? recorded : { ...recorded, fingerprints }, + ); + } + intention = { eventId, forgottenAt, reason, fingerprints }; + } else { + intention = await intentions.intend({ + personId, + reason: reason ?? "requested", + keys: storage.personalKeys, + }); + } const correlationId = ids.next(); return transactRetrying({ storage, concurrencyRetries, work: async (transaction) => { - if (!(await isDue(transaction))) return null; - return forgetWithin({ + if (!(await isDue(transaction, intention.fingerprints ?? fingerprints))) return null; + const outcome = await forgetWithin({ transaction, ids, clock, personId, - reason: "restored", + intention, context: startedBy(correlationId, actor), }); + return outcome === STALE_INTENTION || "held" in outcome ? null : outcome; }, }); }; @@ -119,9 +187,10 @@ export interface ReconciliationFailed { export interface ForgetLedgerReconciliation { /** - * Reads the ledger past the database's cursor and forgets again whom `forgetAgain` finds; then - * moves the cursor. Resolves to how many persons it forgot, or to why it could not read the - * ledger through; never rejects: a failure is logged and the worker tries again later. + * Reads the ledger past the database's cursor and commits again what `forgetAgain` finds due, + * leaving entries younger than `FORGET_LEDGER_GRACE_MS` to a reconciliation due once they are + * not; then moves the cursor. Resolves to how many persons it forgot, or to why it could not read + * the ledger through; never rejects: a failure is logged and the worker tries again later. */ reconcile(now: Date): Promise; /** @@ -132,8 +201,9 @@ export interface ForgetLedgerReconciliation { */ sweep(now: Date): Promise; /** - * When the next reconciliation after a failure, or for a ledger's delay, is due; `null` when - * none is. The periodic one is left out: a host that only wakes for it would never sleep. + * When the next reconciliation after a failure, for a ledger's delay or for an entry left for + * being young, is due; `null` when none is. The periodic one is left out: a host that only wakes + * for it would never sleep. */ nextDueAt(): Date | null; /** @@ -145,10 +215,12 @@ export interface ForgetLedgerReconciliation { export interface CreateForgetLedgerReconciliationArgs { readonly ledger: ForgetLedger; + readonly intentions: ForgetIntentions; + readonly retentionDue: RetentionDueFunction; /** * The store as written. */ - readonly storage: Pick; + readonly storage: ForgetAgainArgs["storage"] & Pick; readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; @@ -184,6 +256,8 @@ export interface CreateForgetLedgerReconciliationFunction { */ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationFunction = ({ ledger, + intentions, + retentionDue, storage, ids, clock, @@ -210,30 +284,52 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF } }; - const run = async (): Promise => { + // Resolves how many it forgot, and when the youngest entry it left comes of age; the cursor + // moves past that entry, which the next read takes again, since the grace is shorter than the + // stretch read again. + const run = async (at: number): Promise<{ forgotten: number; young: number | null }> => { const cursor = await checkpointStore.get(FORGET_LEDGER_CURSOR); let after: ForgetLedgerPosition | undefined = cursor === 0 ? undefined : { forgottenAt: new Date(cursor - FORGET_LEDGER_REREAD_MS).toISOString(), eventId: "" }; let forgotten = 0; + let youngest: number | null = null; + const ended = () => ({ + forgotten, + young: youngest === null ? null : youngest + FORGET_LEDGER_GRACE_MS, + }); for (;;) { const page: readonly ForgetLedgerEntry[] = await ledger.read({ ...(after === undefined ? {} : { after }), limit: pageSize, }); for (const entry of page) { - const again = await forgetAgain({ storage, ids, clock, concurrencyRetries, entry }); + const time = Date.parse(entry.forgottenAt); + if (time + FORGET_LEDGER_GRACE_MS > at) { + youngest = Math.max(youngest ?? time, time); + continue; + } + const again = await forgetAgain({ + storage, + intentions, + ids, + clock, + concurrencyRetries, + entry, + retentionDue, + record: false, + }); if (again === null) continue; logForgotten(logger, again); onForgotten?.(entry.personId); forgotten += 1; } const last = page.at(-1); - if (last === undefined) return forgotten; + if (last === undefined) return ended(); const time = Date.parse(last.forgottenAt); if (Number.isFinite(time)) await advance(FORGET_LEDGER_CURSOR, time); - if (page.length < pageSize) return forgotten; + if (page.length < pageSize) return ended(); after = last; } }; @@ -275,8 +371,9 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF const reconcile = async (now: Date): Promise => { const at = now.getTime(); let forgotten: number; + let young: number | null; try { - forgotten = await run(); + ({ forgotten, young } = await run(at)); } catch (error) { done = false; logger.error( @@ -288,10 +385,12 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF ); return { failed: error }; } - if (forgotten > 0) - logger.warn("forgot again persons a restored backup brought back", { forgotten }); + if (forgotten > 0) { + logger.warn("committed again forgettings the database did not hold", { forgotten }); + } done = true; - dueAt = settled ? null : at + (ledger.visibilityDelayMs ?? 0); + const delayed = settled ? null : at + (ledger.visibilityDelayMs ?? 0); + dueAt = delayed === null || young === null ? (delayed ?? young) : Math.min(delayed, young); settled = true; periodicAt = at + reconcileEveryMs; await release().catch((error: unknown) => diff --git a/packages/core/src/kernel/personal-data/retention-app.test.ts b/packages/core/src/kernel/personal-data/retention-app.test.ts index 168555bf..8059135f 100644 --- a/packages/core/src/kernel/personal-data/retention-app.test.ts +++ b/packages/core/src/kernel/personal-data/retention-app.test.ts @@ -282,7 +282,13 @@ describe("forget in a reaction", () => { expect(await forgotten()).toEqual([]); await advance(2); expect(await forgotten()).toEqual([ - { personId: "c-1", reason: "requested", cancelledCommands: [], removedDeadLetters: [] }, + { + personId: "c-1", + reason: "requested", + fingerprints: [expect.any(String)], + cancelledCommands: [], + removedDeadLetters: [], + }, ]); expect(await storage.personalKeys.get("c-1")).toBeNull(); expect(await storage.personalKeys.get("c-2")).not.toBeNull(); diff --git a/packages/core/src/kernel/personal-data/retention.test.ts b/packages/core/src/kernel/personal-data/retention.test.ts index 2e17f794..aca5a827 100644 --- a/packages/core/src/kernel/personal-data/retention.test.ts +++ b/packages/core/src/kernel/personal-data/retention.test.ts @@ -1,21 +1,27 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { StoredPerson } from "../../adapter/storage/person-store.ts"; -import { createFixedClock } from "../../contracts/clock.ts"; +import { testPersonalKey } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock, type FixedClock } from "../../contracts/clock.ts"; import type { StoredEvent } from "../../contracts/event.ts"; -import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import { createSequentialIdGenerator, type IdGenerator } from "../../contracts/ids.ts"; +import { silentLogger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; import { createKeptPersonStore } from "../../memory/person-store.ts"; import { tracer } from "../telemetry.ts"; -import { createRecordingLogger } from "../test-support.ts"; +import { createRecordingLogger, withoutLedger } from "../test-support.ts"; import { forgetPersonalData } from "./forget.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; import { HOLD_MAX_MS, HOLD_MS } from "./hold.ts"; +import { createForgetIntentions } from "./intention.ts"; import { restrictProcessing } from "./restriction.ts"; import { createRetentionSubscriber, createRetentionSweep, RETENTION_SUBSCRIBER, RETENTION_SWEEP_LIMIT, + retentionDueOf, } from "./retention.ts"; const DAY = 86_400_000; @@ -124,6 +130,7 @@ const setUp = async (storage?: Storage) => { const sweepOver = (storage: Storage) => createRetentionSweep({ storage, + intentions: withoutLedger({ ids, clock }), retentionMs: 30 * DAY, ids, clock, @@ -249,6 +256,7 @@ describe("createRetentionSweep", () => { await touch("c-1"); await forgetPersonalData({ storage, + intentions: withoutLedger({ ids, clock }), ids, clock, logger, @@ -281,6 +289,7 @@ describe("createRetentionSweep", () => { await setUp(); const other = createRetentionSweep({ storage, + intentions: withoutLedger({ ids, clock }), retentionMs: 30 * DAY, ids, clock, @@ -680,6 +689,7 @@ describe("createRetentionSweep", () => { let lists = 0; const { touch, record, advance, clock, sweepOver } = await setUp(live); const told: string[] = []; + const toldIds = createSequentialIdGenerator(); const sweep = createRetentionSweep({ storage: { ...live, @@ -691,8 +701,9 @@ describe("createRetentionSweep", () => { }, }, }, + intentions: withoutLedger({ ids: toldIds, clock }), retentionMs: 30 * DAY, - ids: createSequentialIdGenerator(), + ids: toldIds, clock, logger: createRecordingLogger().logger, concurrencyRetries: 1, @@ -723,10 +734,12 @@ describe("createRetentionSweep", () => { const { storage, record, touch, clock, sweepOver } = await setUp(); await touch("c-1"); await record(); + const longerIds = createSequentialIdGenerator(); const longer = createRetentionSweep({ storage, + intentions: withoutLedger({ ids: longerIds, clock }), retentionMs: 60 * DAY, - ids: createSequentialIdGenerator(), + ids: longerIds, clock, logger: createRecordingLogger().logger, concurrencyRetries: 0, @@ -760,3 +773,82 @@ describe("createRetentionSweep", () => { expect(asked).toEqual([]); }); }); + +describe("the retention and the forget ledger", () => { + const recordingIntentions = (ids: IdGenerator, clock: FixedClock) => { + const recorded: ForgetLedgerEntry[] = []; + const intentions = createForgetIntentions({ + ledger: boundForgetLedger({ + ledger: { + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + ids, + clock, + logger: silentLogger, + }); + return { intentions, recorded }; + }; + + it("records the intention of each forgetting before it commits, and none for a person it skips", async () => { + const { record, touch, advance, clock, storage, ids, logger } = await setUp(); + const { intentions, recorded } = recordingIntentions(ids, clock); + const sweep = createRetentionSweep({ + storage, + intentions, + retentionMs: 30 * DAY, + ids, + clock, + logger, + concurrencyRetries: 1, + batchSize: 2, + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + await touch("c-1"); + await touch("c-2"); + await record(); + await storage.persons.restrict("c-2", new Date(day(0))); + advance(31); + expect(await sweep.sweep(clock.now())).toBe(1); + const [forgetting] = ( + await storage.eventStore.readAll({ afterPosition: 0, limit: 100 }) + ).filter((event) => event.type === "PersonalDataForgotten"); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: forgetting?.id, + forgottenAt: forgetting?.timestamp, + reason: "retention", + fingerprints: ["f-1"], + }, + ]); + }); + + it("says a person is due as a sweep would forget them now, by their row and what it has not recorded", async () => { + const { record, touch, advance, clock, storage } = await setUp(); + const due = retentionDueOf({ + retentionMs: 30 * DAY, + batchSize: 2, + checkpoints: storage.checkpointStore, + }); + const isDue = (person: string) => due(storage, person, clock.now()); + for (const person of ["c-1", "c-2", "c-3", "c-4"]) await touch(person); + await record(); + expect(await isDue("c-1")).toBe(false); + advance(30); + expect(await isDue("c-1")).toBe(true); + expect(await isDue("c-9")).toBe(false); + await storage.persons.restrict("c-2", new Date(day(30))); + expect(await isDue("c-2")).toBe(false); + await touch("c-3"); + expect(await isDue("c-3")).toBe(false); + await storage.persons.record({ person: "c-4", position: 99, forgottenAt: day(30) }); + expect(await isDue("c-4")).toBe(false); + }); +}); diff --git a/packages/core/src/kernel/personal-data/retention.ts b/packages/core/src/kernel/personal-data/retention.ts index df9b0f3c..b8de8b2b 100644 --- a/packages/core/src/kernel/personal-data/retention.ts +++ b/packages/core/src/kernel/personal-data/retention.ts @@ -1,4 +1,5 @@ import type { Storage } from "../../adapter/adapter.ts"; +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; import type { EventStore } from "../../adapter/storage/event-store.ts"; import { isReleasedBy } from "../../adapter/storage/hold.ts"; import type { @@ -12,8 +13,10 @@ import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; import type { Subscriber } from "../dispatch/delivery.ts"; -import { forgetWithin, logForgotten } from "./forget.ts"; +import { forgetWithin, logForgotten, STALE_INTENTION } from "./forget.ts"; import { createPause, holdAfter } from "./hold.ts"; +import type { ForgetIntentions } from "./intention.ts"; +import type { RetentionDueFunction } from "./reconciliation.ts"; import { transactRetrying } from "./transact.ts"; export const RETENTION_SUBSCRIBER: "retention" = "retention"; @@ -33,6 +36,55 @@ const forgettingOf = (event: StoredEvent & { readonly person: string }): PersonA forgottenAt: event.timestamp, }); +const rowIsCurrent = async ( + eventStore: EventStore, + person: string, + through: number, + batchSize: number, +): Promise => { + for (let after = through; ; ) { + const events = await eventStore.readByPerson({ + person, + afterPosition: after, + limit: batchSize, + }); + if (events.some((event) => isActivity(event) || isForget(event))) return false; + const last = events[batchSize - 1]; + if (last === undefined) return true; + after = last.position; + } +}; + +const isIdle = (row: StoredPerson | null): row is StoredPerson => + row !== null && row.restrictedSince === null && row.forgottenAt === null; + +export interface RetentionDueArgs { + readonly retentionMs: number; + readonly batchSize: number; + /** + * Where the retention subscriber keeps how far its rows reach. + */ + readonly checkpoints: CheckpointStore; +} + +export interface RetentionDueOfFunction { + (args: RetentionDueArgs): RetentionDueFunction; +} + +/** + * Whether the retention would forget the person at `now`, as a sweep decides it in the transaction + * that forgets them. + */ +export const retentionDueOf: RetentionDueOfFunction = + ({ retentionMs, batchSize, checkpoints }) => + async (stores, personId, now) => { + const row = await stores.persons.get(personId); + if (!isIdle(row) || row.lastActiveAt === null) return false; + if (Date.parse(row.lastActiveAt) + retentionMs > now.getTime()) return false; + const through = await checkpoints.get(RETENTION_SUBSCRIBER); + return rowIsCurrent(stores.eventStore, personId, through, batchSize); + }; + export interface CreateRetentionSubscriberArgs { readonly persons: PersonStore; } @@ -90,6 +142,7 @@ export interface RetentionSweep { export interface CreateRetentionSweepArgs { readonly storage: Storage; + readonly intentions: ForgetIntentions; readonly retentionMs: number; readonly ids: IdGenerator; readonly clock: Clock; @@ -116,6 +169,7 @@ export interface CreateRetentionSweepFunction { */ export const createRetentionSweep: CreateRetentionSweepFunction = ({ storage, + intentions, retentionMs, ids, clock, @@ -124,53 +178,45 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ batchSize, forgotten: onForgotten, }) => { - const rowIsCurrent = async ( - eventStore: EventStore, - person: string, - through: number, - ): Promise => { - for (let after = through; ; ) { - const events = await eventStore.readByPerson({ - person, - afterPosition: after, - limit: batchSize, - }); - if (events.some((event) => isActivity(event) || isForget(event))) return false; - const last = events[batchSize - 1]; - if (last === undefined) return true; - after = last.position; - } - }; - - // Resolves `undone` for a person whose row is behind, who is then left for a later pass, and - // `skipped` for one restricted, or forgotten by another instance, since the list was read. + // Resolves `undone` for a person whose row is behind, or who was given a key since the + // intention, who is then left for a later pass, and `skipped` for one restricted, or forgotten by + // another instance, since the list was read. The row is read once before the intention, so a + // person plainly skipped leaves none; an intention left by a transaction that skips is committed + // again only if the retention would still forget the person. const forget = async ( personId: string, through: number, ): Promise<"forgotten" | "skipped" | "undone"> => { + if (!isIdle(await storage.persons.get(personId))) return "skipped"; + const intention = await intentions.intend({ + personId, + reason: "retention", + keys: storage.personalKeys, + }); const correlationId = ids.next(); const forgotten = await transactRetrying({ storage, concurrencyRetries, work: async (transaction) => { - const row = await transaction.persons.get(personId); - if (row === null || row.restrictedSince !== null || row.forgottenAt !== null) { - return "skipped"; + if (!isIdle(await transaction.persons.get(personId))) return "skipped"; + if (!(await rowIsCurrent(transaction.eventStore, personId, through, batchSize))) { + return "undone"; } - if (!(await rowIsCurrent(transaction.eventStore, personId, through))) return "undone"; - const forgotten = await forgetWithin({ + const outcome = await forgetWithin({ transaction, ids, clock, personId, - reason: "retention", + intention, context: { correlationId, causationId: correlationId, depth: 0 }, }); + if (outcome === STALE_INTENTION) return "undone"; + if ("held" in outcome) return "skipped"; // Recorded here as the subscriber will record it, so the row leaves the idle list, its // hold with it, before any pass can take the person again; the subscriber's record of the // same event then changes nothing. - await transaction.persons.record(forgettingOf({ ...forgotten.event, person: personId })); - return forgotten; + await transaction.persons.record(forgettingOf({ ...outcome.event, person: personId })); + return outcome; }, }); if (typeof forgotten === "string") return forgotten; diff --git a/packages/core/src/kernel/policy/executor.ts b/packages/core/src/kernel/policy/executor.ts index 2a77cc83..5866c295 100644 --- a/packages/core/src/kernel/policy/executor.ts +++ b/packages/core/src/kernel/policy/executor.ts @@ -7,6 +7,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import { createReactionCommands } from "../command/reaction-commands.ts"; import { openedUnder } from "../personal-data/event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { actedBy, causedBy } from "../shared/causation.ts"; import { deriveIdempotencyKey } from "../shared/idempotency-key.ts"; import { withTimeout } from "../shared/timeout.ts"; @@ -51,6 +52,7 @@ export interface CreatePolicyExecutorArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreatePolicyExecutorFunction { @@ -64,6 +66,7 @@ export const createPolicyExecutor: CreatePolicyExecutorFunction = ({ ids, clock, logger, + intentions, }) => ({ run: async ({ policy, event, attempt, retryId, actor, within }) => { within.read(openedUnder(event)); @@ -83,6 +86,7 @@ export const createPolicyExecutor: CreatePolicyExecutorFunction = ({ logger, ids, clock, + intentions, maxChainDepth: config.forAggregate(policy.aggregate).policies.maxChainDepth, }); const handled = traced({ diff --git a/packages/core/src/kernel/process/handlers.ts b/packages/core/src/kernel/process/handlers.ts index 33e8bdc1..e0e6e6f8 100644 --- a/packages/core/src/kernel/process/handlers.ts +++ b/packages/core/src/kernel/process/handlers.ts @@ -10,6 +10,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import { createReactionCommands, type ReactionCommands } from "../command/reaction-commands.ts"; import { openedUnder } from "../personal-data/event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { checkStatePerson } from "../personal-data/process-state.ts"; import { refuseRestricted } from "../personal-data/refusal.ts"; import { actedBy, causedBy } from "../shared/causation.ts"; @@ -89,6 +90,7 @@ export interface CreateProcessHandlersArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreateProcessHandlersFunction { @@ -102,6 +104,7 @@ export const createProcessHandlers: CreateProcessHandlersFunction = ({ ids, clock, logger, + intentions, }) => { const reactionFor = ( process: ProcessRuntime, @@ -118,6 +121,7 @@ export const createProcessHandlers: CreateProcessHandlersFunction = ({ logger, ids, clock, + intentions, maxChainDepth: config.forAggregate(process.aggregate).policies.maxChainDepth, }); diff --git a/packages/core/src/kernel/process/runner.ts b/packages/core/src/kernel/process/runner.ts index 16b1abcc..6792f9b6 100644 --- a/packages/core/src/kernel/process/runner.ts +++ b/packages/core/src/kernel/process/runner.ts @@ -6,6 +6,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import type { Subscriber } from "../dispatch/dispatcher.ts"; import type { AppStorage } from "../events/app-event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { deliverInOrder } from "../shared/in-order.ts"; import type { PendingRetries } from "../shared/pending-retries.ts"; import type { ProcessesRuntime } from "./build-processes.ts"; @@ -33,6 +34,7 @@ export interface CreateProcessRunnerArgs { readonly clock: Clock; readonly pendingRetries: PendingRetries; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreateProcessRunnerFunction { @@ -53,6 +55,7 @@ export const createProcessRunner: CreateProcessRunnerFunction = ({ clock, pendingRetries, logger, + intentions, }) => { const units = createProcessUnits({ storage, config, ids, clock }); const failures = createProcessFailures({ @@ -69,6 +72,7 @@ export const createProcessRunner: CreateProcessRunnerFunction = ({ ids, clock, logger, + intentions, }); const schedule = createDeadlineSchedule({ storage, units }); const deadlineStep = createDeadlineStep({ units, handlers, ids }); diff --git a/packages/core/src/kernel/reactive-harness.ts b/packages/core/src/kernel/reactive-harness.ts index 940a987f..82f88088 100644 --- a/packages/core/src/kernel/reactive-harness.ts +++ b/packages/core/src/kernel/reactive-harness.ts @@ -14,6 +14,7 @@ import { createRestrictedRetries } from "./dead-letters/restricted.ts"; import { createDeadLetterRetry, type DeadLetterRetry } from "./dead-letters/retry.ts"; import { createDispatcher, type Dispatcher } from "./dispatch/dispatcher.ts"; import type { AppStorage } from "./events/app-event-store.ts"; +import { createForgetIntentions } from "./personal-data/intention.ts"; import { buildPolicies, type PoliciesRuntime } from "./policy/build-policies.ts"; import { createDelayedPolicies } from "./policy/delayed.ts"; import { createPolicyExecutor, type PolicyExecutor } from "./policy/executor.ts"; @@ -104,6 +105,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ logger, }); const policies = buildPolicies({ registry, aggregates }); + const intentions = createForgetIntentions({ ledger: null, strict: false, ids, clock, logger }); const policyExecutor = createPolicyExecutor({ aggregates, pipeline, @@ -111,6 +113,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ ids, clock, logger, + intentions, }); const processes = createProcessRunner({ processes: buildProcesses({ registry, aggregates, config }), @@ -122,6 +125,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ clock, pendingRetries: ignoredRetries, logger, + intentions, }); const retries = createDeadLetterRetry({ storage, diff --git a/packages/core/src/kernel/shared/runtime-text.ts b/packages/core/src/kernel/shared/runtime-text.ts index d7498d80..51e18a11 100644 --- a/packages/core/src/kernel/shared/runtime-text.ts +++ b/packages/core/src/kernel/shared/runtime-text.ts @@ -3,6 +3,8 @@ import { ClaimLostError, ConcurrencyError, DeadLetterSettledError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, ProcessingRestrictedError, RebuildSupersededError, @@ -18,6 +20,8 @@ const COMPOSED: readonly (abstract new (...args: never[]) => Error)[] = [ ClaimLostError, ConcurrencyError, DeadLetterSettledError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, HandlerTimeoutError, ProcessingRestrictedError, diff --git a/packages/core/src/kernel/system-events.test.ts b/packages/core/src/kernel/system-events.test.ts index 61e7baee..a246e948 100644 --- a/packages/core/src/kernel/system-events.test.ts +++ b/packages/core/src/kernel/system-events.test.ts @@ -68,6 +68,23 @@ describe("appendSystemEvent", () => { ).toEqual([appended]); }); + it("takes the id and timestamp set before it, when given", async () => { + const storage = await memory().createStorage(); + const appended = await appendSystemEvent({ + eventStore: storage.eventStore, + ids: createSequentialIdGenerator(), + clock: createFixedClock(), + aggregateType: "bounda", + aggregateId: "c-1", + type: "PersonalDataForgotten", + payload: {}, + context, + identity: { id: "intended", timestamp: "2026-01-02T03:04:05.000Z" }, + }); + expect(appended.id).toBe("intended"); + expect(appended.timestamp).toBe("2026-01-02T03:04:05.000Z"); + }); + it("lets a stream that moved fail, for the unit of work to run again", async () => { const { storage, append } = await setup(); let appends = 0; diff --git a/packages/core/src/kernel/system-events.ts b/packages/core/src/kernel/system-events.ts index d25c4832..26db33b4 100644 --- a/packages/core/src/kernel/system-events.ts +++ b/packages/core/src/kernel/system-events.ts @@ -30,6 +30,11 @@ export interface AppendSystemEventArgs { * The person the event is about, on their own stream. */ readonly person?: string; + /** + * The event's id and timestamp when they were set before, as a forgetting's intention sets them; + * from `ids` and `clock` when omitted. + */ + readonly identity?: { readonly id: string; readonly timestamp: string }; } export interface AppendSystemEventFunction { @@ -51,6 +56,7 @@ export const appendSystemEvent: AppendSystemEventFunction = async ({ payload, context, person = null, + identity, }) => { const { version } = await eventStore.load({ aggregateType, @@ -63,13 +69,13 @@ export const appendSystemEvent: AppendSystemEventFunction = async ({ expectedVersion: version, events: [ { - id: ids.next(), + id: identity?.id ?? ids.next(), aggregateType, aggregateId, version: version + 1, type, payload, - timestamp: clock.now().toISOString(), + timestamp: identity?.timestamp ?? clock.now().toISOString(), metadata: { ...context, schemaVersion: 1, system: true }, person, }, diff --git a/packages/core/src/kernel/test-support.ts b/packages/core/src/kernel/test-support.ts index a33e1f96..c2dcfcb1 100644 --- a/packages/core/src/kernel/test-support.ts +++ b/packages/core/src/kernel/test-support.ts @@ -6,7 +6,7 @@ import { resolveConfig } from "../config/schema.ts"; import type { Config, ErrorText, PortsConfig, ResolvedConfig } from "../config/types.ts"; import { type Clock, createFixedClock, type FixedClock } from "../contracts/clock.ts"; import type { CommandRejection } from "../contracts/command.ts"; -import { createSequentialIdGenerator } from "../contracts/ids.ts"; +import { createSequentialIdGenerator, type IdGenerator } from "../contracts/ids.ts"; import { type LogFields, type Logger, silentLogger } from "../contracts/logger.ts"; import { memory } from "../memory/index.ts"; import type { RejectFunction } from "../modules/command.ts"; @@ -27,6 +27,7 @@ import { toBase64, withRawKey, } from "./personal-data/crypto.ts"; +import { createForgetIntentions, type ForgetIntentions } from "./personal-data/intention.ts"; import { createPersonalKeyWrapping, type PersonalKeys } from "./personal-data/wrapped-keys.ts"; import { createPorts, type ModulePorts } from "./ports/ports.ts"; import { guardedLogger } from "./shared/guarded-logger.ts"; @@ -485,3 +486,14 @@ export const openedKey: OpenedKeyFunction = async ({ key, fingerprint }) => ({ fingerprint, cryptoKey: await importRawKey({ raw: fromBase64(key), extractable: false }), }); + +export interface WithoutLedgerArgs { + readonly ids: IdGenerator; + readonly clock: Clock; +} + +/** + * The intentions of an app without a forget ledger: ids and times, nothing recorded. + */ +export const withoutLedger = ({ ids, clock }: WithoutLedgerArgs): ForgetIntentions => + createForgetIntentions({ ledger: null, strict: false, ids, clock, logger: silentLogger }); diff --git a/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts b/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts index 82f9db56..d5cb63d8 100644 --- a/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts +++ b/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts @@ -331,6 +331,47 @@ describe("createUnitOfWork", () => { expect(transactions).toBe(1); }); + it("runs the steps before the commit's transaction, in order, on the store, and writes nothing when one throws", async () => { + const storage = await memory().createStorage(); + await storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-1", kid: null }); + const transact = storage.transact.bind(storage); + const ran: string[] = []; + storage.transact = (work) => { + ran.push("transaction"); + return transact(work); + }; + const unit = createUnitOfWork({ storage }); + unit.beforeCommit(async ({ personalKeys }) => { + ran.push(`first ${(await personalKeys.get("c-1"))?.fingerprint}`); + }); + unit.beforeCommit(async () => { + ran.push("second"); + }); + unit.atCommit(async () => { + ran.push("at commit"); + }); + expect(ran).toEqual([]); + await unit.commit(); + expect(ran).toEqual(["first f-1", "second", "transaction", "at commit"]); + + const refused = createUnitOfWork({ storage }); + await refused.scheduler.schedule(entry("command:refused")); + refused.beforeCommit(async () => { + throw new Error("ledger is down"); + }); + refused.afterCommit(() => ran.push("after")); + await expect(refused.commit()).rejects.toThrow("ledger is down"); + expect(await storage.scheduler.list()).toEqual([]); + expect(ran).toEqual(["first f-1", "second", "transaction", "at commit"]); + + const idle = createUnitOfWork({ storage }); + idle.beforeCommit(async () => { + ran.push("idle"); + }); + await idle.commit(); + expect(ran).not.toContain("idle"); + }); + it("commits nothing when a stream moved since the unit loaded it", async () => { const storage = await memory().createStorage(); const unit = createUnitOfWork({ storage }); diff --git a/packages/core/src/kernel/unit-of-work/unit-of-work.ts b/packages/core/src/kernel/unit-of-work/unit-of-work.ts index 01209253..2fc46b25 100644 --- a/packages/core/src/kernel/unit-of-work/unit-of-work.ts +++ b/packages/core/src/kernel/unit-of-work/unit-of-work.ts @@ -11,7 +11,14 @@ import { asError } from "../shared/retry.ts"; */ export type UnitStores = Pick< UnitOfWork, - "eventStore" | "scheduler" | "idempotencyStore" | "persons" | "atCommit" | "afterCommit" | "read" + | "eventStore" + | "scheduler" + | "idempotencyStore" + | "persons" + | "beforeCommit" + | "atCommit" + | "afterCommit" + | "read" >; /** @@ -26,6 +33,12 @@ export type UnitStores = Pick< */ export interface UnitOfWork extends Omit { commit(): Promise; + /** + * Runs `step` before the commit's transaction opens, in the order given, for what has to happen + * outside the store first, as recording a forgetting's intention does: what it throws fails the + * commit, and nothing is written. Never for a unit that does not commit. + */ + beforeCommit(step: (storage: Pick) => Promise): void; /** * Runs `write` on the commit's transaction, after the unit's events and other writes, in the * order given: for a write that needs what the transaction's stores answer, as forgetting a @@ -59,6 +72,7 @@ export interface CreateUnitOfWorkFunction { export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { const staged = createStagedEventStore(storage.eventStore); const { stores, flush, pending } = deferWrites(storage); + const beforeCommit: ((storage: Pick) => Promise)[] = []; const atCommit: ((transaction: StorageTransaction) => Promise)[] = []; const committed: (() => void)[] = []; const read: KeyedPerson[] = []; @@ -72,6 +86,7 @@ export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { const batches = staged.batches(); const reseals = staged.reseals(); if (batches.length > 0 || reseals.length > 0 || pending() || atCommit.length > 0) { + for (const step of beforeCommit) await step(storage); await storage.transact(async (transaction) => { for (const batch of batches) await transaction.eventStore.append(batch); if (reseals.length > 0) await transaction.eventStore.reseal(reseals); @@ -81,6 +96,9 @@ export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { } for (const callback of committed.splice(0)) callback(); }, + beforeCommit: (step) => { + beforeCommit.push(step); + }, atCommit: (write) => { atCommit.push(write); }, diff --git a/packages/core/src/memory/forget-ledger.ts b/packages/core/src/memory/forget-ledger.ts index 1ef7664c..524a946e 100644 --- a/packages/core/src/memory/forget-ledger.ts +++ b/packages/core/src/memory/forget-ledger.ts @@ -33,7 +33,13 @@ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () = return { append: async (appended) => { for (const entry of appended) { - if (!entries.has(entry.eventId)) entries.set(entry.eventId, { ...entry }); + if (entries.has(entry.eventId)) continue; + entries.set( + entry.eventId, + entry.fingerprints === undefined + ? { ...entry } + : { ...entry, fingerprints: [...entry.fingerprints] }, + ); } }, read: async ({ after, limit }) => @@ -41,7 +47,11 @@ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () = .filter((entry) => after === undefined || byForgetLedgerOrder(entry, after) > 0) .sort(byForgetLedgerOrder) .slice(0, limit) - .map((entry) => ({ ...entry })), + .map((entry) => + entry.fingerprints === undefined + ? { ...entry } + : { ...entry, fingerprints: [...entry.fingerprints] }, + ), prune: async ({ before }) => { for (const [eventId, entry] of entries) { if (entry.forgottenAt < before) entries.delete(eventId); diff --git a/packages/core/src/modules/bounda-events.ts b/packages/core/src/modules/bounda-events.ts index 6e74fb8c..71227588 100644 --- a/packages/core/src/modules/bounda-events.ts +++ b/packages/core/src/modules/bounda-events.ts @@ -12,7 +12,8 @@ export const BOUNDA_AGGREGATE: "bounda" = "bounda"; * a policy or process that forgets them, or the retention: every event's personal field of theirs * reads as `null` from then on, and every read model that projects such fields handles it. It * names the stored commands that held the person's data and went with the key, since no other - * event records them. + * event records them. With a forget ledger, the reconciliation may append it again, with the same + * id, time and reason, to a store that lost it. */ export const PERSONAL_DATA_FORGOTTEN_EVENT: "PersonalDataForgotten" = "PersonalDataForgotten"; @@ -33,16 +34,22 @@ export interface RemovedDeadLetter { } /** - * Why a person was forgotten: `requested` by `forget`, from a client or a policy or process, - * `retention`, past `runtime.personalData.retention` without activity, or `restored`, again, - * because a backup restored from before they were forgotten brought their key back, as - * `runtime.personalData.forgetLedger` or `bounda forget --from` found. + * Why a person was forgotten: `requested` by `forget`, from a client or a policy or process, or + * by `bounda forget --from` for a bare id, or `retention`, past `runtime.personalData.retention` + * without activity. A forgetting committed again from `runtime.personalData.forgetLedger`, after + * a restore or a commit that never happened, keeps the reason it was asked with. */ -export type PersonalDataForgottenReason = "requested" | "retention" | "restored"; +export type PersonalDataForgottenReason = "requested" | "retention"; export interface PersonalDataForgottenPayload { readonly personId: string; readonly reason: PersonalDataForgottenReason; + /** + * The fingerprints of the keys the forgetting destroyed, none when the person held none: a key + * the person holds whose fingerprint is not here was given to them after. Absent from a + * forgetting recorded before Bounda 0.4.0. + */ + readonly fingerprints?: readonly string[]; readonly cancelledCommands: readonly CancelledCommand[]; readonly removedDeadLetters: readonly RemovedDeadLetter[]; } diff --git a/packages/core/src/modules/policy.ts b/packages/core/src/modules/policy.ts index 1eea8fde..f1304147 100644 --- a/packages/core/src/modules/policy.ts +++ b/packages/core/src/modules/policy.ts @@ -40,7 +40,9 @@ export type PolicyHandlerArgs; diff --git a/packages/core/test-types/config.test-d.ts b/packages/core/test-types/config.test-d.ts index 1ae7624a..30ff4c8a 100644 --- a/packages/core/test-types/config.test-d.ts +++ b/packages/core/test-types/config.test-d.ts @@ -1,3 +1,4 @@ +import type { PersonalDataForgottenReason } from "@bounda-dev/core"; import { type DurationInput, type ErrorFields, @@ -371,7 +372,13 @@ describe("defineConfig", () => { ports, runtime: { personalData: { - forgetLedger: { store: own, reconcileEvery: "1m", strict: true, keepFor: "35d" }, + forgetLedger: { + store: own, + reconcileEvery: "1m", + strict: true, + keepFor: "35d", + timeout: "5s", + }, }, }, }); @@ -380,11 +387,14 @@ describe("defineConfig", () => { readonly reconcileEvery?: DurationInput; readonly strict?: boolean; readonly keepFor?: DurationInput; + readonly timeout?: DurationInput; }>(); expectTypeOf().toEqualTypeOf<{ readonly personId: string; readonly eventId: string; readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + readonly fingerprints?: readonly string[]; }>(); expectTypeOf().toEqualTypeOf<{ readonly before: string }>(); expectTypeOf().toEqualTypeOf(); diff --git a/packages/core/test-types/inference.test-d.ts b/packages/core/test-types/inference.test-d.ts index 04d9e55e..aa36aa61 100644 --- a/packages/core/test-types/inference.test-d.ts +++ b/packages/core/test-types/inference.test-d.ts @@ -707,15 +707,14 @@ describe("personal data", () => { expectTypeOf().toEqualTypeOf<{ readonly personId: string; readonly reason: PersonalDataForgottenReason; + readonly fingerprints?: readonly string[]; readonly cancelledCommands: readonly CancelledCommand[]; readonly removedDeadLetters: readonly RemovedDeadLetter[]; }>(); - expectTypeOf().toEqualTypeOf< - "requested" | "retention" | "restored" - >(); + expectTypeOf().toEqualTypeOf<"requested" | "retention">(); expectTypeOf< AuditOnPersonalDataForgotten.HandlerArgs["event"]["payload"]["reason"] - >().toEqualTypeOf<"requested" | "retention" | "restored">(); + >().toEqualTypeOf<"requested" | "retention">(); expectTypeOf().toEqualTypeOf<{ readonly commandId: string; readonly type: string; diff --git a/packages/postgresql/src/migrations.ts b/packages/postgresql/src/migrations.ts index 0959fd57..db6399f2 100644 --- a/packages/postgresql/src/migrations.ts +++ b/packages/postgresql/src/migrations.ts @@ -1,5 +1,6 @@ import { ConfigurationError } from "@bounda-dev/core"; import { + forgetLedgerReasonStatements, forgetLedgerStatements, migrateSchemas, postgresqlDialect, @@ -170,6 +171,12 @@ export const postgresqlForgetLedgerScope: PostgresqlForgetLedgerScopeFunction = } }, }, + { + version: 2, + migrate: async (tx) => { + for (const statement of forgetLedgerReasonStatements(table)) await tx.run(statement, []); + }, + }, ], }); diff --git a/packages/sqlite/src/forget-ledger.test.ts b/packages/sqlite/src/forget-ledger.test.ts index 67c085ba..b909366f 100644 --- a/packages/sqlite/src/forget-ledger.test.ts +++ b/packages/sqlite/src/forget-ledger.test.ts @@ -4,13 +4,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type BoundaApp, + type Clock, createApp, type EventPayloadArgs, type PayloadArgs, type PersonalDataForgottenPayload, type Registry, + systemClock, } from "@bounda-dev/core"; -import type { ForgetLedger } from "@bounda-dev/core/adapter"; +import type { Adapter, ForgetLedger } from "@bounda-dev/core/adapter"; import { forgetLedgerContract } from "@bounda-dev/core/adapter/testing"; import { createClient } from "@libsql/client"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -92,13 +94,24 @@ interface Paths { readonly ledger: string; } -const boot = (paths: Paths): Promise => +// How far ahead a start after a restore runs: past the grace a reconciliation gives an intention, +// as a restore takes longer than that. +const LATER_MS = 2 * 60_000; + +// The system's clock, `LATER_MS` ahead. +const later: Clock = { + now: () => new Date(Date.now() + LATER_MS), + after: (milliseconds, callback, options) => systemClock.after(milliseconds, callback, options), +}; + +const boot = (paths: Paths, clock: Clock = systemClock): Promise => createApp({ registry, config: { storage: sqlite({ path: paths.store }), runtime: { personalData: { forgetLedger: { store: sqlite({ path: paths.ledger }) } } }, }, + clock, }); const register = async (app: BoundaApp, customerId: string): Promise => { @@ -129,7 +142,7 @@ const startElsewhere = (paths: Paths): Promise => [ "--input-type=module", "-e", - `import { createApp } from "@bounda-dev/core"; + `import { createApp, systemClock } from "@bounda-dev/core"; import { sqlite } from "./index.ts"; const app = await createApp({ registry: { aggregates: {}, readModels: {} }, @@ -137,6 +150,10 @@ const startElsewhere = (paths: Paths): Promise => storage: sqlite({ path: ${JSON.stringify(paths.store)} }), runtime: { personalData: { forgetLedger: { store: sqlite({ path: ${JSON.stringify(paths.ledger)} }) } } }, }, + clock: { + now: () => new Date(Date.now() + ${LATER_MS}), + after: (milliseconds, callback, options) => systemClock.after(milliseconds, callback, options), + }, }); await app.stop();`, ], @@ -180,15 +197,15 @@ describe("restoring a backup with a forget ledger", () => { await before.stop(); const restoredPaths = { ...paths, store: await restore(backup) }; - const restored = await boot(restoredPaths); - expect(await forgotten(restored)).toEqual([["c-1", "restored"]]); + const restored = await boot(restoredPaths, later); + expect(await forgotten(restored)).toEqual([["c-1", "requested"]]); expect(await nameOf(restored, "c-1")).toBeNull(); expect(await nameOf(restored, "c-2")).toBe("name of c-2"); await restored.runUntilIdle(); await restored.stop(); - const again = await boot(restoredPaths); - expect(await forgotten(again)).toEqual([["c-1", "restored"]]); + const again = await boot(restoredPaths, later); + expect(await forgotten(again)).toEqual([["c-1", "requested"]]); expect(await again.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); await again.stop(); }); @@ -210,11 +227,65 @@ describe("restoring a backup with a forget ledger", () => { await client.execute("PRAGMA journal_mode = WAL"); client.close(); await Promise.all([startElsewhere(restoredPaths), startElsewhere(restoredPaths)]); - const after = await boot(restoredPaths); - expect(await forgotten(after)).toEqual([["c-1", "restored"]]); + const after = await boot(restoredPaths, later); + expect(await forgotten(after)).toEqual([["c-1", "requested"]]); await after.stop(); }); + it("commits on the restored copy the forgettings after the backup, a pending intention's included, each under its id", async () => { + const paths = fresh(); + const backup = pathOf(`backup-${databases}`); + const before = await boot(paths); + await register(before, "c-1"); + await register(before, "c-2"); + await register(before, "c-3"); + await backUp(paths.store, backup); + await before.forget("c-1"); + await before.stop(); + // The process stops between the intention and the commit of c-2's forgetting. + let crashing = true; + const store = sqlite({ path: paths.store }); + const stopped: Adapter = { + ...store, + createStorage: async () => { + const opened = await store.createStorage(); + return { + ...opened, + transact: (work) => + crashing ? Promise.reject(new Error("the process stopped")) : opened.transact(work), + }; + }, + }; + const stopping = await createApp({ + registry, + config: { + storage: stopped, + runtime: { personalData: { forgetLedger: { store: sqlite({ path: paths.ledger }) } } }, + }, + }); + await expect(stopping.forget("c-2")).rejects.toThrow("the process stopped"); + crashing = false; + await stopping.stop(); + + const ledger = await sqlite({ path: paths.ledger }).createForgetLedger({}); + opened.push(ledger); + const intentions = await ledger.read({ limit: 10 }); + expect(intentions.map(({ personId, reason }) => [personId, reason])).toEqual([ + ["c-1", "requested"], + ["c-2", "requested"], + ]); + const restored = await boot({ ...paths, store: await restore(backup) }, later); + expect( + (await restored.events.read({ limit: 1000 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => [event.aggregateId, event.id]), + ).toEqual(intentions.map(({ personId, eventId }) => [personId, eventId])); + expect(await nameOf(restored, "c-1")).toBeNull(); + expect(await nameOf(restored, "c-2")).toBeNull(); + expect(await nameOf(restored, "c-3")).toBe("name of c-3"); + await restored.stop(); + }); + it("brings the person back without a ledger, which bounda forget --from then forgets again", async () => { const store = fresh().store; const backup = pathOf(`backup-${databases}`); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 236c410a..e319bd26 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -462,6 +462,9 @@ importers: specifier: 'catalog:' version: 15.0.0 devDependencies: + '@bounda-dev/cloudflare': + specifier: workspace:* + version: link:../cloudflare '@bounda-dev/core': specifier: workspace:* version: link:../core @@ -483,6 +486,9 @@ importers: vitest: specifier: 'catalog:' version: 5.1.0(@opentelemetry/api@1.10.0)(@types/node@26.6.6)(@vitest/coverage-v8@5.1.0)(vite@8.3.6(@types/node@26.6.6)(esbuild@0.28.2)(yaml@2.9.1)) + wrangler: + specifier: 'catalog:' + version: 4.157.0(@cloudflare/workers-types@5.20261103.1)(@types/node@26.6.6) packages/cloudflare: devDependencies: From 294c75f87a21c8728419c2d2d2686d2621f5ce2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:32:44 +0100 Subject: [PATCH 2/6] refactor: tighten the forget ledger's intentions after review and mutation testing Pass the ledger to the intentions with its strictness and open it only when configured, so nothing guards a ledger that cannot be missing. A whole entry is committed again without reading its event and key a third time, a bare id given a key meanwhile takes a new intention, and the retention checks the person's row is current before it writes an intention it would leave. The CLI opens a project's storage through isAdapter, now public, and a ledger of your own is told to keep every field of an entry. --- .changeset/forget-ledger-first.md | 2 +- docs/src/content/docs/guides/personal-data.md | 5 +- packages/cli/src/forget-ledger-expire.test.ts | 7 +- packages/cli/src/forget-ledger-expire.ts | 22 +-- packages/cli/src/forgotten.test.ts | 8 + packages/cli/src/run.test.ts | 58 +++++++ packages/cli/src/run.ts | 7 +- .../cloudflare/test/forget-ledger.test.ts | 1 + packages/core/src/adapter/adapter.ts | 4 + packages/core/src/adapter/index.ts | 2 + .../core/src/adapter/sql/forget-ledger.ts | 3 +- packages/core/src/kernel/app.ts | 101 +++++++----- .../kernel/command/reaction-commands.test.ts | 30 ++-- .../src/kernel/personal-data/forget-ledger.ts | 5 +- .../src/kernel/personal-data/forget.test.ts | 64 ++++++-- .../core/src/kernel/personal-data/forget.ts | 15 +- .../kernel/personal-data/intention.test.ts | 3 +- .../src/kernel/personal-data/intention.ts | 13 +- .../personal-data/ledger-outbox.test.ts | 1 - .../personal-data/reconciliation-app.test.ts | 24 ++- .../personal-data/reconciliation.test.ts | 134 ++++++++++++++-- .../kernel/personal-data/reconciliation.ts | 64 ++++---- .../kernel/personal-data/retention.test.ts | 148 ++++++++++++++++-- .../src/kernel/personal-data/retention.ts | 7 +- packages/core/src/kernel/reactive-harness.ts | 2 +- packages/core/src/kernel/test-support.ts | 2 +- packages/core/src/memory/forget-ledger.ts | 14 +- 27 files changed, 563 insertions(+), 183 deletions(-) diff --git a/.changeset/forget-ledger-first.md b/.changeset/forget-ledger-first.md index d2331d9f..6f1baace 100644 --- a/.changeset/forget-ledger-first.md +++ b/.changeset/forget-ledger-first.md @@ -10,6 +10,6 @@ With a forget ledger, every forgetting is in the ledger before it commits. `forg Ledger entries and `PersonalDataForgotten` name the keys a forgetting destroys by their fingerprint: `ForgetLedgerEntry` gains `reason` and `fingerprints`, the payload gains `fingerprints`, and a reconciliation leaves a key the entry does not name, given to the person after it. The reason `"restored"` is gone: a forgetting committed again keeps its own. The SQL ledger table gains `reason` and `fingerprints` in version 2 of its `forget-ledger` scope, and the KV ledger keeps them in each key's metadata; an entry written before reads as a request that names no keys. -`forgetLedger.strict` now also means nothing is forgotten that the ledger has not recorded: a forgetting whose intention the ledger does not take fails with `ForgetLedgerUnavailableError` (`FORGET_LEDGER_UNAVAILABLE`) and forgets nobody, so trying again is safe. `forgetLedger.timeout` (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS`, ten seconds) bounds each call to the ledger, given up with `ForgetLedgerTimeoutError` (`FORGET_LEDGER_TIMEOUT`), and how long a start that is not strict waits for the reconciliation; a strict start fails once a read runs past it. `parseDuration` is exported. +`forgetLedger.strict` now also means nothing is forgotten that the ledger has not recorded: a forgetting whose intention the ledger does not take fails with `ForgetLedgerUnavailableError` (`FORGET_LEDGER_UNAVAILABLE`) and forgets nobody, so trying again is safe. `forgetLedger.timeout` (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS`, ten seconds) bounds each call to the ledger, given up with `ForgetLedgerTimeoutError` (`FORGET_LEDGER_TIMEOUT`), and how long a start that is not strict waits for the reconciliation; a strict start fails once a read runs past it. `parseDuration` is exported, and `isAdapter` from `@bounda-dev/core/adapter`. `bounda forgotten` reads the event store without starting the app, so a strict ledger that is down does not stop it, and prints each forgetting's reason and fingerprints; `bounda forget --from` commits each whole line again under its own id, time and reason. `bounda forget-ledger expire --binding --keep-for ` gives every key of a Workers KV ledger the expiration `keepFor` sets, those written before it was set included, through the project's wrangler; `expireKvForgetLedger` and `planKvForgetLedgerExpiry` from `@bounda-dev/cloudflare`, and its new `./forget-ledger` subpath, do the same in code. diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index 02f12da2..de0564e5 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -873,8 +873,9 @@ Every object keeps its entries under its own name, so an object reached through name fails to start; `createWorker` names each by its tenant. KV may take up to a minute to list a key it just wrote, so an object that starts reads the ledger once more a minute later, and wakes for it. Anything else is a ledger of your own through `defineForgetLedger`, which receives the host's -environment, the tenant and `keepForMs` when the app starts: its `read` returns entries in the -order of their time, then of their event's id, and `byForgetLedgerOrder` from +environment, the tenant and `keepForMs` when the app starts: it keeps every field of an entry, the +reason and the fingerprints included, which the reconciliation decides by; its `read` returns +entries in the order of their time, then of their event's id, and `byForgetLedgerOrder` from `@bounda-dev/core/adapter` sorts them so; its optional `prune` removes the entries older than a time; `forgetLedgerContract` from `@bounda-dev/core/adapter/testing` is the test it has to pass. diff --git a/packages/cli/src/forget-ledger-expire.test.ts b/packages/cli/src/forget-ledger-expire.test.ts index 628489de..59752727 100644 --- a/packages/cli/src/forget-ledger-expire.test.ts +++ b/packages/cli/src/forget-ledger-expire.test.ts @@ -1,6 +1,7 @@ +import { existsSync } from "node:fs"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import * as ledger from "@bounda-dev/cloudflare/forget-ledger"; import { afterAll, describe, expect, it } from "vitest"; import { expireKvLedger, type Wrangler, wranglerIn } from "./forget-ledger-expire.ts"; @@ -51,6 +52,8 @@ describe("expiring a KV forget ledger through wrangler", () => { now: NOW, }); expect(expired).toEqual({ expired: 2, removed: 1 }); + // The files wrangler read are gone with their directory. + for (const args of calls.slice(1)) expect(existsSync(dirname(args[3] ?? ""))).toBe(false); expect(calls.map((args) => args.slice(0, 3))).toEqual([ ["kv", "key", "list"], ["kv", "bulk", "put"], @@ -120,7 +123,7 @@ describe("expiring a KV forget ledger through wrangler", () => { }); it("refuses a listing that is not a list of keys, changing nothing", async () => { - for (const listed of [{}, [{ name: 3 }], [{ name: "k", expiration: "soon" }], [null]]) { + for (const listed of [{}, [{ name: 3 }], [{ name: "k", expiration: "soon" }], [null], ["k"]]) { const { wrangler, calls } = fakeWrangler(listed); await expect( expireKvLedger({ diff --git a/packages/cli/src/forget-ledger-expire.ts b/packages/cli/src/forget-ledger-expire.ts index ff01c178..55c0463b 100644 --- a/packages/cli/src/forget-ledger-expire.ts +++ b/packages/cli/src/forget-ledger-expire.ts @@ -8,9 +8,7 @@ import type { KvLedgerKey } from "@bounda-dev/cloudflare/forget-ledger"; const run = promisify(execFile); -/** - * Runs wrangler with the arguments given and resolves what it printed to stdout. - */ +// Resolves what wrangler printed to stdout. export interface Wrangler { (args: readonly string[]): Promise; } @@ -39,10 +37,8 @@ const installedIn = (root: string, name: string): string | undefined => { } }; -/** - * The project's own wrangler, run through Node rather than a shell, so its `.cmd` shim on Windows - * is not needed. Wrangler's stderr, where it reports progress, is left out. - */ +// Through Node rather than a shell, so the `.cmd` shim it has on Windows is not needed; its stderr, +// where it reports progress, is left out. export const wranglerIn: WranglerInFunction = (root) => { let bin: string | undefined; return async (args) => { @@ -61,9 +57,6 @@ export const wranglerIn: WranglerInFunction = (root) => { }; }; -/** - * Where wrangler finds the namespace: the deployed one, or the local state `wrangler dev` keeps. - */ export interface WranglerTarget { readonly local: boolean; readonly env?: string | undefined; @@ -71,9 +64,7 @@ export interface WranglerTarget { readonly persistTo?: string | undefined; } -/** - * What `@bounda-dev/cloudflare/forget-ledger` gives this command, which owns the keys' layout. - */ +// The KV ledger's own package lays its keys out, so the plan comes from it. export type KvLedgerModule = Pick< typeof import("@bounda-dev/cloudflare/forget-ledger"), "KV_FORGET_LEDGER_ROOT" | "planKvForgetLedgerExpiry" @@ -103,11 +94,6 @@ const isListedKey = (value: unknown): value is KvLedgerKey => typeof Reflect.get(value, "name") === "string" && ["number", "undefined"].includes(typeof Reflect.get(value, "expiration")); -/** - * Lists every object's entries in the namespace, and writes again, through wrangler's bulk - * commands, those `planKvForgetLedgerExpiry` gives an expiration, removing those already past - * `keepFor`. - */ export const expireKvLedger: ExpireKvLedgerFunction = async ({ wrangler, ledger, diff --git a/packages/cli/src/forgotten.test.ts b/packages/cli/src/forgotten.test.ts index 370323f8..7b1ac54c 100644 --- a/packages/cli/src/forgotten.test.ts +++ b/packages/cli/src/forgotten.test.ts @@ -69,6 +69,8 @@ describe("readForgotten", () => { reason: "restored", fingerprints: [1], }), + event(1_009, "bounda", "c-4", "PersonalDataForgotten", undefined, null), + event(1_010, "bounda", "c-5", "PersonalDataForgotten", undefined, "text"), ); const { stream, reads } = streamOf([ event(0, "bounda", "c-0", "PersonalDataForgotten"), @@ -95,6 +97,12 @@ describe("readForgotten", () => { forgottenAt: "2026-11-01T00:00:00.000Z", reason: "requested", }, + ...["c-4", "c-5"].map((personId, index) => ({ + personId, + eventId: `e-${1_009 + index}`, + forgottenAt: "2026-11-01T00:00:00.000Z", + reason: "requested", + })), ]); expect(reads).toEqual([ { after: 2, limit: 1_000 }, diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 5eb312d7..795ab356 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -731,6 +731,45 @@ describe("bounda forgotten and bounda forget", () => { ); }); + it("fails a listing whose project cannot load, or whose storage cannot open, and closes what opened", async () => { + const missing = await cli( + ["forgotten", "--since", "0", "--config", "missing.config.ts"], + fixture, + ); + expect(missing.code).toBe(EXIT_FAILURE); + expect(missing.stdout).toBe(""); + const root = await list(); + await writeFile( + join(root, "registry.ts"), + "export const registry = { aggregates: {}, readModels: {} };\n", + ); + const adapter = (open: string) => + `import { writeFileSync } from "node:fs";\nexport default { storage: { kind: "bounda-adapter", name: "broken", options: {}, createReadModel: async () => undefined, rebuildReadModel: async () => undefined, ${open} } };\n`; + await writeFile( + join(root, "down.config.ts"), + adapter(`createStorage: async () => { throw new Error("database is down"); }`), + ); + const down = await cli( + ["forgotten", "--since", "0", "--config", "down.config.ts", "--registry", "registry.ts"], + root, + ); + expect(down.code).toBe(EXIT_FAILURE); + expect(besidesLogs(down.stderr)).toMatch(/^error: Error; /); + await writeFile( + join(root, "failing.config.ts"), + adapter( + `createStorage: async () => ({ eventStore: { readAll: async () => { throw new Error("read failed"); } }, close: async () => writeFileSync(${JSON.stringify(join(root, "closed"))}, "") })`, + ), + ); + const failing = await cli( + ["forgotten", "--since", "0", "--config", "failing.config.ts", "--registry", "registry.ts"], + root, + ); + expect(failing.code).toBe(EXIT_FAILURE); + expect(besidesLogs(failing.stderr)).toMatch(/^error: Error; /); + await expect(stat(join(root, "closed"))).resolves.toBeDefined(); + }); + it("says a storage without its factories cannot be listed", async () => { const root = await list(); await writeFile( @@ -906,6 +945,21 @@ describe("bounda forget-ledger expire", () => { }); it("documents itself in its help", async () => { + const parent = await runCli({ + argv: ["forget-ledger", "--help"], + cwd: "/project", + stdout: capture(), + stderr: capture(), + }); + expect(parent).toBe(EXIT_OK); + const group = capture(); + await runCli({ + argv: ["forget-ledger", "--help"], + cwd: "/project", + stdout: group, + stderr: capture(), + }); + expect(group.text()).toContain("look after a forget ledger outside the app"); const help = (await expire(["--help"])).stdout.replace(/\s+/g, " "); for (const text of [ "give every entry of a Workers KV forget ledger the expiration keepFor sets, those written before it was set included", @@ -916,9 +970,13 @@ describe("bounda forget-ledger expire", () => { "--local", "the local state of wrangler dev instead of the deployed namespace", "--env ", + "the wrangler environment", "--wrangler-config ", + "the wrangler configuration file", "--persist-to ", + "with --local, where wrangler dev keeps its state", "--root ", + "the project's root directory, where wrangler is installed", ]) { expect(help).toContain(text); } diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index f296a8d4..3164c143 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -12,7 +12,7 @@ import { parseDuration, rebuildReadModel, } from "@bounda-dev/core"; -import type { Adapter, AdapterDefinition, Storage } from "@bounda-dev/core/adapter"; +import { type AdapterDefinition, isAdapter, type Storage } from "@bounda-dev/core/adapter"; import type { Config, ErrorText } from "@bounda-dev/core/config"; import { boot, createConsoleLogger, loadProject } from "@bounda-dev/core/node"; import { Command, CommanderError, InvalidArgumentError, Option } from "commander"; @@ -319,9 +319,6 @@ interface ForgottenOptions extends ProjectOptions { readonly since: Since; } -const hasFactories = (definition: AdapterDefinition): definition is Adapter => - typeof Reflect.get(definition, "createStorage") === "function"; - // Reads the event store alone, without starting the app: no ports, read models or forget ledger, // so a strict ledger that is down does not keep it from listing what the database holds. const runForgotten = async ( @@ -333,7 +330,7 @@ const runForgotten = async ( const project = await load(options, cwd, logTo(stderr), stderr); if (project === undefined) return EXIT_FAILURE; const definition: AdapterDefinition = project.config.storage; - if (!hasFactories(definition)) { + if (!isAdapter(definition)) { line( stderr, `error: storage "${definition.name}" is a definition without factories. Import the adapter package's factory.`, diff --git a/packages/cloudflare/test/forget-ledger.test.ts b/packages/cloudflare/test/forget-ledger.test.ts index 2ad35292..0c806c08 100644 --- a/packages/cloudflare/test/forget-ledger.test.ts +++ b/packages/cloudflare/test/forget-ledger.test.ts @@ -332,6 +332,7 @@ describe("expiring a KV forget ledger", () => { { name: "due", metadata: at(2), expiration: (NOW + 33 * DAY) / 1000 }, { name: "soonest", metadata: soonest }, { name: "stray" }, + { name: "empty", metadata: null }, { name: "unreadable", metadata: { forgottenAt: "not a time" } }, { name: "untimed", metadata: { forgottenAt: 3 } }, { name: "text", metadata: "kept" }, diff --git a/packages/core/src/adapter/adapter.ts b/packages/core/src/adapter/adapter.ts index 00059c02..a58b43bb 100644 --- a/packages/core/src/adapter/adapter.ts +++ b/packages/core/src/adapter/adapter.ts @@ -217,6 +217,10 @@ export interface IsAdapterFunction { (value: unknown): value is Adapter; } +/** + * Whether a value is an adapter with its factories, as `sqlite(...)` returns, rather than a bare + * definition: what opens a project's storage outside the app, as the CLI does. + */ export const isAdapter: IsAdapterFunction = (value): value is Adapter => typeof value === "object" && value !== null && diff --git a/packages/core/src/adapter/index.ts b/packages/core/src/adapter/index.ts index 10d68117..23545dbf 100644 --- a/packages/core/src/adapter/index.ts +++ b/packages/core/src/adapter/index.ts @@ -3,6 +3,7 @@ export type { CommitReadModelRebuildArgs, CreateReadModelArgs, CreateReadModelRebuildArgs, + IsAdapterFunction, ReadModelRebuild, ReadModelStorage, ReadModelTransactArgs, @@ -11,6 +12,7 @@ export type { Storage, StorageTransaction, } from "./adapter.ts"; +export { isAdapter } from "./adapter.ts"; export type { AdapterDefinition } from "./adapter-definition.ts"; export type { RebuildFencing, RebuildFencingFunction } from "./rebuild-fencing.ts"; export { rebuildFencing } from "./rebuild-fencing.ts"; diff --git a/packages/core/src/adapter/sql/forget-ledger.ts b/packages/core/src/adapter/sql/forget-ledger.ts index a3a31bc3..0d636e65 100644 --- a/packages/core/src/adapter/sql/forget-ledger.ts +++ b/packages/core/src/adapter/sql/forget-ledger.ts @@ -68,8 +68,7 @@ const entryOf = (row: Record): ForgetLedgerEntry => { forgottenAt: String(row.forgotten_at), reason: row.reason === "retention" ? "retention" : "requested", }; - const fingerprints: unknown = - typeof row.fingerprints === "string" ? JSON.parse(row.fingerprints) : null; + const fingerprints: unknown = JSON.parse(String(row.fingerprints ?? null)); return Array.isArray(fingerprints) ? { ...entry, fingerprints: fingerprints.map(String) } : entry; }; diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index 7ab9b2af..577a0041 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -2,7 +2,12 @@ import type { ObservableResult } from "@opentelemetry/api"; import { isAdapter } from "../adapter/adapter.ts"; import { checkConfigNames } from "../config/names.ts"; import { resolveConfig } from "../config/schema.ts"; -import type { Config, ResolvedConfig, RuntimeRole } from "../config/types.ts"; +import type { + Config, + ResolvedConfig, + ResolvedForgetLedgerConfig, + RuntimeRole, +} from "../config/types.ts"; import { type Clock, systemClock } from "../contracts/clock.ts"; import type { CommandRejection, DispatchResult, StoredDispatch } from "../contracts/command.ts"; import { ConfigurationError, ForgetLedgerUnreconciledError } from "../contracts/errors.ts"; @@ -30,6 +35,7 @@ import { createPersonalDataBackfill } from "./personal-data/backfill.ts"; import { exportPersonalData, type PersonalDataExport } from "./personal-data/export.ts"; import { forgetPersonalData, logForgotten } from "./personal-data/forget.ts"; import { + type BoundedForgetLedger, boundForgetLedger, openForgetLedger, waitForLedger, @@ -460,25 +466,30 @@ export const assembleApp: AssembleAppFunction = async ({ opened.unshift(() => adapted.close()); const ledgerConfig = config.runtime.personalData.forgetLedger; const keepForMs = ledgerConfig?.keepForMs ?? null; - const ledger = await openForgetLedger({ - source: ledgerConfig?.store ?? null, - storage: config.storage, - env, - tenant, - keepForMs: keepForMs ?? undefined, - }); - if (ledger !== null) witness?.onForgetLedger(ledger); - if (ledger?.close !== undefined) { - const close = ledger.close.bind(ledger); - opened.unshift(close); + let forgetLedger: { + readonly config: ResolvedForgetLedgerConfig; + readonly store: BoundedForgetLedger; + } | null = null; + if (ledgerConfig !== null) { + const ledger = await openForgetLedger({ + source: ledgerConfig.store, + storage: config.storage, + env, + tenant, + keepForMs: keepForMs ?? undefined, + }); + witness?.onForgetLedger(ledger); + if (ledger.close !== undefined) opened.unshift(ledger.close.bind(ledger)); + forgetLedger = { + config: ledgerConfig, + store: boundForgetLedger({ ledger, timeoutMs: ledgerConfig.timeoutMs, clock }), + }; } - const bounded = - ledger === null || ledgerConfig === null - ? null - : boundForgetLedger({ ledger, timeoutMs: ledgerConfig.timeoutMs, clock }); const intentions = createForgetIntentions({ - ledger: bounded, - strict: ledgerConfig?.strict ?? false, + ledger: + forgetLedger === null + ? null + : { store: forgetLedger.store, strict: forgetLedger.config.strict }, ids, clock, logger, @@ -624,22 +635,26 @@ export const assembleApp: AssembleAppFunction = async ({ batchSize: config.runtime.dispatcher.batchSize, checkpoints: written.checkpointStore, }); - const reconciliation = - bounded === null || ledgerConfig === null - ? undefined - : createForgetLedgerReconciliation({ - ledger: bounded, - intentions, - retentionDue, - storage: written, - ids, - clock, - logger, - concurrencyRetries, - reconcileEveryMs: ledgerConfig.reconcileEveryMs, - keepForMs, - forgotten: wrapping.drop, - }); + const reconciling = + forgetLedger === null + ? null + : { + config: forgetLedger.config, + reconciliation: createForgetLedgerReconciliation({ + ledger: forgetLedger.store, + intentions, + retentionDue, + storage: written, + ids, + clock, + logger, + concurrencyRetries, + reconcileEveryMs: forgetLedger.config.reconcileEveryMs, + keepForMs, + forgotten: wrapping.drop, + }), + }; + const reconciliation = reconciling?.reconciliation; const rotation = wrapping.wrapper === null ? undefined @@ -695,7 +710,9 @@ export const assembleApp: AssembleAppFunction = async ({ ...(retention === undefined ? [] : [createRetentionSubscriber({ persons: storage.persons })]), - ...(bounded === null ? [] : [createLedgerOutbox({ ledger: bounded, keepForMs, clock })]), + ...(forgetLedger === null + ? [] + : [createLedgerOutbox({ ledger: forgetLedger.store, keepForMs, clock })]), ], batchSize: config.runtime.dispatcher.batchSize, pollIntervalMs: config.runtime.dispatcher.pollIntervalMs, @@ -791,19 +808,17 @@ export const assembleApp: AssembleAppFunction = async ({ // the read models that show them hear of it by the time the start resolves. A ledger that // does not answer in time is left to finish behind the start, as one that fails is retried, // unless the ledger is strict, whose start waits and fails with it. - if (reconciliation !== undefined) { - const reconciling = reconciliation.reconcile(clock.now()); - const timeoutMs = ledgerConfig?.timeoutMs ?? 0; - const forgotten = ledgerConfig?.strict - ? await reconciling - : await waitForLedger(clock, timeoutMs, reconciling); + if (reconciling !== null) { + const { strict, timeoutMs } = reconciling.config; + const started = reconciling.reconciliation.reconcile(clock.now()); + const forgotten = strict ? await started : await waitForLedger(clock, timeoutMs, started); if (forgotten === "late") { logger.error( "forget ledger did not answer in time; the app starts and reconciles in the background", { timeoutMs }, ); } else if (typeof forgotten !== "number") { - if (ledgerConfig?.strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); + if (strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); } else if (forgotten > 0) { await dispatcher.catchUp("projection"); } @@ -961,7 +976,7 @@ export const assembleApp: AssembleAppFunction = async ({ await storage.close(); } finally { wrapping.close(); - await ledger?.close?.(); + await forgetLedger?.store.close?.(); } } } finally { diff --git a/packages/core/src/kernel/command/reaction-commands.test.ts b/packages/core/src/kernel/command/reaction-commands.test.ts index ae0e9db1..899e3915 100644 --- a/packages/core/src/kernel/command/reaction-commands.test.ts +++ b/packages/core/src/kernel/command/reaction-commands.test.ts @@ -68,18 +68,20 @@ const withLedger = async () => { ids, clock: harness.clock, intentions: createForgetIntentions({ - ledger: boundForgetLedger({ - ledger: { - append: async (appended) => { - recorded.push(...appended); - keysAtRecord.push(await harness.storage.personalKeys.get("c-1")); + ledger: { + store: boundForgetLedger({ + ledger: { + append: async (appended) => { + recorded.push(...appended); + keysAtRecord.push(await harness.storage.personalKeys.get("c-1")); + }, + read: async () => [], }, - read: async () => [], - }, - timeoutMs: 1_000, - clock: harness.clock, - }), - strict: true, + timeoutMs: 1_000, + clock: harness.clock, + }), + strict: true, + }, ids, clock: harness.clock, logger, @@ -434,7 +436,11 @@ describe("createReactionCommands", () => { expect(recorded.map(({ fingerprints }) => fingerprints)).toEqual([[]]); expect(await harness.storage.personalKeys.get("c-1")).toBeNull(); const [forgotten] = await forgottenEvents(harness.storage); - expect(forgotten?.payload).toMatchObject({ fingerprints: ["f-2"] }); + expect(forgotten).toMatchObject({ + id: recorded[0]?.eventId, + timestamp: recorded[0]?.forgottenAt, + payload: { reason: "requested", fingerprints: ["f-2"] }, + }); }); it("answers nothing more for a forgetting a reconciliation committed under the intention's id", async () => { diff --git a/packages/core/src/kernel/personal-data/forget-ledger.ts b/packages/core/src/kernel/personal-data/forget-ledger.ts index 2b796707..d0f1677d 100644 --- a/packages/core/src/kernel/personal-data/forget-ledger.ts +++ b/packages/core/src/kernel/personal-data/forget-ledger.ts @@ -67,12 +67,12 @@ export const isForgetLedgerSource: IsForgetLedgerSourceFunction = ( }; export interface OpenForgetLedgerArgs extends CreateForgetLedgerFromEnvArgs { - readonly source: ForgetLedgerSource | null; + readonly source: ForgetLedgerSource; readonly storage: AdapterDefinition; } export interface OpenForgetLedgerFunction { - (args: OpenForgetLedgerArgs): Promise; + (args: OpenForgetLedgerArgs): Promise; } // The same adapter, or one of the same kind with the same options: a database a restore rolls back @@ -93,7 +93,6 @@ export const openForgetLedger: OpenForgetLedgerFunction = async ({ tenant, keepForMs, }) => { - if (source === null) return null; if (source.kind === "bounda-adapter" && sharesStorage(source, storage)) { throw new ConfigurationError( "runtime.personalData.forgetLedger.store points at the storage's own database, which a restore rolls back with it: give it another database", diff --git a/packages/core/src/kernel/personal-data/forget.test.ts b/packages/core/src/kernel/personal-data/forget.test.ts index 1fd7e136..6bbf0f4a 100644 --- a/packages/core/src/kernel/personal-data/forget.test.ts +++ b/packages/core/src/kernel/personal-data/forget.test.ts @@ -15,7 +15,7 @@ import { ConcurrencyError, ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { memory } from "../../memory/index.ts"; import { createRecordingLogger, withoutLedger } from "../test-support.ts"; -import { forgetIntended, forgetPersonalData } from "./forget.ts"; +import { forgetIntended, forgetPersonalData, forgetWithin, STALE_INTENTION } from "./forget.ts"; import { boundForgetLedger } from "./forget-ledger.ts"; import { createForgetIntentions } from "./intention.ts"; @@ -202,17 +202,19 @@ describe("forgetIntended", () => { const recorded: ForgetLedgerEntry[] = []; const clock = createFixedClock(); const intentions = createForgetIntentions({ - ledger: boundForgetLedger({ - ledger: { - append: async (entries) => { - recorded.push(...entries); + ledger: { + store: boundForgetLedger({ + ledger: { + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], }, - read: async () => [], - }, - timeoutMs: 1_000, - clock, - }), - strict: true, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, ids, clock, logger: createRecordingLogger().logger, @@ -325,4 +327,44 @@ describe("forgetIntended", () => { expect(await storage.eventStore.lastPosition()).toBe(1); expect(logs).toEqual([]); }); + + it("reads the held event and checks the key only when its caller has not", async () => { + const storage = await memory().createStorage(); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + let reads = 0; + const forget = (checked: boolean, fingerprints: readonly string[]) => + storage.transact((transaction) => { + const { readOne } = transaction.eventStore; + return forgetWithin({ + transaction: { + ...transaction, + eventStore: { + ...transaction.eventStore, + readOne: (args) => { + reads += 1; + return readOne.call(transaction.eventStore, args); + }, + }, + }, + ids: createSequentialIdGenerator(), + clock: createFixedClock(), + personId: "c-1", + intention: { + eventId: `e-${checked}`, + forgottenAt: "t", + reason: "requested", + fingerprints, + }, + context: testContext, + checked, + }); + }); + expect(await forget(false, ["f-other"])).toBe(STALE_INTENTION); + expect(reads).toBe(1); + expect(await forget(true, ["f-other"])).toMatchObject({ + hadKey: true, + event: { id: "e-true" }, + }); + expect(reads).toBe(1); + }); }); diff --git a/packages/core/src/kernel/personal-data/forget.ts b/packages/core/src/kernel/personal-data/forget.ts index 400b5959..71b936c9 100644 --- a/packages/core/src/kernel/personal-data/forget.ts +++ b/packages/core/src/kernel/personal-data/forget.ts @@ -24,6 +24,11 @@ export interface ForgetWithinArgs { readonly personId: string; readonly intention: ForgetIntention; readonly context: CausationContext; + /** + * The caller found already, in this transaction, the intention's event not held and the + * person's key named by it. + */ + readonly checked?: boolean; } export interface Forgotten { @@ -65,9 +70,10 @@ export const forgetWithin: ForgetWithinFunction = async ({ personId, intention, context, + checked = false, }) => { const { fingerprints } = intention; - if (fingerprints !== undefined) { + if (fingerprints !== undefined && !checked) { const held = await transaction.eventStore.readOne({ aggregateType: BOUNDA_AGGREGATE, aggregateId: personId, @@ -76,7 +82,12 @@ export const forgetWithin: ForgetWithinFunction = async ({ if (held !== null) return { held }; } const key = await transaction.personalKeys.get(personId); - if (key !== null && fingerprints !== undefined && !fingerprints.includes(key.fingerprint)) { + if ( + !checked && + key !== null && + fingerprints !== undefined && + !fingerprints.includes(key.fingerprint) + ) { return STALE_INTENTION; } // The key goes first: a snapshot written beside the transaction then either waits for it and diff --git a/packages/core/src/kernel/personal-data/intention.test.ts b/packages/core/src/kernel/personal-data/intention.test.ts index 249ffb33..fb66b940 100644 --- a/packages/core/src/kernel/personal-data/intention.test.ts +++ b/packages/core/src/kernel/personal-data/intention.test.ts @@ -21,8 +21,7 @@ const setUp = async ({ const storage = await memory().createStorage(); const { logger, entries: logs } = createRecordingLogger(); const intentions = createForgetIntentions({ - ledger, - strict, + ledger: ledger === null ? null : { store: ledger, strict }, ids: createSequentialIdGenerator(), clock: createFixedClock(NOW), logger, diff --git a/packages/core/src/kernel/personal-data/intention.ts b/packages/core/src/kernel/personal-data/intention.ts index 4b405a7f..e20ba631 100644 --- a/packages/core/src/kernel/personal-data/intention.ts +++ b/packages/core/src/kernel/personal-data/intention.ts @@ -49,8 +49,10 @@ export interface CreateForgetIntentionsArgs { /** * `null` without a ledger: nothing is recorded, and no key is read for it. */ - readonly ledger: BoundedForgetLedger | null; - readonly strict: boolean; + readonly ledger: { + readonly store: BoundedForgetLedger; + readonly strict: boolean; + } | null; readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; @@ -68,15 +70,16 @@ export interface CreateForgetIntentionsFunction { */ export const createForgetIntentions: CreateForgetIntentionsFunction = ({ ledger, - strict, ids, clock, logger, }) => { const record = async (entry: ForgetLedgerEntry): Promise => { - if (ledger === null || (!strict && ledger.late())) return; + if (ledger === null) return; + const { store, strict } = ledger; + if (!strict && store.late()) return; try { - await ledger.append([entry]); + await store.append([entry]); } catch (error) { if (strict) throw new ForgetLedgerUnavailableError(error); logger.warn( diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts index 8d22ae0b..2c67abd2 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts @@ -212,7 +212,6 @@ describe("a forget ledger's source", () => { await (await adapter.createForgetLedger({ tenant: "t-1" })).read({ limit: 1 }), ).toHaveLength(1); expect(await (await adapter.createForgetLedger({})).read({ limit: 1 })).toEqual([]); - expect(await openForgetLedger({ source: null, storage: memory(), env })).toBeNull(); }); it("refuses an adapter that points at the storage's own database", async () => { diff --git a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts index 837e5f2b..4ef8e836 100644 --- a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts @@ -661,11 +661,19 @@ describe("the forget ledger", () => { const ledgerAdapter = memory(); const { app, register, forgotten } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); + await register("c-2"); await app.forgetAgain([{ personId: "c-1" }]); expect(await forgotten()).toEqual([["c-1", "requested"]]); expect( (await (await opened(ledgerAdapter)).read({ limit: 10 })).map(({ personId }) => personId), ).toEqual(["c-1"]); + const elsewhere = entry("c-2", "from-elsewhere", LOST); + await app.forgetAgain([elsewhere]); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toContainEqual(elsewhere); + expect(await forgotten()).toEqual([ + ["c-1", "requested"], + ["c-2", "requested"], + ]); }); it("records what the retention forgets before it commits, and the stream copies it", async () => { @@ -1248,9 +1256,14 @@ describe("the intention a forgetting records first", () => { it("commits again a forgetting the retention made only while the retention would still forget the person", async () => { const storage = memory(); const ledgerAdapter = memory(); - const config = (retention: "1h" | "2h") => ({ + const config = (retention?: "1h" | "2h") => ({ storage, - runtime: { personalData: { retention, forgetLedger: { store: ledgerAdapter } } }, + runtime: { + personalData: { + ...(retention === undefined ? {} : { retention }), + forgetLedger: { store: ledgerAdapter }, + }, + }, }); const clock = createFixedClock(START); const first = await createApp({ registry, config: config("1h"), clock }); @@ -1276,6 +1289,13 @@ describe("the intention a forgetting records first", () => { ), ).toEqual([]); await later.stop(); + const without = await createApp({ registry, config: config(), clock }); + expect( + (await without.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ), + ).toEqual([]); + await without.stop(); const due = await createApp({ registry, config: config("1h"), clock }); stops.push(() => due.stop()); expect( diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index d5ec8c65..27a5466e 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -769,24 +769,36 @@ describe("forgetAgain", () => { }); const ledgered = async () => { - const storage = await memory().createStorage(); + const base = await memory().createStorage(); + let transactions = 0; + const beforeTransaction: (() => Promise)[] = []; + const storage: Storage = { + ...base, + transact: async (work) => { + transactions += 1; + await beforeTransaction.shift()?.(); + return base.transact(work); + }, + }; const recorded: ForgetLedgerEntry[] = []; const ids = createSequentialIdGenerator(); const clock = createFixedClock(NOW); let down = false; const intentions = createForgetIntentions({ - ledger: boundForgetLedger({ - ledger: { - append: async (entries) => { - if (down) throw new Error("ledger is down"); - recorded.push(...entries); + ledger: { + store: boundForgetLedger({ + ledger: { + append: async (entries) => { + if (down) throw new Error("ledger is down"); + recorded.push(...entries); + }, + read: async () => [], }, - read: async () => [], - }, - timeoutMs: 1_000, - clock, - }), - strict: true, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, ids, clock, logger: silentLogger, @@ -806,6 +818,11 @@ describe("forgetAgain", () => { storage, recorded, run, + intentions, + ids, + clock, + transactions: () => transactions, + beforeTransaction: (step: () => Promise) => beforeTransaction.push(step), fail: () => { down = true; }, @@ -828,12 +845,12 @@ describe("forgetAgain", () => { await keyed(storage, ["c-1"]); expect(await run({ personId: "c-2" })).toBeNull(); expect(await run({ personId: "c-1", eventId: "elsewhere" })).toMatchObject({ - event: { id: "id-1", payload: { reason: "requested", fingerprints: ["f-c-1"] } }, + event: { id: "id-2", payload: { reason: "requested", fingerprints: ["f-c-1"] } }, }); expect(recorded).toEqual([ { personId: "c-1", - eventId: "id-1", + eventId: "id-2", forgottenAt: NOW.toISOString(), reason: "requested", fingerprints: ["f-c-1"], @@ -862,4 +879,93 @@ describe("forgetAgain", () => { expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).toBeNull(); expect(await storage.personalKeys.get("c-1")).not.toBeNull(); }); + + it("leaves a key the entry does not name without a transaction", async () => { + const { storage, run, transactions } = await ledgered(); + await keyed(storage, ["c-1"]); + const keying = transactions(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-other"] })).toBeNull(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: [] })).toBeNull(); + expect(transactions()).toBe(keying); + }); + + it("leaves an entry naming no keys whose forgetting the store holds, whatever key the person holds", async () => { + const { storage, run } = await ledgered(); + await keyed(storage, ["c-1"]); + const legacy = entry("c-1", "e-1", at(3)); + expect(await run(legacy)).not.toBeNull(); + await keyed(storage, ["c-1"]); + expect(await run(legacy)).toBeNull(); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); + + it("takes an intention of its own for an entry missing its id, its time or its reason", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1", "c-2", "c-3"]); + const { forgottenAt, eventId } = entry("c-1", "e-1", at(3)); + await run({ personId: "c-1", eventId, reason: "requested" }); + await run({ personId: "c-2", forgottenAt, reason: "requested" }); + await run({ personId: "c-3", eventId: "e-3", forgottenAt }); + expect( + recorded.map(({ personId, eventId: id, forgottenAt: at, reason }) => [ + personId, + id, + at, + reason, + ]), + ).toEqual([ + ["c-1", "id-2", NOW.toISOString(), "requested"], + ["c-2", "id-4", NOW.toISOString(), "requested"], + ["c-3", "id-6", NOW.toISOString(), "requested"], + ]); + }); + + it("takes a new intention for a bare id when the person is given a key between the intention and the commit", async () => { + const { storage, recorded, run, beforeTransaction } = await ledgered(); + await keyed(storage, ["c-1"]); + beforeTransaction(async () => { + await storage.personalKeys.remove("c-1"); + await storage.personalKeys.add("c-1", { key: "k-2", fingerprint: "f-2", kid: null }); + }); + expect(await run({ personId: "c-1" })).toMatchObject({ + event: { id: "id-3", payload: { fingerprints: ["f-2"] } }, + }); + expect(recorded.map(({ eventId, fingerprints }) => [eventId, fingerprints])).toEqual([ + ["id-2", ["f-c-1"]], + ["id-3", ["f-2"]], + ]); + }); + + it("records nothing as it reconciles: the entries are the ledger's own", async () => { + const { storage, recorded, intentions, ids, clock } = await ledgered(); + await keyed(storage, ["c-1"]); + const reconciliation = createForgetLedgerReconciliation({ + ledger: recordingLedger([{ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] }]).ledger, + intentions, + retentionDue: NEVER_DUE, + storage, + ids, + clock, + logger: silentLogger, + concurrencyRetries: 3, + reconcileEveryMs: EVERY, + keepForMs: null, + }); + expect(await reconciliation.reconcile(NOW)).toBe(1); + expect(recorded).toEqual([]); + }); + + it("never commits again a forgetting the retention made without a retention", async () => { + expect(await NEVER_DUE(await memory().createStorage(), "c-1", NOW)).toBe(false); + }); + + it("forgets nobody whose key went between its check and its transaction", async () => { + const { storage, run, beforeTransaction } = await ledgered(); + await keyed(storage, ["c-1"]); + beforeTransaction(async () => { + await storage.personalKeys.remove("c-1"); + }); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).toBeNull(); + expect(await storage.eventStore.lastPosition()).toBe(0); + }); }); diff --git a/packages/core/src/kernel/personal-data/reconciliation.ts b/packages/core/src/kernel/personal-data/reconciliation.ts index 9a2a0473..ee3e533e 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.ts @@ -146,39 +146,40 @@ export const forgetAgain: ForgetAgainFunction = async ({ return reason !== "retention" || retentionDue(stores, personId, clock.now()); }; if (!(await isDue(storage, fingerprints))) return null; - let intention: ForgetIntention; - if (eventId !== undefined && forgottenAt !== undefined && reason !== undefined) { + const whole = eventId !== undefined && forgottenAt !== undefined && reason !== undefined; + if (whole && record) { const recorded = { personId, eventId, forgottenAt, reason }; - if (record) { - await intentions.record( - fingerprints === undefined ? recorded : { ...recorded, fingerprints }, - ); - } - intention = { eventId, forgottenAt, reason, fingerprints }; - } else { - intention = await intentions.intend({ - personId, - reason: reason ?? "requested", - keys: storage.personalKeys, - }); + await intentions.record(fingerprints === undefined ? recorded : { ...recorded, fingerprints }); } const correlationId = ids.next(); - return transactRetrying({ - storage, - concurrencyRetries, - work: async (transaction) => { - if (!(await isDue(transaction, intention.fingerprints ?? fingerprints))) return null; - const outcome = await forgetWithin({ - transaction, - ids, - clock, - personId, - intention, - context: startedBy(correlationId, actor), - }); - return outcome === STALE_INTENTION || "held" in outcome ? null : outcome; - }, - }); + // A whole entry is due only while the person holds a key it names, so it is never stale; an + // intention of its own takes another that names a key given meanwhile, as `forget` does. + for (;;) { + const intention: ForgetIntention = whole + ? { eventId, forgottenAt, reason, fingerprints } + : await intentions.intend({ + personId, + reason: reason ?? "requested", + keys: storage.personalKeys, + }); + const outcome = await transactRetrying({ + storage, + concurrencyRetries, + work: async (transaction) => + (await isDue(transaction, fingerprints)) + ? forgetWithin({ + transaction, + ids, + clock, + personId, + intention, + context: startedBy(correlationId, actor), + checked: whole, + }) + : null, + }); + if (outcome !== STALE_INTENTION) return outcome === null || "held" in outcome ? null : outcome; + } }; export interface ReconciliationFailed { @@ -307,7 +308,8 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF for (const entry of page) { const time = Date.parse(entry.forgottenAt); if (time + FORGET_LEDGER_GRACE_MS > at) { - youngest = Math.max(youngest ?? time, time); + // Entries come in the order of their time: the last one left is the youngest. + youngest = time; continue; } const again = await forgetAgain({ diff --git a/packages/core/src/kernel/personal-data/retention.test.ts b/packages/core/src/kernel/personal-data/retention.test.ts index aca5a827..795710d9 100644 --- a/packages/core/src/kernel/personal-data/retention.test.ts +++ b/packages/core/src/kernel/personal-data/retention.test.ts @@ -778,17 +778,19 @@ describe("the retention and the forget ledger", () => { const recordingIntentions = (ids: IdGenerator, clock: FixedClock) => { const recorded: ForgetLedgerEntry[] = []; const intentions = createForgetIntentions({ - ledger: boundForgetLedger({ - ledger: { - append: async (entries) => { - recorded.push(...entries); + ledger: { + store: boundForgetLedger({ + ledger: { + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], }, - read: async () => [], - }, - timeoutMs: 1_000, - clock, - }), - strict: true, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, ids, clock, logger: silentLogger, @@ -851,4 +853,130 @@ describe("the retention and the forget ledger", () => { await storage.persons.record({ person: "c-4", position: 99, forgottenAt: day(30) }); expect(await isDue("c-4")).toBe(false); }); + + // A sweep through a store that runs `meanwhile` once, between the list and the person's forget, + // or before the forget's transaction. + const racing = async ( + when: "listed" | "transaction", + meanwhile: (storage: Storage) => Promise, + ) => { + const setup = await setUp(); + const { storage, ids, clock, logger } = setup; + let pending: ((storage: Storage) => Promise) | undefined = meanwhile; + const once = async () => { + const step = pending; + pending = undefined; + await step?.(storage); + }; + const raced: Storage = { + ...storage, + persons: { + ...storage.persons, + listIdle: async (args) => { + const listed = await storage.persons.listIdle(args); + if (when === "listed" && listed.length > 0) await once(); + return listed; + }, + }, + transact: async (work) => { + if (when === "transaction") await once(); + return storage.transact(work); + }, + }; + const { intentions, recorded } = recordingIntentions(ids, clock); + const sweep = createRetentionSweep({ + storage: raced, + intentions, + retentionMs: 30 * DAY, + ids, + clock, + logger, + concurrencyRetries: 1, + batchSize: 2, + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + await setup.touch("c-1"); + await setup.record(); + setup.advance(31); + return { ...setup, sweep, recorded }; + }; + + it("records no intention for a person restricted between the list and their forget", async () => { + const { sweep, clock, recorded, forgottenFor } = await racing("listed", (storage) => + storage.persons.restrict("c-1", new Date(day(31))).then(() => undefined), + ); + expect(await sweep.sweep(clock.now())).toBe(0); + expect(recorded).toEqual([]); + expect(await forgottenFor()).toEqual([]); + }); + + it("forgets nobody whose transaction finds them restricted after the intention, nor given a key since, and holds them", async () => { + const restricted = await racing("transaction", (storage) => + storage.persons.restrict("c-1", new Date(day(31))).then(() => undefined), + ); + expect(await restricted.sweep.sweep(restricted.clock.now())).toBe(0); + expect(restricted.recorded).toHaveLength(1); + expect(await restricted.forgottenFor()).toEqual([]); + + const rekeyed = await racing("transaction", async (storage) => { + await storage.personalKeys.remove("c-1"); + await storage.personalKeys.add("c-1", testPersonalKey("2")); + }); + expect(await rekeyed.sweep.sweep(rekeyed.clock.now())).toBe(0); + expect(rekeyed.entries.filter(({ level }) => level === "error")).toEqual([]); + expect(await rekeyed.forgottenFor()).toEqual([]); + expect(await rekeyed.storage.personalKeys.get("c-1")).toEqual(testPersonalKey("2")); + const [row] = await rekeyed.storage.persons.listIdle({ limit: 1 }); + expect(row?.hold).not.toBeNull(); + expect(await rekeyed.sweep.sweep(rekeyed.clock.now())).toBe(0); + }); + + it("leaves a person a reconciliation forgot under the intention's id meanwhile", async () => { + let intended = ""; + const { sweep, clock, recorded, forgottenFor, storage } = await racing( + "transaction", + async (store) => { + intended = recorded[0]?.eventId ?? ""; + await store.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + { + ...forgotten(99, "c-1"), + aggregateId: "c-1", + id: intended, + payload: { personId: "c-1", reason: "retention" }, + }, + ], + }); + }, + ); + expect(await sweep.sweep(clock.now())).toBe(0); + expect((await forgottenFor()).map(([person]) => person)).toEqual(["c-1"]); + expect(intended).not.toBe(""); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); + + it("says a person whose row never saw them active is not due", async () => { + const { storage, clock } = await setUp(); + await storage.persons.record({ person: "c-5", position: 1, forgottenAt: null }); + const due = retentionDueOf({ + retentionMs: DAY, + batchSize: 2, + checkpoints: storage.checkpointStore, + }); + expect(await due(storage, "c-5", new Date(clock.now().getTime() + 2 * DAY))).toBe(false); + }); + + it("records no intention for a person active since their row was recorded", async () => { + const { sweep, clock, recorded, forgottenFor, touch } = await racing( + "listed", + async () => undefined, + ); + await touch("c-1"); + expect(await sweep.sweep(clock.now())).toBe(0); + expect(recorded).toEqual([]); + expect(await forgottenFor()).toEqual([]); + }); }); diff --git a/packages/core/src/kernel/personal-data/retention.ts b/packages/core/src/kernel/personal-data/retention.ts index b8de8b2b..78ebd19b 100644 --- a/packages/core/src/kernel/personal-data/retention.ts +++ b/packages/core/src/kernel/personal-data/retention.ts @@ -180,14 +180,15 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ }) => { // Resolves `undone` for a person whose row is behind, or who was given a key since the // intention, who is then left for a later pass, and `skipped` for one restricted, or forgotten by - // another instance, since the list was read. The row is read once before the intention, so a - // person plainly skipped leaves none; an intention left by a transaction that skips is committed - // again only if the retention would still forget the person. + // another instance, since the list was read. Both are decided once before the intention, so a + // person plainly left leaves none, and again in the transaction; an intention it then leaves is + // committed again only if the retention would still forget the person. const forget = async ( personId: string, through: number, ): Promise<"forgotten" | "skipped" | "undone"> => { if (!isIdle(await storage.persons.get(personId))) return "skipped"; + if (!(await rowIsCurrent(storage.eventStore, personId, through, batchSize))) return "undone"; const intention = await intentions.intend({ personId, reason: "retention", diff --git a/packages/core/src/kernel/reactive-harness.ts b/packages/core/src/kernel/reactive-harness.ts index 82f88088..fdfec034 100644 --- a/packages/core/src/kernel/reactive-harness.ts +++ b/packages/core/src/kernel/reactive-harness.ts @@ -105,7 +105,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ logger, }); const policies = buildPolicies({ registry, aggregates }); - const intentions = createForgetIntentions({ ledger: null, strict: false, ids, clock, logger }); + const intentions = createForgetIntentions({ ledger: null, ids, clock, logger }); const policyExecutor = createPolicyExecutor({ aggregates, pipeline, diff --git a/packages/core/src/kernel/test-support.ts b/packages/core/src/kernel/test-support.ts index c2dcfcb1..9273b0b0 100644 --- a/packages/core/src/kernel/test-support.ts +++ b/packages/core/src/kernel/test-support.ts @@ -496,4 +496,4 @@ export interface WithoutLedgerArgs { * The intentions of an app without a forget ledger: ids and times, nothing recorded. */ export const withoutLedger = ({ ids, clock }: WithoutLedgerArgs): ForgetIntentions => - createForgetIntentions({ ledger: null, strict: false, ids, clock, logger: silentLogger }); + createForgetIntentions({ ledger: null, ids, clock, logger: silentLogger }); diff --git a/packages/core/src/memory/forget-ledger.ts b/packages/core/src/memory/forget-ledger.ts index 524a946e..1ef7664c 100644 --- a/packages/core/src/memory/forget-ledger.ts +++ b/packages/core/src/memory/forget-ledger.ts @@ -33,13 +33,7 @@ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () = return { append: async (appended) => { for (const entry of appended) { - if (entries.has(entry.eventId)) continue; - entries.set( - entry.eventId, - entry.fingerprints === undefined - ? { ...entry } - : { ...entry, fingerprints: [...entry.fingerprints] }, - ); + if (!entries.has(entry.eventId)) entries.set(entry.eventId, { ...entry }); } }, read: async ({ after, limit }) => @@ -47,11 +41,7 @@ export const createMemoryForgetLedgers: CreateMemoryForgetLedgersFunction = () = .filter((entry) => after === undefined || byForgetLedgerOrder(entry, after) > 0) .sort(byForgetLedgerOrder) .slice(0, limit) - .map((entry) => - entry.fingerprints === undefined - ? { ...entry } - : { ...entry, fingerprints: [...entry.fingerprints] }, - ), + .map((entry) => ({ ...entry })), prune: async ({ before }) => { for (const [eventId, entry] of entries) { if (entry.forgottenAt < before) entries.delete(eventId); From f74f4a9125f31f6c84ace310c8428f982dcad09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:36:34 +0100 Subject: [PATCH 3/6] test: give the forget ledger entries of expectForgotten's tests their reason --- packages/core/src/adapter/testing/inspection.contract.ts | 8 +++++++- packages/core/src/testing/expect-forgotten.test.ts | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/core/src/adapter/testing/inspection.contract.ts b/packages/core/src/adapter/testing/inspection.contract.ts index cd1ebda7..990f730a 100644 --- a/packages/core/src/adapter/testing/inspection.contract.ts +++ b/packages/core/src/adapter/testing/inspection.contract.ts @@ -146,7 +146,13 @@ export const inspectionContract: InspectionContractFunction = ({ create }) => { const adapter = await create(); const opened = await adapter.createForgetLedger({ tenant: "t-1" }); await opened.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: "2026-11-07T12:00:00.000Z" }, + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-11-07T12:00:00.000Z", + reason: "requested", + fingerprints: [], + }, ]); const ledger = tableOf(await adapter.inspect(), "bounda_forget_ledger"); const { rows } = ledger; diff --git a/packages/core/src/testing/expect-forgotten.test.ts b/packages/core/src/testing/expect-forgotten.test.ts index 58099b98..f7eac456 100644 --- a/packages/core/src/testing/expect-forgotten.test.ts +++ b/packages/core/src/testing/expect-forgotten.test.ts @@ -803,6 +803,7 @@ describe("expectForgotten", () => { personId: `p-${index}`, eventId: `ledger-event-${String(index).padStart(4, "0")}`, forgottenAt: new Date(Date.UTC(2025, 0, 1, 0, 0, index)).toISOString(), + reason: "requested" as const, })), ); const { app } = await createTestApp({ From ae1f4267d7dbc5d0f47a3e9a759098bff796e083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 21:57:52 +0100 Subject: [PATCH 4/6] test(cli): expect forget-ledger expire's root as the platform resolves it --- packages/cli/src/run.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 795ab356..9a8718a1 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -909,8 +909,8 @@ describe("bounda forget-ledger expire", () => { expect(one.code).toBe(EXIT_FAILURE); expect(one.stderr).toBe("error: wrangler: not authorized\n"); expect(one.calls.map(({ root, args }) => [root, args.at(-1)])).toEqual([ - ["/project", "--remote"], - ["/project", "--remote"], + [resolve("/project"), "--remote"], + [resolve("/project"), "--remote"], ]); }); From 0a5e5ec7b909dd1b6ab4b19baf779a93880a1bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 22:19:48 +0100 Subject: [PATCH 5/6] ci: name each test as it starts in the Windows job, so a native crash says where it happened --- .github/workflows/ci.yml | 5 ++++- packages/cli/vitest.config.ts | 2 ++ packages/core/vitest.config.ts | 2 ++ packages/create-bounda/vitest.config.ts | 2 ++ packages/react-router/vitest.config.ts | 1 + packages/sqlite/vitest.config.ts | 2 ++ vitest.trace.ts | 11 +++++++++++ 7 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 vitest.trace.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cc6c36d..6277c163 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,9 @@ jobs: if: needs.changes.outputs.packages != '[]' runs-on: windows-latest timeout-minutes: 30 + env: + # A worker that dies in native code, as libSQL can there, names the test it was running. + BOUNDA_TEST_TRACE: "1" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 @@ -112,7 +115,7 @@ jobs: run: pnpm --filter @bounda-dev/core test - name: sqlite if: ${{ !cancelled() && steps.build.outcome == 'success' }} - run: pnpm --filter @bounda-dev/sqlite exec vitest run --exclude src/libsql-server.test.ts + run: pnpm --filter @bounda-dev/sqlite exec vitest run --exclude src/libsql-server.test.ts --reporter=verbose - name: cli if: ${{ !cancelled() && steps.build.outcome == 'success' }} run: pnpm --filter @bounda-dev/cli test diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 19a7de97..7127eee8 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "cli", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 80f5bb81..3af2fb93 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "core", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], typecheck: { enabled: true, include: ["test-types/**/*.test-d.ts"], diff --git a/packages/create-bounda/vitest.config.ts b/packages/create-bounda/vitest.config.ts index ed0ae200..e54e2b3a 100644 --- a/packages/create-bounda/vitest.config.ts +++ b/packages/create-bounda/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "create-bounda", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], testTimeout: 60_000, }, diff --git a/packages/react-router/vitest.config.ts b/packages/react-router/vitest.config.ts index 2d5a24ea..f46b5cc8 100644 --- a/packages/react-router/vitest.config.ts +++ b/packages/react-router/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ }, test: { name: "react-router", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/packages/sqlite/vitest.config.ts b/packages/sqlite/vitest.config.ts index 05541e08..88ce18a3 100644 --- a/packages/sqlite/vitest.config.ts +++ b/packages/sqlite/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "sqlite", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/vitest.trace.ts b/vitest.trace.ts new file mode 100644 index 00000000..94038c3c --- /dev/null +++ b/vitest.trace.ts @@ -0,0 +1,11 @@ +import { writeSync } from "node:fs"; +import { beforeEach } from "vitest"; + +// With BOUNDA_TEST_TRACE set, names each test as it starts, straight on the process's stderr: a +// test whose process dies in native code leaves its name behind, which the reporters, fed by the +// worker over RPC, lose with it. +if (process.env.BOUNDA_TEST_TRACE) { + beforeEach(({ task }) => { + writeSync(2, `[test] ${task.file.name} > ${task.name}\n`); + }); +} From 20f4f3d4e1dc812d12e20acb60b8b96d2faa1034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Be=C3=B1at=20Espi=C3=B1a?= Date: Mon, 9 Nov 2026 22:44:47 +0100 Subject: [PATCH 6/6] fix(sqlite): close the libSQL connection only once its work has settled Closing the client under a statement or transaction still running frees what libSQL's native code is using; a ledger call given up for its timeout and then closed with the app did just that. The connection now waits for everything it started before it closes. The two tests that opened a file from six worker threads and terminated them crashed the test process on Windows now and then (0xC0000005): libSQL's native code does not survive a thread that ends under it. They open the file from processes instead, as instances do. --- .changeset/forget-ledger-first.md | 2 + packages/sqlite/src/database.ts | 50 +++++--- packages/sqlite/src/index.ts | 26 ++-- packages/sqlite/src/sqlite.test.ts | 190 +++++++++++++++-------------- 4 files changed, 150 insertions(+), 118 deletions(-) diff --git a/.changeset/forget-ledger-first.md b/.changeset/forget-ledger-first.md index 6f1baace..c4cd141c 100644 --- a/.changeset/forget-ledger-first.md +++ b/.changeset/forget-ledger-first.md @@ -13,3 +13,5 @@ Ledger entries and `PersonalDataForgotten` name the keys a forgetting destroys b `forgetLedger.strict` now also means nothing is forgotten that the ledger has not recorded: a forgetting whose intention the ledger does not take fails with `ForgetLedgerUnavailableError` (`FORGET_LEDGER_UNAVAILABLE`) and forgets nobody, so trying again is safe. `forgetLedger.timeout` (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS`, ten seconds) bounds each call to the ledger, given up with `ForgetLedgerTimeoutError` (`FORGET_LEDGER_TIMEOUT`), and how long a start that is not strict waits for the reconciliation; a strict start fails once a read runs past it. `parseDuration` is exported, and `isAdapter` from `@bounda-dev/core/adapter`. `bounda forgotten` reads the event store without starting the app, so a strict ledger that is down does not stop it, and prints each forgetting's reason and fingerprints; `bounda forget --from` commits each whole line again under its own id, time and reason. `bounda forget-ledger expire --binding --keep-for ` gives every key of a Workers KV ledger the expiration `keepFor` sets, those written before it was set included, through the project's wrangler; `expireKvForgetLedger` and `planKvForgetLedgerExpiry` from `@bounda-dev/cloudflare`, and its new `./forget-ledger` subpath, do the same in code. + +`sqlite()` closes its libSQL connection only once the statements and transactions it started have settled, so closing an app or a ledger under a call still running, as one given up for its timeout, no longer frees what libSQL's native code is using. diff --git a/packages/sqlite/src/database.ts b/packages/sqlite/src/database.ts index db2937ff..7aebbbee 100644 --- a/packages/sqlite/src/database.ts +++ b/packages/sqlite/src/database.ts @@ -17,8 +17,16 @@ export interface CreateSqliteDatabaseOptions { readonly wal: boolean; } +export interface LibsqlConnection extends SqliteConnection { + /** + * Resolves once no statement or transaction it started is still running: closing the client + * under one frees what libSQL's native code is still using. + */ + settled(): Promise; +} + export interface CreateSqliteDatabaseFunction { - (client: Client, options: CreateSqliteDatabaseOptions): SqliteConnection; + (client: Client, options: CreateSqliteDatabaseOptions): LibsqlConnection; } const toRows = (result: ResultSet): readonly Record[] => @@ -52,6 +60,15 @@ const QUEUED_METHODS: ReadonlySet = new Set(["execute", "batch", "e export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { local, wal }) => { const serially = createSerialQueue(); + const running = new Set>(); + const tracked = (work: Promise): Promise => { + running.add(work); + const done = (): void => { + running.delete(work); + }; + work.then(done, done); + return work; + }; let ready: Promise | undefined; const prepared = (task: () => Promise): Promise => { if (ready === undefined && wal) { @@ -66,7 +83,7 @@ export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { loc return ready === undefined ? task() : ready.then(task); }; const outside = (task: () => Promise): Promise => - prepared(() => (local ? serially(task) : task())); + tracked(prepared(() => (local ? serially(task) : task()))); const direct = executorOf(client); const raw = local ? new Proxy(client, { @@ -83,24 +100,29 @@ export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { loc return { raw, isBusy, + settled: async () => { + while (running.size > 0) await Promise.allSettled([...running]); + }, db: { run: (sql, params) => outside(() => direct.run(sql, params)), all: (sql, params) => outside(() => direct.all(sql, params)), // One write transaction at a time within the process, so it never waits on the engine's busy // handler for another transaction of its own. write: (work) => - prepared(() => - serially(async () => { - const transaction = await client.transaction("write"); - try { - const result = await work({ ...executorOf(transaction), raw: transaction }); - await transaction.commit(); - return result; - } catch (error) { - await transaction.rollback(); - throw error; - } - }), + tracked( + prepared(() => + serially(async () => { + const transaction = await client.transaction("write"); + try { + const result = await work({ ...executorOf(transaction), raw: transaction }); + await transaction.commit(); + return result; + } catch (error) { + await transaction.rollback(); + throw error; + } + }), + ), ), }, }; diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 49c81ac3..e18687d3 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -1,13 +1,9 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import type { Adapter } from "@bounda-dev/core/adapter"; -import { - createSqliteAdapter, - SQLITE_BUSY_TIMEOUT_MS, - type SqliteConnection, -} from "@bounda-dev/core/adapter/sqlite"; +import { createSqliteAdapter, SQLITE_BUSY_TIMEOUT_MS } from "@bounda-dev/core/adapter/sqlite"; import { type Client, createClient } from "@libsql/client"; -import { createSqliteDatabase } from "./database.ts"; +import { createSqliteDatabase, type LibsqlConnection } from "./database.ts"; import { resolveSqliteOptions, type SqliteOptions } from "./options.ts"; /** @@ -20,7 +16,7 @@ export interface SqliteFunction { (options: SqliteOptions): SqliteAdapter; } -interface Connection extends SqliteConnection { +interface Connection extends LibsqlConnection { readonly client: Client; uses: number; } @@ -44,21 +40,25 @@ export const sqlite: SqliteFunction = (options) => { ...(authToken === undefined ? {} : { authToken }), ...(location === "file" ? { timeout: SQLITE_BUSY_TIMEOUT_MS } : {}), }); - const { db, raw } = createSqliteDatabase(client, { + const opened = createSqliteDatabase(client, { local: location !== "remote", wal: location === "file", }); - connection = { client, db, raw, uses: 0 }; + connection = { ...opened, client, uses: 0 }; } connection.uses += 1; return connection; }; + // The client closes once what it runs has settled, unless it was handed out again meanwhile. const release = async (): Promise => { - if (connection === null) return; - connection.uses -= 1; - if (connection.uses > 0) return; - connection.client.close(); + const releasing = connection; + if (releasing === null) return; + releasing.uses -= 1; + if (releasing.uses > 0) return; + await releasing.settled(); + if (releasing.uses > 0 || connection !== releasing) return; + releasing.client.close(); connection = null; }; diff --git a/packages/sqlite/src/sqlite.test.ts b/packages/sqlite/src/sqlite.test.ts index a31720ad..7b8a07a1 100644 --- a/packages/sqlite/src/sqlite.test.ts +++ b/packages/sqlite/src/sqlite.test.ts @@ -2,7 +2,6 @@ import { spawn } from "node:child_process"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { Worker } from "node:worker_threads"; import { ConfigurationError, type FieldsArgs, @@ -109,6 +108,57 @@ describe("sqlite adapter in memory, under concurrent use", () => { }); }); +const OPENING_PROCESSES = 6; + +// Opens the file's storage from processes of their own, each loaded first and then let go at +// once, so the first statements of every connection meet on the file: processes, as instances +// on several machines are, and as libSQL's native code wants, since a worker thread that ends +// under it can bring the whole process down. Resolves what each said. +const openTogether = async (path: string): Promise => { + const children = Array.from({ length: OPENING_PROCESSES }, () => + spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { sqlite } from "./index.ts"; + process.stdout.write("loaded\\n"); + process.stdin.once("data", async () => { + try { + await (await sqlite({ path: ${JSON.stringify(path)} }).createStorage()).close(); + process.stdout.write("opened\\n"); + } catch (error) { + process.stdout.write(String(error) + "\\n"); + } + process.stdin.destroy(); + });`, + ], + { cwd: import.meta.dirname, stdio: ["pipe", "pipe", "inherit"] }, + ), + ); + const said = children.map((child) => { + const lines: string[] = []; + let loaded: () => void = () => undefined; + const ready = new Promise((resolve) => { + loaded = resolve; + }); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + for (const line of chunk.split("\n").filter(Boolean)) { + if (line === "loaded") loaded(); + else lines.push(line); + } + }); + const exited = new Promise((resolve) => { + child.once("exit", (code) => resolve(lines.join(" ") || `exited with ${code}`)); + }); + return { ready, exited }; + }); + await Promise.all(said.map(({ ready }) => ready)); + for (const child of children) child.stdin.write("go\n"); + return Promise.all(said.map(({ exited }) => exited)); +}; + describe("sqlite adapter on a file", () => { let directory: string; let counter = 0; @@ -167,6 +217,47 @@ describe("sqlite adapter on a file", () => { locking: "single-writer", }); + it("closes the file only once what runs on it has settled, a transaction in flight included", async () => { + const path = freshPath(); + const adapter = sqlite({ path }); + const storage = await adapter.createStorage(); + const ledger = await adapter.createForgetLedger({}); + // Its table made, so its read below is one statement, queued behind the transaction. + await ledger.read({ limit: 1 }); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + let entered: () => void = () => undefined; + const inside = new Promise((resolve) => { + entered = resolve; + }); + const writing = storage.transact(async (tx) => { + entered(); + await held; + await tx.eventStore.append({ + aggregateType: "order", + aggregateId: "1", + expectedVersion: 0, + events: [pendingEvent({ aggregateId: "1", version: 1 })], + }); + }); + await inside; + const reading = ledger.read({ limit: 1 }); + let closed = false; + const closing = Promise.all([storage.close(), ledger.close?.()]).then(() => { + closed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(closed).toBe(false); + release(); + await writing; + expect(await reading).toEqual([]); + await closing; + const reopened = await openFile(path); + expect(await reopened.eventStore.lastPosition()).toBe(1); + }); + it("lets its own statements wait for its own write transaction, without blocking the process", async () => { const storage = await openFile(); let release: () => void = () => undefined; @@ -232,22 +323,7 @@ describe("sqlite adapter on a file", () => { await readModel.close(); }); - it("opens a file not in WAL yet, such as one just restored, from several connections at once", async () => { - const connections = 6; - // Each worker loads the adapter, waits at the gate with the others, then opens its storage: - // the first statements of every connection meet on the file at the same moment. - const worker = ` - const { workerData, parentPort } = require("node:worker_threads"); - import(workerData.module).then(async ({ sqlite }) => { - parentPort.postMessage("loaded"); - Atomics.wait(new Int32Array(workerData.gate), 0, 0); - try { - await (await sqlite({ path: workerData.path }).createStorage()).close(); - parentPort.postMessage("opened"); - } catch (error) { - parentPort.postMessage(String(error)); - } - });`; + it("opens a file not in WAL yet, such as one just restored, from several processes at once", async () => { for (let round = 0; round < 3; round += 1) { const sourcePath = freshPath(); const source = await openFile(sourcePath); @@ -265,86 +341,14 @@ describe("sqlite adapter on a file", () => { expect((await restored.execute("PRAGMA journal_mode")).rows[0]?.[0]).toBe("delete"); restored.close(); - const gate = new SharedArrayBuffer(4); - const workers = Array.from( - { length: connections }, - () => - new Worker(worker, { - eval: true, - workerData: { module: new URL("./index.ts", import.meta.url).href, path, gate }, - }), - ); - const loaded = workers.map( - (each) => - new Promise((resolve) => { - each.once("message", () => resolve()); - each.once("error", () => resolve()); - }), - ); - const outcomes = workers.map( - (each) => - new Promise((resolve) => { - each.on("message", (message: string) => { - if (message !== "loaded") resolve(message); - }); - each.on("error", (error) => resolve(String(error))); - }), - ); - await Promise.all(loaded); - Atomics.store(new Int32Array(gate), 0, 1); - Atomics.notify(new Int32Array(gate), 0); - expect(await Promise.all(outcomes)).toEqual(Array(connections).fill("opened")); - await Promise.all(workers.map((each) => each.terminate())); + expect(await openTogether(path)).toEqual(Array(OPENING_PROCESSES).fill("opened")); expect(await (await openFile(path)).eventStore.lastPosition()).toBe(1); } }); it("migrates a new file once when processes open it at the same moment", async () => { - const connections = 6; const path = freshPath(); - const worker = ` - const { workerData, parentPort } = require("node:worker_threads"); - import(workerData.module).then(async ({ sqlite }) => { - parentPort.postMessage("loaded"); - Atomics.wait(new Int32Array(workerData.gate), 0, 0); - try { - await (await sqlite({ path: workerData.path }).createStorage()).close(); - parentPort.postMessage("opened"); - } catch (error) { - parentPort.postMessage(String(error)); - } - });`; - const gate = new SharedArrayBuffer(4); - const workers = Array.from( - { length: connections }, - () => - new Worker(worker, { - eval: true, - workerData: { module: new URL("./index.ts", import.meta.url).href, path, gate }, - }), - ); - const outcomes = workers.map( - (each) => - new Promise((resolve) => { - each.on("message", (message: string) => { - if (message !== "loaded") resolve(message); - }); - each.on("error", (error) => resolve(String(error))); - }), - ); - await Promise.all( - workers.map( - (each) => - new Promise((resolve) => { - each.once("message", () => resolve()); - each.once("error", () => resolve()); - }), - ), - ); - Atomics.store(new Int32Array(gate), 0, 1); - Atomics.notify(new Int32Array(gate), 0); - expect(await Promise.all(outcomes)).toEqual(Array(connections).fill("opened")); - await Promise.all(workers.map((each) => each.terminate())); + expect(await openTogether(path)).toEqual(Array(OPENING_PROCESSES).fill("opened")); const client = createClient({ url: `file:${path}` }); const { rows } = await client.execute( 'SELECT "scope", "version" FROM "bounda_migrations" ORDER BY "scope", "version"', @@ -552,6 +556,10 @@ describe("sqlite adapter connection sharing", () => { const stillOpen = await adapter.createStorage(); expect(await stillOpen.eventStore.lastPosition()).toBe(1); await stillOpen.close(); + const client = readModel.client.raw as Client; + expect(client.closed).toBe(false); + await readModel.close(); + expect(client.closed).toBe(true); await readModel.close(); const fresh = await adapter.createStorage();