diff --git a/.changeset/forget-ledger-first.md b/.changeset/forget-ledger-first.md new file mode 100644 index 00000000..6801e843 --- /dev/null +++ b/.changeset/forget-ledger-first.md @@ -0,0 +1,17 @@ +--- +"@bounda-dev/core": minor +"@bounda-dev/sqlite": minor +"@bounda-dev/postgresql": minor +"@bounda-dev/cloudflare": minor +"@bounda-dev/cli": minor +--- + +With a forget ledger, every forgetting is in the ledger before it commits. `forget`, `forgetAgain`, a policy's or process's `forget` and the retention write the forgetting's intention first, its event's id, time and reason set before the transaction, which appends the event with them; a reconciliation commits again, under the same id, time and reason, every forgetting the database does not hold whose person still holds a key it names, so an intention whose commit never happened, because the process stopped or the transaction failed, is applied at the next start, after a restore too. A `forget` that rejects after its intention was written still forgets the person then. A forgetting the retention made is committed again only while the retention would still forget the person. A reconciliation leaves an entry younger than `FORGET_LEDGER_GRACE_MS` to a later one, due once it is old enough, but for the one a restore found, which the copy cannot be committing. + +Ledger entries and `PersonalDataForgotten` name the keys a forgetting destroys by their fingerprint: `ForgetLedgerEntry` gains `reason` and `fingerprints`, the payload gains `fingerprints`, and a reconciliation leaves a key the entry does not name, given to the person after it. The reason `"restored"` is gone: a forgetting committed again keeps its own. The SQL ledger table gains `reason` and `fingerprints` in version 2 of its `forget-ledger` scope, and the KV ledger keeps them in each key's metadata; an entry written before reads as a request that names no keys. + +`forgetLedger.strict` now also means nothing is forgotten that the ledger has not recorded: a forgetting whose intention the ledger does not take fails with `ForgetLedgerUnavailableError` (`FORGET_LEDGER_UNAVAILABLE`) and forgets nobody, so trying again is safe. `forgetLedger.timeout` (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS`, ten seconds) bounds each call to the ledger, given up with `ForgetLedgerTimeoutError` (`FORGET_LEDGER_TIMEOUT`), and how long a start that is not strict waits for the reconciliation; a strict start fails once a read runs past it. `parseDuration` is exported, and `isAdapter` from `@bounda-dev/core/adapter`. + +`bounda forgotten` reads the event store without starting the app, so a strict ledger that is down does not stop it, and prints each forgetting's reason and fingerprints; `bounda forget --from` commits each whole line again under its own id, time and reason. `bounda forget-ledger expire --binding --keep-for ` gives every key of a Workers KV ledger the expiration `keepFor` sets, those written before it was set included, through the project's wrangler; `expireKvForgetLedger` and `planKvForgetLedgerExpiry` from `@bounda-dev/cloudflare`, and its new `./forget-ledger` subpath, do the same in code. + +`sqlite()` closes its libSQL connection only once the statements and transactions it started have settled, so closing an app or a ledger under a call still running, as one given up for its timeout, no longer frees what libSQL's native code is using. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cc6c36d..6277c163 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,9 @@ jobs: if: needs.changes.outputs.packages != '[]' runs-on: windows-latest timeout-minutes: 30 + env: + # A worker that dies in native code, as libSQL can there, names the test it was running. + BOUNDA_TEST_TRACE: "1" steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 @@ -112,7 +115,7 @@ jobs: run: pnpm --filter @bounda-dev/core test - name: sqlite if: ${{ !cancelled() && steps.build.outcome == 'success' }} - run: pnpm --filter @bounda-dev/sqlite exec vitest run --exclude src/libsql-server.test.ts + run: pnpm --filter @bounda-dev/sqlite exec vitest run --exclude src/libsql-server.test.ts --reporter=verbose - name: cli if: ${{ !cancelled() && steps.build.outcome == 'success' }} run: pnpm --filter @bounda-dev/cli test diff --git a/docs/src/content/docs/adapters/cloudflare.md b/docs/src/content/docs/adapters/cloudflare.md index 371b2e6a..8fdb15cc 100644 --- a/docs/src/content/docs/adapters/cloudflare.md +++ b/docs/src/content/docs/adapters/cloudflare.md @@ -166,7 +166,11 @@ in `wrangler.jsonc` and name it in `runtime.personalData.forgetLedger.store`; an `ConfigurationError`. Each object keeps its entries under its own name, and reads the ledger once more a minute after it starts, since KV may take that long to list a key. The periodic reconciliation never wakes an object on its own, and with `keepFor` each key expires in KV by -itself. See [backups](/guides/personal-data/#backups). +itself; `bounda forget-ledger expire` gives the keys written before `keepFor` was set theirs. +Every forgetting writes its intention to KV before the object commits it, so one the object did not +get to commit, because it was evicted or its transaction failed, is committed at its next start; +with `strict`, a forgetting KV does not take fails and forgets nobody. See +[backups](/guides/personal-data/#backups). ## One object per tenant diff --git a/docs/src/content/docs/guides/personal-data.md b/docs/src/content/docs/guides/personal-data.md index bfaa41f9..3a090462 100644 --- a/docs/src/content/docs/guides/personal-data.md +++ b/docs/src/content/docs/guides/personal-data.md @@ -201,11 +201,12 @@ context of React Router. A client bound with `as` records who asked on `Personal `metadata.actor`, and so on `ProcessingRestricted` and `ProcessingRestrictionLifted`. In one transaction it deletes the person's key, cancels the scheduled commands that hold their data, removes the dead letters of those that gave up, and appends `PersonalDataForgotten` to the stream `bounda:`: the runtime's own aggregate, `bounda`, -has one stream per person it forgot. Its payload is `{ personId, reason, cancelledCommands, -removedDeadLetters }`: `reason` is `"requested"` for `forget`, `"retention"` when the -[retention](#retention) forgot them and `"restored"` when they were forgotten again after a -[backup was restored](#backups), and the rest are the ids and types of what went, the only -record of commands that never became events. It resolves like a command, with the event's +has one stream per person it forgot. Its payload is `{ personId, reason, fingerprints, +cancelledCommands, removedDeadLetters }`: `reason` is `"requested"` for `forget` and +`"retention"` when the [retention](#retention) forgot them, `fingerprints` names the key it +destroyed, by its check value, which opens nothing, none for a person who held none, and the rest +are the ids and types of what went, the only record of commands that never became events. A +forgetting a [forget ledger](#backups) commits again keeps its id, its time and its reason. It resolves like a command, with the event's position, so `catchUpReadModels({ through })` waits for the read models that handle it, and the Durable Object's `forget` and the one in React Router's `bounda` context wait for them themselves unless `consistency` is `"eventual"`. A person without a key is forgotten all the same, since a read model may hold what their events carried in @@ -831,9 +832,11 @@ is restored, to apply the erasures again. A forget ledger is how Bounda does the ### The forget ledger The ledger is a record of every forgetting kept outside the database: for each `PersonalDataForgotten`, -the person's id, the event's id and its time, and nothing else. It is a suppression list, the one -thing about a forgotten person that has to outlive their data, so it holds no field of theirs and -no reason. Give it a store a restore of the database does not touch: +the person's id, the event's id, its time, its reason, and the fingerprints of the keys it +destroys, and nothing else. It is a suppression list, the one thing about a forgotten person that +has to outlive their data, so it holds no field of theirs; a fingerprint is a check value of a key, +which opens nothing and names nothing once the key is gone. Give it a store a restore of the +database does not touch: ```ts import { defineConfig, masterKey } from "@bounda-dev/core/config"; @@ -886,8 +889,9 @@ Every object keeps its entries under its own name, so an object reached through name fails to start; `createWorker` names each by its tenant. KV may take up to a minute to list a key it just wrote, so an object that starts reads the ledger once more a minute later, and wakes for it. Anything else is a ledger of your own through `defineForgetLedger`, which receives the host's -environment, the tenant and `keepForMs` when the app starts: its `read` returns entries in the -order of their time, then of their event's id, and `byForgetLedgerOrder` from +environment, the tenant and `keepForMs` when the app starts: it keeps every field of an entry, the +reason and the fingerprints included, which the reconciliation decides by; its `read` returns +entries in the order of their time, then of their event's id, and `byForgetLedgerOrder` from `@bounda-dev/core/adapter` sorts them so; its optional `prune` removes the entries older than a time; its `readMark` and `raiseMark` keep numbers that only go up, which the runtime uses to tell a copy [restored while the app was stopped](#restores); `forgetLedgerContract` from @@ -895,32 +899,57 @@ a copy [restored while the app was stopped](#restores); `forgetLedgerContract` f ### What the runtime does with it -Every transaction that forgets someone appends the forgetting to the ledger as soon as it commits, -before `forget` resolves, whoever forgot them: `forget`, a policy or process, the retention or a -reconciliation. The event store is also the ledger's outbox: a subscriber of the dispatcher, -`forget-ledger`, copies every `PersonalDataForgotten` to the ledger again in batches, with a -policy's guarantee, and the ledger keeps one entry per event however often it receives it. So a -forgetting the ledger refused, or did not take within ten seconds, which the app logs as a -warning, still reaches it: a batch the ledger refuses is delivered again, and the subscriber's -lag in `getLag` and `bounda.dispatcher.lag` shows how far behind it is. Configuring a ledger on an +Every forgetting writes its intention to the ledger first, and commits after: `forget`, a policy +or process, the retention, `forgetAgain`. The intention is the entry the forgetting will be: its +event's id and time are set before the transaction, and the transaction appends the event with +them, so the ledger and the database name one forgetting by one id. A policy's or process's +`forget` writes it just before its run's transaction opens, once the run has decided. + +What happens to an intention whose commit does not happen, because the process stopped in between +or the database refused the transaction, is the same in every case: the next reconciliation +commits it, since forgetting is what was asked. A `forget` that rejects after its intention was +written still forgets the person then; calling it again forgets them at once, and is never an +error. The retention is the one exception, as it decides on the person's activity: a forgetting +the retention made is committed again only while the retention would still forget the person, +read on the database as it is then, so a person who came back, or whose processing was +restricted, keeps their data. + +An intention names the key the person held when it was written, and a reconciliation commits it +only while the person still holds that key: a key they hold that it does not name was given to +them after, as when they come back and register again, and is kept. That also settles the order +between several intentions of one person, which never changes the result: each commits only while +the key it names is there, so the first to commit destroys it and the others find it gone. A +reconciliation leaves an entry younger than a minute, `FORGET_LEDGER_GRACE_MS` from +`@bounda-dev/core`, to the next one, due once it is old enough: it may be the intention of a +forgetting still committing, which then commits with its own actor. After a restore the instances find, it +commits them at once: the copy cannot be committing any of them. + +The event store is also the ledger's outbox: a subscriber of the dispatcher, `forget-ledger`, +copies every `PersonalDataForgotten` to the ledger again in batches, with a policy's guarantee, +and the ledger keeps one entry per event however often it receives it. So a forgetting whose +intention the ledger refused, or did not take within its `timeout`, which an app that is not +strict logs as a warning and commits all the same, still reaches it: a batch the ledger refuses is +delivered again, and the subscriber's lag in `getLag` and `bounda.dispatcher.lag` shows how far +behind it is. While an intention the ledger did not take in time is still pending, the next +forgettings do not wait for it, and leave their copy to the subscriber. Configuring a ledger on an app that already forgot people copies those forgettings first. When the app starts, before it serves anything, it reads the ledger past a cursor the database -keeps, the time of the last entry it applied, and for each entry whose forgetting the database -does not hold, it forgets the person again, with `reason: "restored"`, if they still hold a key. -After a normal start that read returns next to nothing. After a restore the cursor went back with -the copy, and the read returns exactly the forgettings made since the copy was taken; the read -models that show those persons hear of it before the start resolves, and the app logs how many it -forgot again: +keeps, the time of the last entry it applied, and commits again, under its own id, time and +reason, each forgetting the database does not hold whose person still holds a key it names. After +a normal start that read returns next to nothing. After a restore the cursor went back with the +copy, and the read returns exactly the forgettings made since the copy was taken, the intentions +whose commit never happened included; the read models that show those persons hear of it before +the start resolves, and the app logs how many it committed again: ``` -forgot again persons a restored backup brought back +committed again forgettings the database did not hold ``` An entry whose forgetting the database holds is skipped even when the person holds a key, since -that key was given to them after they were forgotten, as when they come back and register again. -The check and the forgetting are one transaction, so instances starting together forget a person -once. A Durable Object does it on every cold start; restoring its storage restarts it. +that key was given to them after they were forgotten. The check and the forgetting are one +transaction, so instances starting together forget a person once. A Durable Object does it on +every cold start; restoring its storage restarts it. While the app runs, it reads the ledger again every `reconcileEvery`, five minutes unless you set it. One instance takes each round, whichever finds it due first, and the time of the last round is @@ -930,31 +959,39 @@ runs it on the passes it wakes for, never waking for it alone, since restoring i restarts it anyway. A database restored under a running app is reconciled at once, on the next pass that [finds the restore](#restores). -A ledger that cannot be read when the app starts does not stop it: the app logs an error, starts, -and its worker tries again after 30 seconds, then twice as long after each failure, up to an hour. +Each call to the ledger is given up after its `timeout`, ten seconds unless you set it, with +[`FORGET_LEDGER_TIMEOUT`](/reference/errors/#codes); the call may still land after it. A ledger +that cannot be read when the app starts does not stop it: the app logs an error, starts, and its +worker tries again after 30 seconds, then twice as long after each failure, up to an hour. The wait is kept in `bounda_checkpoints`, beside the cursor, so every instance waits for the longest and a restart backs off from where it was, though a start always tries once; the first reconciliation that succeeds clears it. -One that does not answer within ten seconds is left to finish in the background, with an error -logged, and the app starts. +A reconciliation that takes longer than the `timeout` as a whole, each call within it, is left to +finish in the background, with an error logged, and the app starts. Until then `getLag()` says `forgetLedger: { reconciled: false }`, and a person the restore brought back can be read; alert on it. An app without a ledger reports `forgetLedger: null`. -To never serve before the ledger was read, make it strict: +To never forget anyone the ledger has not recorded, nor serve before the ledger was read, make it +strict: ```ts forgetLedger: { store: sqlite({ path: "./ledger/forget-ledger.db" }), strict: true }, ``` -A strict app waits at start for the reconciliation however long it takes, and a ledger it cannot -read fails the start with +A strict app forgets nobody whose intention the ledger did not take: `forget`, `forgetAgain` and +a policy's `forget` fail with [`FORGET_LEDGER_UNAVAILABLE`](/reference/errors/#codes), the +ledger's error as its `cause`, and nothing is forgotten, so trying again is safe; a policy's run +fails and is retried as any other, and the retention holds the person and tries again later. It +waits at start for the reconciliation however long it takes, each call within the `timeout`, and a +ledger it cannot read fails the start with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes), the ledger's error as its `cause`; -starting again tries again. The CLI starts the app too, so its commands need the ledger then. -Availability now depends on the ledger's: give it a store at least as available as the database. +starting again tries again. The CLI's commands that start the app need the ledger then; +`bounda forgotten` does not, since it reads the event store alone. Availability now depends on the +ledger's: give it a store at least as available as the database. -A restore loses only what the ledger never received: a forgetting committed in the moment before -the process stopped, before its append, and that the subscriber had not copied yet, or one made -while the ledger was down whose copy was still waiting when the database was lost. Keep the +A strict app loses no forgetting to a restore: each is in the ledger before it commits. One that is +not strict loses only a forgetting whose intention the ledger did not take, while it was down or +slow, and that the subscriber had not copied yet when the database was lost. Keep the `forget-ledger` subscriber's lag near zero, as for any other. ### Keeping the ledger short @@ -970,8 +1007,22 @@ The instance that takes a periodic round prunes after it, through the ledger's ` forgetting older than `keepFor` is not appended at all. KV expires each key on its own instead, `keepFor` after its forgetting, since it cannot list its oldest keys first. A backup older than `keepFor` brings back whom the removed entries named, so when you keep backups longer, raise it -first. Without `keepFor` entries are kept for good: ids and times only, but still the ids of -people who asked to be forgotten. +first. Without `keepFor` entries are kept for good: ids, times and fingerprints only, but still the +ids of people who asked to be forgotten. + +A KV key gets its expiration when it is written, so the keys written before `keepFor` was set, or +while it was longer, keep theirs. Give them the one `keepFor` sets now, from the project's root, +with the binding and the `keepFor` of the configuration: + +```bash +bounda forget-ledger expire --binding FORGET_LEDGER --keep-for 35d +``` + +It goes through the project's wrangler, logged in to the account, to the deployed namespace, +`--local` to the state of `wrangler dev`, with `--env` and `--wrangler-config` as wrangler takes +them; it writes every object's keys again with their expiration and removes those already past +it. `expireKvForgetLedger({ namespace, keepForMs })` from `@bounda-dev/cloudflare` does the same +from a Worker, within the KV operations one invocation may make. ### Without a ledger @@ -989,13 +1040,15 @@ Then, once the copy is back: bounda forget --from forgotten.jsonl ``` -`bounda forgotten` prints one line of JSON per forgetting, the entry a ledger would hold; -`bounda forget` forgets again, with `reason: "restored"`, each person listed who still holds a key, -unless the database holds the forgetting a line names. A line may also be a bare person's id, for -a list kept some other way, which is forgotten whenever the person holds a key: list only those -forgotten after the copy was taken. `--actor ` names who does it on each `PersonalDataForgotten` -it appends. `app.forgetAgain(entries, { actor })` is the same in code. Then run `bounda restored`, -so the jobs start over from the copy ([restores](#restores)). +`bounda forgotten` prints one line of JSON per forgetting, the entry a ledger would hold, reading +the event store without starting the app; `bounda forget` commits again each forgetting listed, +under its own id, time and reason, as a reconciliation would: unless the database holds it, and +only while the person holds a key it names. With a ledger configured, it records each in the +ledger first. A line may also be a bare person's id, for a list kept some other way, which is +forgotten, as a request, whenever the person holds a key: list only those forgotten after the copy +was taken. `--actor ` names who does it on each `PersonalDataForgotten` it appends. +`app.forgetAgain(entries, { actor })` is the same in code. Then run `bounda restored`, so the +jobs start over from the copy ([restores](#restores)). Without either, a restored backup brings back the data of everyone forgotten after it was taken, until they are forgotten again. diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 9592875c..5b8704f2 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -354,8 +354,9 @@ migration failed. ## `bounda forgotten` and `bounda forget` Apply again, by hand, the forgettings a restored backup undid, for an app without a -[forget ledger](/guides/personal-data/#backups). Both boot the project as `bounda dead-letters` -does. +[forget ledger](/guides/personal-data/#backups). `bounda forget` boots the project as +`bounda dead-letters` does; `bounda forgotten` only opens the project's storage and reads its +event store, without starting the app, so a strict forget ledger that is down does not stop it. ```bash bounda forgotten --since 2026-11-01T03:00:00Z > forgotten.jsonl @@ -366,18 +367,20 @@ bounda forget --from forgotten.jsonl `bounda forgotten` reads the stream and prints, one line of JSON each, the `PersonalDataForgotten` of every person forgotten since `--since`: a date, compared with each event's time, which reads the whole stream, or a position of the stream to read after. Each line is -`{"personId":…,"eventId":…,"forgottenAt":…}`, the entry a ledger would hold; the count goes to +`{"personId":…,"eventId":…,"forgottenAt":…,"reason":…,"fingerprints":[…]}`, the entry a ledger +would hold, without `fingerprints` for a forgetting recorded before Bounda 0.4.0; the count goes to standard error, so standard output is the list alone. Run it against the database before it is restored, or regularly next to the backups. -`bounda forget --from ` forgets again, with `reason: "restored"`, each person listed who -still holds a key, unless the database holds the forgetting the line names by `eventId`. A line -may also be a bare person's id, forgotten whenever they hold a key; blank lines are skipped. It -prints each person it forgot and how many of those listed: +`bounda forget --from ` commits again each forgetting listed, under its own id, time and +reason, unless the database holds it, and only while the person holds a key the line names, as a +reconciliation would; with a forget ledger configured, it records each in the ledger first. A +line may also be a bare person's id, forgotten as a request whenever they hold a key; blank lines +are skipped. It prints each person it forgot and how many of those listed: ``` forgot again c-42 -forgot again 1 of 3 listed; the rest hold no key, or this store holds their forgetting +forgot again 1 of 3 listed; the rest hold no key the line names, or this store holds their forgetting ``` | Option | Applies to | Meaning | @@ -390,6 +393,38 @@ forgot again 1 of 3 listed; the rest hold no key, or this store holds their forg Exit codes: `0` done, `1` bad arguments, `2` the project could not be loaded, or the file could not be read or has a line that is neither a person's id nor an entry, in which case nobody is forgotten. +## `bounda forget-ledger expire` + +Gives every entry of a Workers KV [forget ledger](/guides/personal-data/#keeping-the-ledger-short) +the expiration `keepFor` sets: KV expires a key only as it was written, so the keys written before +`keepFor` was set, or while it was longer, keep the expiration they had, none at first. Run it once +after setting or shortening `keepFor`, from the project's root. + +```bash +bounda forget-ledger expire --binding FORGET_LEDGER --keep-for 35d +``` + +It runs the project's own wrangler, logged in to the account: it lists every object's keys under +the ledger, writes again with their expiration those that expire later than `keepFor` after their +forgetting or never, and removes those already past it, through `wrangler kv bulk`. A key that is +not an entry of the ledger is never touched. It prints what it changed: + +``` +gave 1204 entries the expiration keepFor sets; removed 37 already past it +``` + +| Option | Meaning | +| --- | --- | +| `--binding ` | The KV namespace's binding in `wrangler.jsonc`, as `kvForgetLedger({ binding })` names it | +| `--keep-for ` | The configuration's `keepFor`, such as `35d` | +| `--local` | The local state of `wrangler dev` instead of the deployed namespace | +| `--env `, `--wrangler-config `, `--persist-to ` | As wrangler takes `--env`, `--config` and `--persist-to` | +| `--root ` | The project's root, where wrangler is installed | +| `--error-text` | Print the message of the error it fails on, as [error text](#error-text) says | + +Exit codes: `0` done, `1` bad arguments, `2` wrangler failed, is not installed, or +`@bounda-dev/cloudflare` is not; the failure is printed as [error text](#error-text) says. + ## Programmatic use `bounda generate` is exported from `@bounda-dev/cli` as `generate`: diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 3961acaf..330ad086 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -123,7 +123,8 @@ a stopped one left: see [delivery guarantees](/concepts/delivery-guarantees/#the warns at start when the registry marks personal fields. It is app-wide. See [where the keys live](/guides/personal-data/#where-the-keys-live). - **`personalData.forgetLedger`**, none by default. Where every forgetting is recorded outside the - database, by person's id, event id and time, and how the app reconciles with it. Without it, a + database, before it commits, by person's id, event id, time, reason and the fingerprints of the + keys it destroys, and how the app reconciles with it. Without it, a restored backup brings back everyone forgotten after it was taken. See [backups](/guides/personal-data/#backups). It holds: - **`store`**, required. The ledger: an adapter pointed at another database than `storage`, such @@ -131,22 +132,34 @@ a stopped one left: see [delivery guarantees](/concepts/delivery-guarantees/#the `bounda_forget_ledger` table there on first use, with `bounda_forget_ledger_marks` for the mark that tells a [restore](/guides/personal-data/#restores) made while the app was stopped; `kvForgetLedger({ binding })` from `@bounda-dev/cloudflare` on Cloudflare; or a ledger of your own through `defineForgetLedger`. - The runtime appends each forgetting to it as the forgetting commits, and copies every + Every forgetting writes its intention to it before it commits, and the runtime copies every `PersonalDataForgotten` to it again from the stream in the background. An adapter pointing at the storage's own database fails the start. - **`reconcileEvery`**, `5m` by default (`DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS` in `@bounda-dev/core/config`). The app reads the ledger when it starts and, while it runs, again - this often, one instance a round, and forgets again, with `reason: "restored"`, whoever a - restored backup brought back; a restore the instances find reconciles at once. A Durable Object - only runs a round on the passes it wakes for anyway; it reconciles on every start. - - **`strict`**, `false` by default. When `false`, a ledger that cannot be read at start, or does - not answer within ten seconds, does not fail it: the app logs an error and reconciles in the - background, and `getLag()` reports `forgetLedger: { reconciled: false }` meanwhile. When - `true`, the start waits for the reconciliation however long it takes, and a ledger it cannot - read fails it with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes). + this often, one instance a round, and commits again, under its own id, time and reason, each + forgetting the database does not hold whose person still holds a key it names: one a restored + backup rolled back, or one whose commit never happened; a restore the instances find + reconciles at once. A Durable Object only runs a round on the passes it wakes for anyway; it + reconciles on every start. + - **`strict`**, `false` by default. When `false`, a forgetting whose intention the ledger does + not take is logged and commits all the same, for the stream to copy later, and a ledger that + cannot be read at start, or a reconciliation longer than `timeout`, does not fail it: the app + logs an error and reconciles in the background, and `getLag()` reports + `forgetLedger: { reconciled: false }` meanwhile. When `true`, nothing is forgotten that the + ledger has not recorded: a forgetting whose intention it does not take fails with + [`FORGET_LEDGER_UNAVAILABLE`](/reference/errors/#codes) and forgets nobody, so trying again is + safe; and the start waits for the reconciliation however long it takes, and a ledger it cannot + read fails it with [`FORGET_LEDGER_UNRECONCILED`](/reference/errors/#codes). `bounda forgotten` + needs neither, since it reads the event store alone. - **`keepFor`**, none by default. How long an entry is kept after the forgetting; set it longer than the oldest backup you can restore. The instance that runs a periodic round prunes the - older entries after it; KV expires each key on its own. Without it entries are kept for good. + older entries after it; KV expires each key on its own, and `bounda forget-ledger expire` gives + the keys written before `keepFor` was set theirs. Without it entries are kept for good. + - **`timeout`**, `10s` by default (`DEFAULT_FORGET_LEDGER_TIMEOUT_MS` in + `@bounda-dev/core/config`). How long each call to the ledger may take before it is given up + with [`FORGET_LEDGER_TIMEOUT`](/reference/errors/#codes), and how long a start that is not + strict waits for the reconciliation. ### Dead letters diff --git a/docs/src/content/docs/reference/errors.md b/docs/src/content/docs/reference/errors.md index a98497b9..cecd19fd 100644 --- a/docs/src/content/docs/reference/errors.md +++ b/docs/src/content/docs/reference/errors.md @@ -75,6 +75,16 @@ dead letter with `errorType: "retriable_exhausted"`. is malformed: an unknown key, a port with no implementation chosen, a module missing an export. Thrown at boot, or when a handler reads a port a test left out. Also thrown at start by a database whose Bounda tables a version before 0.2.0 created. +- **`FORGET_LEDGER_UNAVAILABLE`** (`ForgetLedgerUnavailableError`; a forgetting's answer). Thrown + by `forget`, `forgetAgain`, `bounda forget` and a policy's or process's `forget` when + `runtime.personalData.forgetLedger.strict` is set and the + [forget ledger](/guides/personal-data/#what-the-runtime-does-with-it) did not take the + forgetting's intention: nothing was forgotten, so trying again is safe. `cause` is the ledger's + error, a `ForgetLedgerTimeoutError` when it did not answer in time. +- **`FORGET_LEDGER_TIMEOUT`** (`ForgetLedgerTimeoutError`, with `operation` and `timeoutMs`). A call + to the forget ledger did not settle within `runtime.personalData.forgetLedger.timeout`: the + `cause` of `FORGET_LEDGER_UNAVAILABLE` and `FORGET_LEDGER_UNRECONCILED`, and what an app that is + not strict logs. The call may still land after it. - **`FORGET_LEDGER_UNRECONCILED`** (`ForgetLedgerUnreconciledError`; a start's answer). Thrown by `createApp`, and so by `boot` and the CLI, when `runtime.personalData.forgetLedger.strict` is set and the [forget ledger](/guides/personal-data/#the-forget-ledger) could not be read through: diff --git a/packages/cli/package.json b/packages/cli/package.json index 8290c946..8ddeb500 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,21 +41,27 @@ "commander": "catalog:" }, "peerDependencies": { + "@bounda-dev/cloudflare": "workspace:*", "@bounda-dev/core": "workspace:*", "typescript": ">=7.0.0" }, "peerDependenciesMeta": { + "@bounda-dev/cloudflare": { + "optional": true + }, "typescript": { "optional": true } }, "devDependencies": { + "@bounda-dev/cloudflare": "workspace:*", "@bounda-dev/core": "workspace:*", "@stryker-mutator/core": "catalog:", "@stryker-mutator/vitest-runner": "catalog:", "@types/node": "catalog:", "tsdown": "catalog:", "typescript": "catalog:", - "vitest": "catalog:" + "vitest": "catalog:", + "wrangler": "catalog:" } } diff --git a/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts b/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts new file mode 100644 index 00000000..b337f1f4 --- /dev/null +++ b/packages/cli/src/fixtures/rebuild-project/strict-ledger.config.ts @@ -0,0 +1,25 @@ +import { defineConfig, defineForgetLedger } from "@bounda-dev/core/config"; +import { memory } from "@bounda-dev/core/memory"; + +const down = async (): Promise => { + throw new Error("ledger is down"); +}; + +// A strict forget ledger that is down: the app does not start, and forgets nobody. +export default defineConfig({ + storage: memory(), + runtime: { + role: "all", + personalData: { + forgetLedger: { + store: defineForgetLedger(() => ({ + append: down, + read: down, + readMark: down, + raiseMark: down, + })), + strict: true, + }, + }, + }, +}); diff --git a/packages/cli/src/forget-ledger-expire.test.ts b/packages/cli/src/forget-ledger-expire.test.ts new file mode 100644 index 00000000..59752727 --- /dev/null +++ b/packages/cli/src/forget-ledger-expire.test.ts @@ -0,0 +1,213 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import * as ledger from "@bounda-dev/cloudflare/forget-ledger"; +import { afterAll, describe, expect, it } from "vitest"; +import { expireKvLedger, type Wrangler, wranglerIn } from "./forget-ledger-expire.ts"; + +const DAY = 86_400_000; +const NOW = Date.UTC(2026, 10, 9); +const ROOT = ledger.KV_FORGET_LEDGER_ROOT; + +const temporary: string[] = []; + +afterAll(async () => { + for (const directory of temporary) await rm(directory, { recursive: true, force: true }); +}); + +const metadata = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(NOW - daysAgo * DAY).toISOString(), + reason: "requested", +}); + +// Answers the listing with `listed`, and keeps what each bulk command was handed. +const fakeWrangler = (listed: unknown) => { + const calls: (readonly string[])[] = []; + const files: unknown[] = []; + const wrangler: Wrangler = async (args) => { + calls.push(args); + if (args[1] === "bulk") files.push(JSON.parse(await readFile(args[3] ?? "", "utf8"))); + return args[1] === "key" ? JSON.stringify(listed) : ""; + }; + return { wrangler, calls, files }; +}; + +describe("expiring a KV forget ledger through wrangler", () => { + it("writes again, with an expiration, the entries kept for good or too long, and removes those past keepFor", async () => { + const { wrangler, calls, files } = fakeWrangler([ + { name: `${ROOT}t/1:e-1`, metadata: metadata(1) }, + { name: `${ROOT}t/2:e-2`, metadata: metadata(40) }, + { name: `${ROOT}t/3:e-3`, metadata: metadata(2), expiration: NOW / 1000 + 400 * 86_400 }, + { name: `${ROOT}t/4:e-4`, metadata: metadata(3), expiration: NOW / 1000 + 86_400 }, + { name: `${ROOT}t/5:e-5` }, + ]); + const expired = await expireKvLedger({ + wrangler, + ledger, + binding: "FORGET_LEDGER", + keepForMs: 35 * DAY, + target: { local: false }, + now: NOW, + }); + expect(expired).toEqual({ expired: 2, removed: 1 }); + // The files wrangler read are gone with their directory. + for (const args of calls.slice(1)) expect(existsSync(dirname(args[3] ?? ""))).toBe(false); + expect(calls.map((args) => args.slice(0, 3))).toEqual([ + ["kv", "key", "list"], + ["kv", "bulk", "put"], + ["kv", "bulk", "delete"], + ]); + expect(calls[0]).toEqual([ + "kv", + "key", + "list", + "--binding", + "FORGET_LEDGER", + "--prefix", + ROOT, + "--remote", + ]); + expect(calls[1]?.slice(4)).toEqual(["--binding", "FORGET_LEDGER", "--remote"]); + expect(calls[2]?.slice(4)).toEqual(["--binding", "FORGET_LEDGER", "--force", "--remote"]); + expect(files).toEqual([ + [ + { + key: `${ROOT}t/1:e-1`, + value: "", + expiration: (NOW + 34 * DAY) / 1000, + metadata: metadata(1), + }, + { + key: `${ROOT}t/3:e-3`, + value: "", + expiration: (NOW + 33 * DAY) / 1000, + metadata: metadata(2), + }, + ], + [`${ROOT}t/2:e-2`], + ]); + }); + + it("calls no bulk command when nothing changes, and names the target it was given", async () => { + const { wrangler, calls } = fakeWrangler([]); + expect( + await expireKvLedger({ + wrangler, + ledger, + binding: "LEDGER", + keepForMs: DAY, + target: { local: true, env: "staging", config: "w.jsonc", persistTo: ".state" }, + now: NOW, + }), + ).toEqual({ expired: 0, removed: 0 }); + expect(calls).toEqual([ + [ + "kv", + "key", + "list", + "--binding", + "LEDGER", + "--prefix", + ROOT, + "--local", + "--env", + "staging", + "--config", + "w.jsonc", + "--persist-to", + ".state", + ], + ]); + }); + + it("refuses a listing that is not a list of keys, changing nothing", async () => { + for (const listed of [{}, [{ name: 3 }], [{ name: "k", expiration: "soon" }], [null], ["k"]]) { + const { wrangler, calls } = fakeWrangler(listed); + await expect( + expireKvLedger({ + wrangler, + ledger, + binding: "LEDGER", + keepForMs: DAY, + target: { local: true }, + now: NOW, + }), + ).rejects.toThrow("wrangler kv key list printed something else than a list of keys"); + expect(calls).toHaveLength(1); + } + }); + + it("says wrangler is missing from a project without it, or installed without its executable", async () => { + const root = await mkdtemp(join(tmpdir(), "bounda-no-wrangler-")); + temporary.push(root); + await expect(wranglerIn(root)(["--version"])).rejects.toThrow( + `wrangler is not installed in ${root}: add it to the project`, + ); + const nested = join(root, "app", "nested"); + await mkdir(join(root, "node_modules", "wrangler"), { recursive: true }); + await mkdir(nested, { recursive: true }); + const packageJson = join(root, "node_modules", "wrangler", "package.json"); + await writeFile(packageJson, JSON.stringify({ name: "wrangler" })); + await expect(wranglerIn(nested)(["--version"])).rejects.toThrow( + `${packageJson} names no wrangler executable`, + ); + await writeFile(packageJson, JSON.stringify({ name: "wrangler", bin: "cli.js" })); + await writeFile(join(root, "node_modules", "wrangler", "cli.js"), 'console.log("4.0.0");'); + expect(await wranglerIn(nested)(["--version"])).toBe("4.0.0\n"); + await writeFile( + packageJson, + JSON.stringify({ name: "wrangler", bin: { wrangler: "cli.js", wrangler2: "other.js" } }), + ); + expect(await wranglerIn(root)(["--version"])).toBe("4.0.0\n"); + }); + + it("gives the entries in wrangler's local state their expiration, as the deployed namespace would get them", async () => { + const directory = await mkdtemp(join(tmpdir(), "bounda-kv-expire-")); + temporary.push(directory); + const config = join(directory, "wrangler.jsonc"); + await writeFile( + config, + JSON.stringify({ + name: "kv-expire", + compatibility_date: "2025-10-08", + kv_namespaces: [{ binding: "FORGET_LEDGER", id: "forget-ledger" }], + }), + ); + const where = ["--local", "--config", config, "--persist-to", join(directory, "state")]; + const wrangler = wranglerIn(resolve(import.meta.dirname, "..")); + const now = Date.now(); + const seeded = join(directory, "seeded.json"); + const at = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(now - daysAgo * DAY).toISOString(), + reason: "requested", + }); + await writeFile( + seeded, + JSON.stringify([ + { key: `${ROOT}t/1:e-1`, value: "", metadata: at(1) }, + { key: `${ROOT}t/2:e-2`, value: "", metadata: at(40) }, + { key: "elsewhere", value: "kept" }, + ]), + ); + await wrangler(["kv", "bulk", "put", seeded, "--binding", "FORGET_LEDGER", ...where]); + const expired = await expireKvLedger({ + wrangler, + ledger, + binding: "FORGET_LEDGER", + keepForMs: 35 * DAY, + target: { local: true, config, persistTo: join(directory, "state") }, + now, + }); + expect(expired).toEqual({ expired: 1, removed: 1 }); + const listed = JSON.parse( + await wrangler(["kv", "key", "list", "--binding", "FORGET_LEDGER", ...where]), + ) as readonly { readonly name: string; readonly expiration?: number }[]; + expect(listed.map(({ name, expiration }) => [name, expiration])).toEqual([ + [`${ROOT}t/1:e-1`, Math.ceil((now + 34 * DAY) / 1000)], + ["elsewhere", undefined], + ]); + }, 120_000); +}); diff --git a/packages/cli/src/forget-ledger-expire.ts b/packages/cli/src/forget-ledger-expire.ts new file mode 100644 index 00000000..55c0463b --- /dev/null +++ b/packages/cli/src/forget-ledger-expire.ts @@ -0,0 +1,143 @@ +import { execFile } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import type { KvLedgerKey } from "@bounda-dev/cloudflare/forget-ledger"; + +const run = promisify(execFile); + +// Resolves what wrangler printed to stdout. +export interface Wrangler { + (args: readonly string[]): Promise; +} + +export interface WranglerInFunction { + (root: string): Wrangler; +} + +const binOf = (packageJson: string): string => { + const { bin } = JSON.parse(readFileSync(packageJson, "utf8")) as { + readonly bin?: string | Readonly>; + }; + const path = typeof bin === "string" ? bin : bin?.wrangler; + if (path === undefined) throw new Error(`${packageJson} names no wrangler executable`); + return resolve(dirname(packageJson), path); +}; + +// The package as the project installed it, in the `node_modules` of the root or a directory above. +const installedIn = (root: string, name: string): string | undefined => { + for (let directory = resolve(root); ; ) { + const packageJson = join(directory, "node_modules", name, "package.json"); + if (existsSync(packageJson)) return packageJson; + const parent = dirname(directory); + if (parent === directory) return undefined; + directory = parent; + } +}; + +// Through Node rather than a shell, so the `.cmd` shim it has on Windows is not needed; its stderr, +// where it reports progress, is left out. +export const wranglerIn: WranglerInFunction = (root) => { + let bin: string | undefined; + return async (args) => { + if (bin === undefined) { + const packageJson = installedIn(root, "wrangler"); + if (packageJson === undefined) { + throw new Error(`wrangler is not installed in ${root}: add it to the project`); + } + bin = binOf(packageJson); + } + const { stdout } = await run(process.execPath, [bin, ...args], { + cwd: root, + maxBuffer: 1024 * 1024 * 1024, + }); + return stdout; + }; +}; + +export interface WranglerTarget { + readonly local: boolean; + readonly env?: string | undefined; + readonly config?: string | undefined; + readonly persistTo?: string | undefined; +} + +// The KV ledger's own package lays its keys out, so the plan comes from it. +export type KvLedgerModule = Pick< + typeof import("@bounda-dev/cloudflare/forget-ledger"), + "KV_FORGET_LEDGER_ROOT" | "planKvForgetLedgerExpiry" +>; + +export interface ExpireKvLedgerArgs { + readonly wrangler: Wrangler; + readonly ledger: KvLedgerModule; + readonly binding: string; + readonly keepForMs: number; + readonly target: WranglerTarget; + readonly now: number; +} + +export interface KvLedgerExpired { + readonly expired: number; + readonly removed: number; +} + +export interface ExpireKvLedgerFunction { + (args: ExpireKvLedgerArgs): Promise; +} + +const isListedKey = (value: unknown): value is KvLedgerKey => + typeof value === "object" && + value !== null && + typeof Reflect.get(value, "name") === "string" && + ["number", "undefined"].includes(typeof Reflect.get(value, "expiration")); + +export const expireKvLedger: ExpireKvLedgerFunction = async ({ + wrangler, + ledger, + binding, + keepForMs, + target, + now, +}) => { + const where = [ + target.local ? "--local" : "--remote", + ...(target.env === undefined ? [] : ["--env", target.env]), + ...(target.config === undefined ? [] : ["--config", target.config]), + ...(target.persistTo === undefined ? [] : ["--persist-to", target.persistTo]), + ]; + const listed: unknown = JSON.parse( + await wrangler([ + "kv", + "key", + "list", + "--binding", + binding, + "--prefix", + ledger.KV_FORGET_LEDGER_ROOT, + ...where, + ]), + ); + if (!Array.isArray(listed) || !listed.every(isListedKey)) { + throw new Error("wrangler kv key list printed something else than a list of keys"); + } + const { expire, remove } = ledger.planKvForgetLedgerExpiry({ keys: listed, keepForMs, now }); + const directory = await mkdtemp(join(tmpdir(), "bounda-forget-ledger-")); + try { + if (expire.length > 0) { + const file = join(directory, "expire.json"); + await writeFile(file, JSON.stringify(expire)); + await wrangler(["kv", "bulk", "put", file, "--binding", binding, ...where]); + } + if (remove.length > 0) { + const file = join(directory, "remove.json"); + await writeFile(file, JSON.stringify(remove)); + await wrangler(["kv", "bulk", "delete", file, "--binding", binding, "--force", ...where]); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } + return { expired: expire.length, removed: remove.length }; +}; diff --git a/packages/cli/src/forgotten.test.ts b/packages/cli/src/forgotten.test.ts index c415ed6f..7b1ac54c 100644 --- a/packages/cli/src/forgotten.test.ts +++ b/packages/cli/src/forgotten.test.ts @@ -9,13 +9,14 @@ const event = ( aggregateId: string, type: string, timestamp = "2026-11-01T00:00:00.000Z", + payload: unknown = {}, ): StoredEvent => ({ id: `e-${position}`, aggregateType, aggregateId, version: 1, type, - payload: {}, + payload, timestamp, metadata: { correlationId: "c", causationId: "c", depth: 0, schemaVersion: 1, system: false }, person: null, @@ -59,16 +60,49 @@ describe("readForgotten", () => { events.push( event(1_004, "bounda", "c-1", "ProcessingRestricted"), event(1_005, "order", "o-x", "PersonalDataForgotten"), - event(1_006, "bounda", "c-1", "PersonalDataForgotten", "2026-11-02T00:00:00.000Z"), + event(1_006, "bounda", "c-1", "PersonalDataForgotten", "2026-11-02T00:00:00.000Z", { + reason: "retention", + fingerprints: ["k-1"], + }), event(1_007, "bounda", "c-2", "PersonalDataForgotten", "1970-01-01T00:00:00.001Z"), + event(1_008, "bounda", "c-3", "PersonalDataForgotten", undefined, { + reason: "restored", + fingerprints: [1], + }), + event(1_009, "bounda", "c-4", "PersonalDataForgotten", undefined, null), + event(1_010, "bounda", "c-5", "PersonalDataForgotten", undefined, "text"), ); const { stream, reads } = streamOf([ event(0, "bounda", "c-0", "PersonalDataForgotten"), ...events, ]); - expect(await readForgotten({ events: stream, since: { kind: "position", value: 2 } })).toEqual([ - { personId: "c-1", eventId: "e-1006", forgottenAt: "2026-11-02T00:00:00.000Z" }, - { personId: "c-2", eventId: "e-1007", forgottenAt: "1970-01-01T00:00:00.001Z" }, + const lines = await readForgotten({ events: stream, since: { kind: "position", value: 2 } }); + expect(lines).toStrictEqual([ + { + personId: "c-1", + eventId: "e-1006", + forgottenAt: "2026-11-02T00:00:00.000Z", + reason: "retention", + fingerprints: ["k-1"], + }, + { + personId: "c-2", + eventId: "e-1007", + forgottenAt: "1970-01-01T00:00:00.001Z", + reason: "requested", + }, + { + personId: "c-3", + eventId: "e-1008", + forgottenAt: "2026-11-01T00:00:00.000Z", + reason: "requested", + }, + ...["c-4", "c-5"].map((personId, index) => ({ + personId, + eventId: `e-${1_009 + index}`, + forgottenAt: "2026-11-01T00:00:00.000Z", + reason: "requested", + })), ]); expect(reads).toEqual([ { after: 2, limit: 1_000 }, @@ -97,9 +131,43 @@ describe("parseForgetList", () => { '{"personId":"c-1","eventId":"e-1","forgottenAt":"x"}\r\n\n c-2 \n{"personId":"c-3"}\n', ), ).toStrictEqual([ - { personId: "c-1", eventId: "e-1" }, + { personId: "c-1", eventId: "e-1", forgottenAt: "x" }, { personId: "c-2" }, { personId: "c-3" }, ]); }); + + it("reads back what bounda forgotten prints: the reason and the keys a forgetting names", () => { + expect( + parseForgetList( + '{"personId":"c-1","eventId":"e-1","forgottenAt":"t","reason":"retention","fingerprints":["k-1"]}\n{"personId":"c-2","reason":"requested","fingerprints":[]}', + ), + ).toStrictEqual([ + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "t", + reason: "retention", + fingerprints: ["k-1"], + }, + { personId: "c-2", reason: "requested", fingerprints: [] }, + ]); + }); + + it("refuses a line whose fields are not what bounda forgotten prints, naming it", () => { + for (const line of [ + '{"personId":3}', + '{"personId":"c-1","eventId":3}', + '{"personId":"c-1","forgottenAt":3}', + '{"personId":"c-1","reason":"restored"}', + '{"personId":"c-1","reason":"toString"}', + '{"personId":"c-1","fingerprints":"k-1"}', + '{"personId":"c-1","fingerprints":["k-1",2]}', + ]) { + expect(() => parseForgetList(`c-0\n${line}`)).toThrow( + 'line 2 needs a "personId" string; "eventId" and "forgottenAt" only as strings, "reason" only as "requested" or "retention", and "fingerprints" only as a list of strings', + ); + } + expect(() => parseForgetList("{not json")).toThrow("line 1 is neither a person's id nor JSON"); + }); }); diff --git a/packages/cli/src/forgotten.ts b/packages/cli/src/forgotten.ts index 8f218ff0..5d8cf29a 100644 --- a/packages/cli/src/forgotten.ts +++ b/packages/cli/src/forgotten.ts @@ -3,7 +3,9 @@ import { type ForgetAgainEntry, type GlobalStream, PERSONAL_DATA_FORGOTTEN_EVENT, + type PersonalDataForgottenReason, } from "@bounda-dev/core"; +import type { ForgetLedgerEntry } from "@bounda-dev/core/adapter"; import { InvalidArgumentError } from "commander"; const PAGE = 1_000; @@ -42,11 +44,21 @@ export const parseSince: ParseSinceFunction = (value) => { * One forgetting, as a line of what `bounda forgotten` prints and `bounda forget --from` reads: * the entry a forget ledger keeps. */ -export interface ForgottenLine { - readonly personId: string; - readonly eventId: string; - readonly forgottenAt: string; -} +export type ForgottenLine = ForgetLedgerEntry; + +const fieldOf = (payload: unknown, name: string): unknown => + typeof payload === "object" && payload !== null ? Reflect.get(payload, name) : undefined; + +const isTextList = (value: unknown): value is readonly string[] => + Array.isArray(value) && value.every((item) => typeof item === "string"); + +const REASONS: Readonly> = { + requested: true, + retention: true, +}; + +const isReason = (value: unknown): value is PersonalDataForgottenReason => + typeof value === "string" && Object.hasOwn(REASONS, value); export interface ReadForgottenArgs { readonly events: GlobalStream; @@ -69,10 +81,14 @@ export const readForgotten: ReadForgottenFunction = async ({ events, since }) => if (event.aggregateType !== BOUNDA_AGGREGATE) continue; if (event.type !== PERSONAL_DATA_FORGOTTEN_EVENT) continue; if (since.kind === "date" && Date.parse(event.timestamp) < since.value) continue; + const reason = fieldOf(event.payload, "reason"); + const fingerprints = fieldOf(event.payload, "fingerprints"); lines.push({ personId: event.aggregateId, eventId: event.id, forgottenAt: event.timestamp, + reason: isReason(reason) ? reason : "requested", + ...(isTextList(fingerprints) ? { fingerprints } : {}), }); } const last = page.at(-1); @@ -89,12 +105,29 @@ const entryOf = (line: string, number: number): ForgetAgainEntry => { } catch { throw new Error(`line ${number} is neither a person's id nor JSON`); } - const personId: unknown = Reflect.get(parsed as object, "personId"); - const eventId: unknown = Reflect.get(parsed as object, "eventId"); - if (typeof personId !== "string" || (eventId !== undefined && typeof eventId !== "string")) { - throw new Error(`line ${number} needs a "personId" string, and "eventId" only as a string`); + const personId = fieldOf(parsed, "personId"); + const eventId = fieldOf(parsed, "eventId"); + const forgottenAt = fieldOf(parsed, "forgottenAt"); + const reason = fieldOf(parsed, "reason"); + const fingerprints = fieldOf(parsed, "fingerprints"); + if ( + typeof personId !== "string" || + (eventId !== undefined && typeof eventId !== "string") || + (forgottenAt !== undefined && typeof forgottenAt !== "string") || + (reason !== undefined && !isReason(reason)) || + (fingerprints !== undefined && !isTextList(fingerprints)) + ) { + throw new Error( + `line ${number} needs a "personId" string; "eventId" and "forgottenAt" only as strings, "reason" only as "requested" or "retention", and "fingerprints" only as a list of strings`, + ); } - return eventId === undefined ? { personId } : { personId, eventId }; + return { + personId, + ...(eventId === undefined ? {} : { eventId }), + ...(forgottenAt === undefined ? {} : { forgottenAt }), + ...(reason === undefined ? {} : { reason }), + ...(fingerprints === undefined ? {} : { fingerprints }), + }; }; export interface ParseForgetListFunction { diff --git a/packages/cli/src/run-without-cloudflare.test.ts b/packages/cli/src/run-without-cloudflare.test.ts new file mode 100644 index 00000000..fa66bb90 --- /dev/null +++ b/packages/cli/src/run-without-cloudflare.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { EXIT_FAILURE, runCli } from "./run.ts"; + +// A project that never installed the package the KV ledger comes from. +vi.mock("@bounda-dev/cloudflare/forget-ledger", () => { + throw new Error("Cannot find package '@bounda-dev/cloudflare'"); +}); + +describe("bounda forget-ledger expire without @bounda-dev/cloudflare", () => { + it("says the package is missing, and runs no wrangler", async () => { + let stderr = ""; + let runs = 0; + const code = await runCli({ + argv: ["forget-ledger", "expire", "--binding", "L", "--keep-for", "35d"], + cwd: "/project", + stdout: { write: () => undefined }, + stderr: { + write: (text) => { + stderr += text; + }, + }, + wrangler: () => async () => { + runs += 1; + return "[]"; + }, + }); + expect(code).toBe(EXIT_FAILURE); + expect(stderr).toBe( + "error: @bounda-dev/cloudflare is not installed, and the KV forget ledger comes from it\n", + ); + expect(runs).toBe(0); + }); +}); diff --git a/packages/cli/src/run.test.ts b/packages/cli/src/run.test.ts index 065292e2..be034f0f 100644 --- a/packages/cli/src/run.test.ts +++ b/packages/cli/src/run.test.ts @@ -710,7 +710,13 @@ describe("bounda forgotten and bounda forget", () => { expect(byPosition.code).toBe(EXIT_OK); expect(besidesLogs(byPosition.stderr)).toBe("1 person forgotten\n"); expect(byPosition.stdout).toBe( - `${JSON.stringify({ personId: forgotten, eventId: event?.id, forgottenAt: event?.timestamp })}\n`, + `${JSON.stringify({ + personId: forgotten, + eventId: event?.id, + forgottenAt: event?.timestamp, + reason: "requested", + fingerprints: Reflect.get(Object(event?.payload), "fingerprints"), + })}\n`, ); const byDate = await cli(["forgotten", "--since", before], fixture); expect(byDate.code).toBe(EXIT_OK); @@ -725,7 +731,7 @@ describe("bounda forgotten and bounda forget", () => { expect(besidesLogs(forget.stderr)).toBe(""); expect(forget.code).toBe(EXIT_OK); expect(forget.stdout).toBe( - `forgot again ${kept}\nforgot again 1 of 2 listed; the rest hold no key, or this store holds their forgetting\n`, + `forgot again ${kept}\nforgot again 1 of 2 listed; the rest hold no key the line names, or this store holds their forgetting\n`, ); const after = await open(); const reasons = (await after.events.read({ after: lastPosition, limit: 100 })) @@ -734,7 +740,7 @@ describe("bounda forgotten and bounda forget", () => { await after.stop(); expect(reasons).toEqual([ [forgotten, "requested"], - [kept, "restored"], + [kept, "requested"], ]); }); @@ -748,7 +754,7 @@ describe("bounda forgotten and bounda forget", () => { const forget = await cli(["forget", "--from", file, "--actor", "operator-1"], fixture); expect(forget.code).toBe(EXIT_OK); expect(logged(forget.stderr, "info", "personal data forgotten")).toEqual([ - expect.objectContaining({ personId: memberId, reason: "restored", actor: "operator-1" }), + expect.objectContaining({ personId: memberId, reason: "requested", actor: "operator-1" }), ]); const after = await open(); const forgetting = (await after.events.read({ limit: 1_000 })).find( @@ -758,6 +764,77 @@ describe("bounda forgotten and bounda forget", () => { expect(forgetting?.metadata.actor).toBe("operator-1"); }); + it("lists what the database holds without starting the app, so a strict ledger that is down does not stop it", async () => { + const strict = ["--config", "strict-ledger.config.ts"]; + const listed = await cli(["forgotten", "--since", "0", ...strict], fixture); + expect(listed).toMatchObject({ code: EXIT_OK, stdout: "" }); + expect(besidesLogs(listed.stderr)).toBe("0 persons forgotten\n"); + const booted = await cli(["dead-letters", "list", ...strict], fixture); + expect(booted.code).toBe(EXIT_FAILURE); + expect(booted.stderr).toContain( + "error: The forget ledger could not be read, and runtime.personalData.forgetLedger.strict keeps the app from starting until it is", + ); + }); + + it("fails a listing whose project cannot load, or whose storage cannot open, and closes what opened", async () => { + const missing = await cli( + ["forgotten", "--since", "0", "--config", "missing.config.ts"], + fixture, + ); + expect(missing.code).toBe(EXIT_FAILURE); + expect(missing.stdout).toBe(""); + expect(besidesLogs(missing.stderr).match(/^error: /gm)).toHaveLength(1); + const root = await list(); + await writeFile( + join(root, "registry.ts"), + "export const registry = { aggregates: {}, readModels: {} };\n", + ); + const adapter = (open: string) => + `import { writeFileSync } from "node:fs";\nexport default { storage: { kind: "bounda-adapter", name: "broken", options: {}, createReadModel: async () => undefined, rebuildReadModel: async () => undefined, ${open} } };\n`; + await writeFile( + join(root, "down.config.ts"), + adapter(`createStorage: async () => { throw new Error("database is down"); }`), + ); + const down = await cli( + ["forgotten", "--since", "0", "--config", "down.config.ts", "--registry", "registry.ts"], + root, + ); + expect(down.code).toBe(EXIT_FAILURE); + expect(besidesLogs(down.stderr)).toBe( + "error: Error; its message may quote personal data: run again with --error-text to print it\n", + ); + await writeFile( + join(root, "failing.config.ts"), + adapter( + `createStorage: async () => ({ eventStore: { readAll: async () => { throw new Error("read failed"); } }, close: async () => writeFileSync(${JSON.stringify(join(root, "closed"))}, "") })`, + ), + ); + const failing = await cli( + ["forgotten", "--since", "0", "--config", "failing.config.ts", "--registry", "registry.ts"], + root, + ); + expect(failing.code).toBe(EXIT_FAILURE); + expect(besidesLogs(failing.stderr)).toMatch(/^error: Error; /); + await expect(stat(join(root, "closed"))).resolves.toBeDefined(); + }); + + it("says a storage without its factories cannot be listed", async () => { + const root = await list(); + await writeFile( + join(root, "bounda.config.ts"), + `export default { storage: { kind: "bounda-adapter", name: "elsewhere", options: {} } };\n`, + ); + await writeFile( + join(root, "registry.ts"), + "export const registry = { aggregates: {}, readModels: {} };\n", + ); + const refused = await cli(["forgotten", "--since", "0", "--registry", "registry.ts"], root); + expect(refused.code).toBe(EXIT_FAILURE); + expect(refused.stderr).toContain( + `error: storage "elsewhere" is a definition without factories. Import the adapter package's factory.`, + ); + }); + it("refuses a --since that is neither a position nor a date", async () => { for (const since of ["yesterday", "-3", "", "2026-13-45"]) { const refused = await cli(["forgotten", "--since", since], fixture); @@ -776,7 +853,7 @@ describe("bounda forgotten and bounda forget", () => { expect(missing.stderr).toMatch(/^error: nowhere\.jsonl: ENOENT/); for (const [content, message] of [ ["m-1\n{not json", "line 2 is neither a person's id nor JSON"], - ['{"eventId":"e-1"}', 'line 1 needs a "personId" string, and "eventId" only as a string'], + ['{"eventId":"e-1"}', 'line 1 needs a "personId" string; "eventId" and "forgottenAt"'], ['{"personId":"m-1","eventId":7}', 'line 1 needs a "personId" string'], ] as const) { await writeFile(join(root, "list.jsonl"), content); @@ -807,6 +884,159 @@ describe("bounda forgotten and bounda forget", () => { }); }); +describe("bounda forget-ledger expire", () => { + const ROOT = "bounda/forget-ledger/"; + const expire = async (argv: readonly string[], listed: unknown = []) => { + const stdout = capture(); + const stderr = capture(); + const calls: { readonly root: string; readonly args: readonly string[] }[] = []; + const code = await runCli({ + argv: ["forget-ledger", "expire", ...argv], + cwd: "/project", + stdout, + stderr, + wrangler: (root) => async (args) => { + calls.push({ root, args }); + if (args[1] === "bulk" && args[2] === "put") throw new Error("wrangler: not authorized"); + return JSON.stringify(listed); + }, + }); + return { code, stdout: stdout.text(), stderr: stderr.text(), calls, listed }; + }; + + it("gives the entries their expiration through the project's wrangler, and says how many", async () => { + const day = 86_400_000; + const old = { personId: "c-1", forgottenAt: new Date(Date.now() - 40 * day).toISOString() }; + const done = await expire( + [ + "--binding", + "FORGET_LEDGER", + "--keep-for", + " 35d ", + "--local", + "--env", + "staging", + "--wrangler-config", + "w.jsonc", + "--persist-to", + ".state", + "--root", + "app", + ], + [{ name: `${ROOT}t/1:e-1`, metadata: old }], + ); + expect(done).toMatchObject({ + code: EXIT_OK, + stdout: "gave 0 entries the expiration keepFor sets; removed 1 already past it\n", + stderr: "", + }); + expect(done.calls.map(({ root }) => root)).toEqual([ + resolve("/project", "app"), + resolve("/project", "app"), + ]); + expect(done.calls[0]?.args).toEqual([ + "kv", + "key", + "list", + "--binding", + "FORGET_LEDGER", + "--prefix", + ROOT, + "--local", + "--env", + "staging", + "--config", + "w.jsonc", + "--persist-to", + ".state", + ]); + const one = await expire( + ["--binding", "L", "--keep-for", "35d"], + [{ name: `${ROOT}t/1:e-1`, metadata: { ...old, forgottenAt: new Date().toISOString() } }], + ); + expect(one.code).toBe(EXIT_FAILURE); + expect(one.stderr).toBe( + "error: Error; its message may quote personal data: run again with --error-text to print it\n", + ); + const told = await expire(["--binding", "L", "--keep-for", "35d", "--error-text"], one.listed); + expect(told.stderr).toBe("error: wrangler: not authorized\n"); + expect(one.calls.map(({ root, args }) => [root, args.at(-1)])).toEqual([ + [resolve("/project"), "--remote"], + [resolve("/project"), "--remote"], + ]); + }); + + it("says how many it gave an expiration, one or more", async () => { + const stdout = capture(); + const listed = [1, 2].map((index) => ({ + name: `${ROOT}t/${index}:e-${index}`, + metadata: { personId: "c-1", forgottenAt: new Date().toISOString() }, + })); + for (const keys of [listed.slice(0, 1), listed]) { + await runCli({ + argv: ["forget-ledger", "expire", "--binding", "L", "--keep-for", "35d"], + cwd: "/project", + stdout, + stderr: capture(), + wrangler: () => async (args) => (args[1] === "key" ? JSON.stringify(keys) : ""), + }); + } + expect(stdout.text()).toBe( + "gave 1 entry the expiration keepFor sets; removed 0 already past it\ngave 2 entries the expiration keepFor sets; removed 0 already past it\n", + ); + }); + + it("refuses a keep-for that is not a duration longer than 0, and a missing binding", async () => { + for (const keepFor of ["0s", "soon", "-1d", ""]) { + const refused = await expire(["--binding", "L", "--keep-for", keepFor]); + expect(refused.code).toBe(EXIT_CONVENTION); + expect(refused.stderr).toContain("expected a duration longer than 0, such as 35d"); + expect(refused.calls).toEqual([]); + } + expect((await expire(["--keep-for", "35d"])).code).toBe(EXIT_CONVENTION); + }); + + it("documents itself in its help", async () => { + const parent = await runCli({ + argv: ["forget-ledger", "--help"], + cwd: "/project", + stdout: capture(), + stderr: capture(), + }); + expect(parent).toBe(EXIT_OK); + const group = capture(); + await runCli({ + argv: ["forget-ledger", "--help"], + cwd: "/project", + stdout: group, + stderr: capture(), + }); + expect(group.text()).toContain("look after a forget ledger outside the app"); + const help = (await expire(["--help"])).stdout.replace(/\s+/g, " "); + for (const text of [ + "give every entry of a Workers KV forget ledger the expiration keepFor sets, those written before it was set included", + "--binding ", + "the KV namespace's binding in wrangler.jsonc", + "--keep-for ", + "runtime.personalData.forgetLedger.keepFor, such as 35d", + "--local", + "the local state of wrangler dev instead of the deployed namespace", + "--env ", + "the wrangler environment", + "--wrangler-config ", + "the wrangler configuration file", + "--persist-to ", + "with --local, where wrangler dev keeps its state", + "--root ", + "the project's root directory, where wrangler is installed", + "--error-text", + "print the message of the error the command fails on", + ]) { + expect(help).toContain(text); + } + }); +}); + describe("bounda status", () => { const fixture = resolve(import.meta.dirname, "fixtures/rebuild-project"); const failing = ["--registry", ".bounda/failing-registry.ts"]; diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 39414e74..74dbf69e 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -10,16 +10,24 @@ import { type Logger, MAX_ACTOR_LENGTH, migrateTables, + parseDuration, type Registry, rebuildReadModel, resolveErrorText, type SubscriberFailure, withErrorText, } from "@bounda-dev/core"; +import { type AdapterDefinition, isAdapter, type Storage } from "@bounda-dev/core/adapter"; import type { Config, ErrorText } from "@bounda-dev/core/config"; import { boot, createConsoleLogger, loadProject } from "@bounda-dev/core/node"; import { Command, CommanderError, InvalidArgumentError, Option } from "commander"; import { formatLetter, formatRetried } from "./dead-letter-output.ts"; +import { + expireKvLedger, + type KvLedgerModule, + type Wrangler, + wranglerIn, +} from "./forget-ledger-expire.ts"; import { parseForgetList, parseSince, readForgotten, type Since } from "./forgotten.ts"; import { generateProject } from "./generate/generate.ts"; import { ConventionError } from "./generate/problems.ts"; @@ -40,6 +48,8 @@ export interface RunCliArgs { readonly stdout: Output; readonly stderr: Output; readonly signal?: AbortSignal; + // How `forget-ledger expire` runs wrangler in a project's root; the project's own by default. + readonly wrangler?: (root: string) => Wrangler; } export interface RunCliFunction { @@ -105,7 +115,8 @@ const errorTextArgs = (options: ProjectOptions, config: Config) => ({ config, }); -const rootOf = (options: ProjectOptions, cwd: string): string => resolve(cwd, options.root ?? "."); +const rootOf = (options: Pick, cwd: string): string => + resolve(cwd, options.root ?? "."); // What fails while the project's modules load, or a `BOUNDA_ERROR_TEXT` it does not know, tells of // files, exports and settings, never of data a handler read, so it is printed in full. @@ -304,18 +315,44 @@ interface ForgottenOptions extends ProjectOptions { readonly since: Since; } -const runForgotten = ( +// Reads the event store alone, without starting the app: no ports, read models or forget ledger, +// so a strict ledger that is down does not keep it from listing what the database holds. +const runForgotten = async ( options: ForgottenOptions, cwd: string, stdout: Output, stderr: Output, -): Promise => - withApp(options, cwd, stderr, async (app) => { - const lines = await readForgotten({ events: app.events, since: options.since }); +): Promise => { + const project = await load(options, cwd, logTo(stderr), stderr); + if (project === undefined) return EXIT_FAILURE; + const definition: AdapterDefinition = project.config.storage; + if (!isAdapter(definition)) { + line( + stderr, + `error: storage "${definition.name}" is a definition without factories. Import the adapter package's factory.`, + ); + return EXIT_FAILURE; + } + let storage: Storage | undefined; + try { + storage = await definition.createStorage(); + const { eventStore } = storage; + const lines = await readForgotten({ + events: { + read: ({ after = 0, limit = 1_000 } = {}) => + eventStore.readAll({ afterPosition: after, limit }), + }, + since: options.since, + }); for (const forgotten of lines) line(stdout, JSON.stringify(forgotten)); line(stderr, `${lines.length} person${lines.length === 1 ? "" : "s"} forgotten`); return EXIT_OK; - }); + } catch (error) { + return failed(stderr, error, resolveErrorText(errorTextArgs(options, project.config))); + } finally { + await storage?.close(); + } +}; interface ForgetOptions extends ActingOptions { readonly from: string; @@ -346,12 +383,76 @@ const runForget = async ( for (const personId of forgotten) line(stdout, `forgot again ${personId}`); line( stdout, - `forgot again ${forgotten.length} of ${entries.length} listed; the rest hold no key, or this store holds their forgetting`, + `forgot again ${forgotten.length} of ${entries.length} listed; the rest hold no key the line names, or this store holds their forgetting`, ); return EXIT_OK; }); }; +interface ExpireOptions { + readonly root?: string; + readonly errorText: boolean; + readonly binding: string; + readonly keepFor: number; + readonly local: boolean; + readonly env?: string; + readonly wranglerConfig?: string; + readonly persistTo?: string; +} + +const parseKeepFor = (value: string): number => { + try { + const keepForMs = parseDuration(value.trim()); + if (keepForMs > 0) return keepForMs; + } catch {} + throw new InvalidArgumentError("expected a duration longer than 0, such as 35d"); +}; + +const runExpire = async ( + options: ExpireOptions, + cwd: string, + stdout: Output, + stderr: Output, + wrangler: (root: string) => Wrangler, +): Promise => { + let ledger: KvLedgerModule; + try { + ledger = await import("@bounda-dev/cloudflare/forget-ledger"); + } catch { + line( + stderr, + "error: @bounda-dev/cloudflare is not installed, and the KV forget ledger comes from it", + ); + return EXIT_FAILURE; + } + try { + const { expired, removed } = await expireKvLedger({ + wrangler: wrangler(rootOf(options, cwd)), + ledger, + binding: options.binding, + keepForMs: options.keepFor, + target: { + local: options.local, + env: options.env, + config: options.wranglerConfig, + persistTo: options.persistTo, + }, + now: Date.now(), + }); + line( + stdout, + `gave ${expired} entr${expired === 1 ? "y" : "ies"} the expiration keepFor sets; removed ${removed} already past it`, + ); + return EXIT_OK; + } catch (error) { + return failed( + stderr, + error, + resolveErrorText({ forced: options.errorText ? "include" : undefined, env: process.env }), + ); + } +}; + interface StatusOptions extends ActingOptions { readonly json: boolean; } @@ -469,7 +570,14 @@ const actingOptions = (command: T): T => parseActor, ); -export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal }) => { +export const runCli: RunCliFunction = async ({ + argv, + cwd, + stdout, + stderr, + signal, + wrangler = wranglerIn, +}) => { let exitCode = EXIT_OK; const program = new Command("bounda") .description("Bounda: event sourcing and CQRS for TypeScript") @@ -597,6 +705,29 @@ export const runCli: RunCliFunction = async ({ argv, cwd, stdout, stderr, signal exitCode = await runForget(options, cwd, stdout, stderr); }); + program + .command("forget-ledger") + .description("look after a forget ledger outside the app") + .command("expire") + .description( + "give every entry of a Workers KV forget ledger the expiration keepFor sets, those written before it was set included", + ) + .requiredOption("--binding ", "the KV namespace's binding in wrangler.jsonc") + .requiredOption( + "--keep-for ", + "runtime.personalData.forgetLedger.keepFor, such as 35d", + parseKeepFor, + ) + .option("--local", "the local state of wrangler dev instead of the deployed namespace", false) + .option("--env ", "the wrangler environment") + .option("--wrangler-config ", "the wrangler configuration file") + .option("--persist-to ", "with --local, where wrangler dev keeps its state") + .option("--root ", "the project's root directory, where wrangler is installed") + .option("--error-text", "print the message of the error the command fails on", false) + .action(async (options: ExpireOptions) => { + exitCode = await runExpire(options, cwd, stdout, stderr, wrangler); + }); + actingOptions( program .command("status") diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 19a7de97..7127eee8 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "cli", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 31d37575..684560e4 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -19,6 +19,10 @@ "types": "./dist/index.d.ts", "default": "./dist/index.js" }, + "./forget-ledger": { + "types": "./dist/forget-ledger.d.ts", + "default": "./dist/forget-ledger.js" + }, "./package.json": "./package.json" }, "publishConfig": { diff --git a/packages/cloudflare/src/forget-ledger.ts b/packages/cloudflare/src/forget-ledger.ts index 12f0d4f4..27a768fd 100644 --- a/packages/cloudflare/src/forget-ledger.ts +++ b/packages/cloudflare/src/forget-ledger.ts @@ -22,10 +22,37 @@ const KV_MIN_TTL_S = 60; interface EntryMetadata { readonly personId: string; readonly forgottenAt: string; + // Absent from a key written before entries kept them. + readonly reason?: string; + readonly fingerprints?: readonly string[]; } const timeKey = (forgottenAt: string): string => String(LATEST_MS - Date.parse(forgottenAt)); +/** + * Where every object's entries live, each under its own name. + */ +export const KV_FORGET_LEDGER_ROOT = "bounda/forget-ledger/"; + +// When an entry kept `keepForMs` expires, in seconds as KV takes it; `null` once it is past. +const expirationOf = (forgottenAt: string, keepForMs: number, now: number): number | null => { + const expiresAt = Date.parse(forgottenAt) + keepForMs; + if (!(expiresAt > now)) return null; + return Math.max(Math.ceil(expiresAt / 1000), Math.floor(now / 1000) + KV_MIN_TTL_S); +}; + +const entryOf = (metadata: EntryMetadata, eventId: string): ForgetLedgerEntry => { + const entry: ForgetLedgerEntry = { + personId: metadata.personId, + eventId, + forgottenAt: metadata.forgottenAt, + reason: metadata.reason === "retention" ? "retention" : "requested", + }; + return Array.isArray(metadata.fingerprints) + ? { ...entry, fingerprints: [...metadata.fingerprints] } + : entry; +}; + export interface CreateKvForgetLedgerArgs { readonly namespace: KVNamespace; /** @@ -62,17 +89,17 @@ export const createKvForgetLedger: CreateKvForgetLedgerFunction = ({ append: async (entries) => { const now = Date.now(); await Promise.all( - entries.flatMap(({ personId, eventId, forgottenAt }) => { - const metadata: EntryMetadata = { personId, forgottenAt }; + entries.flatMap(({ personId, eventId, forgottenAt, reason, fingerprints }) => { + const metadata: EntryMetadata = { + personId, + forgottenAt, + reason, + ...(fingerprints === undefined ? {} : { fingerprints }), + }; const key = `${prefix}${timeKey(forgottenAt)}:${eventId}`; if (keepForMs === undefined) return [namespace.put(key, "", { metadata })]; - const expiresAt = Date.parse(forgottenAt) + keepForMs; - if (!(expiresAt > now)) return []; - const expiration = Math.max( - Math.ceil(expiresAt / 1000), - Math.floor(now / 1000) + KV_MIN_TTL_S, - ); - return [namespace.put(key, "", { metadata, expiration })]; + const expiration = expirationOf(forgottenAt, keepForMs, now); + return expiration === null ? [] : [namespace.put(key, "", { metadata, expiration })]; }), ); }, @@ -95,11 +122,7 @@ export const createKvForgetLedger: CreateKvForgetLedgerFunction = ({ break; } if (metadata === null || metadata === undefined) continue; - found.push({ - personId: metadata.personId, - eventId: rest.slice(separator + 1), - forgottenAt: metadata.forgottenAt, - }); + found.push(entryOf(metadata, rest.slice(separator + 1))); } if (past || page.list_complete) break; cursor = page.cursor; @@ -162,7 +185,119 @@ export const kvForgetLedger: KvForgetLedgerFunction = ({ binding }) => } return createKvForgetLedger({ namespace, - prefix: `bounda/forget-ledger/${encodeURIComponent(tenant)}/`, + prefix: `${KV_FORGET_LEDGER_ROOT}${encodeURIComponent(tenant)}/`, keepForMs, }); }); + +/** + * A key of the ledger's namespace as KV lists it. + */ +export interface KvLedgerKey { + readonly name: string; + /** + * When KV expires it, in seconds since 1970; absent for a key kept for good. + */ + readonly expiration?: number | undefined; + readonly metadata?: unknown; +} + +/** + * A key written again, as it was, to expire at `expiration`. + */ +export interface KvLedgerPut { + readonly key: string; + readonly value: ""; + readonly expiration: number; + readonly metadata: unknown; +} + +export interface KvForgetLedgerExpiry { + readonly expire: readonly KvLedgerPut[]; + /** + * The keys already older than `keepFor`, which KV would not take an expiration for. + */ + readonly remove: readonly string[]; +} + +export interface PlanKvForgetLedgerExpiryArgs { + readonly keys: readonly KvLedgerKey[]; + readonly keepForMs: number; + readonly now: number; +} + +export interface PlanKvForgetLedgerExpiryFunction { + (args: PlanKvForgetLedgerExpiryArgs): KvForgetLedgerExpiry; +} + +/** + * What gives every entry the expiration `keepFor` sets, for keys written before `keepFor` was set, + * or while it was longer: a key that expires no later is left alone, and a key that is not an + * entry of the ledger is never touched. + */ +export const planKvForgetLedgerExpiry: PlanKvForgetLedgerExpiryFunction = ({ + keys, + keepForMs, + now, +}) => { + const expire: KvLedgerPut[] = []; + const remove: string[] = []; + for (const { name, expiration: current, metadata } of keys) { + const forgottenAt: unknown = + typeof metadata === "object" && metadata !== null + ? Reflect.get(metadata, "forgottenAt") + : undefined; + if (typeof forgottenAt !== "string" || Number.isNaN(Date.parse(forgottenAt))) continue; + const expiration = expirationOf(forgottenAt, keepForMs, now); + if (expiration === null) remove.push(name); + else if (current === undefined || current > expiration) { + expire.push({ key: name, value: "", expiration, metadata }); + } + } + return { expire, remove }; +}; + +export interface ExpireKvForgetLedgerArgs { + readonly namespace: KVNamespace; + readonly keepForMs: number; + /** + * Defaults to now. + */ + readonly now?: number; +} + +export interface ExpireKvForgetLedgerFunction { + (args: ExpireKvForgetLedgerArgs): Promise; +} + +/** + * Gives every object's entries in the namespace the expiration `keepFor` sets, as + * `planKvForgetLedgerExpiry` decides, from a Worker, as `bounda forget-ledger expire` does from a + * terminal. Resolves to what it changed. Run it once after setting or shortening `keepFor`: the + * entries written from then on expire on their own. + */ +export const expireKvForgetLedger: ExpireKvForgetLedgerFunction = async ({ + namespace, + keepForMs, + now = Date.now(), +}) => { + const keys: KvLedgerKey[] = []; + for (let cursor: string | undefined; ; ) { + const page = await namespace.list({ + prefix: KV_FORGET_LEDGER_ROOT, + limit: LIST_LIMIT, + ...(cursor === undefined ? {} : { cursor }), + }); + keys.push(...page.keys); + if (page.list_complete) break; + cursor = page.cursor; + } + const expiry = planKvForgetLedgerExpiry({ keys, keepForMs, now }); + await Promise.all([ + ...expiry.expire.map(({ key, value, expiration, metadata }) => + namespace.put(key, value, { expiration, metadata }), + ), + ...expiry.remove.map((key) => namespace.delete(key)), + ]); + return expiry; +}; diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index b23048bd..2fe9b737 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -16,8 +16,24 @@ export type { IsCloudflareDefinitionFunction, } from "./definition.ts"; export { cloudflare, isCloudflareDefinition } from "./definition.ts"; -export type { KvForgetLedgerFunction, KvForgetLedgerOptions } from "./forget-ledger.ts"; -export { KV_LIST_DELAY_MS, kvForgetLedger } from "./forget-ledger.ts"; +export type { + ExpireKvForgetLedgerArgs, + ExpireKvForgetLedgerFunction, + KvForgetLedgerExpiry, + KvForgetLedgerFunction, + KvForgetLedgerOptions, + KvLedgerKey, + KvLedgerPut, + PlanKvForgetLedgerExpiryArgs, + PlanKvForgetLedgerExpiryFunction, +} from "./forget-ledger.ts"; +export { + expireKvForgetLedger, + KV_FORGET_LEDGER_ROOT, + KV_LIST_DELAY_MS, + kvForgetLedger, + planKvForgetLedgerExpiry, +} from "./forget-ledger.ts"; export { workersLogger } from "./logger.ts"; export type { RpcOutcome, RpcRefusal } from "./outcome.ts"; export type { DurableSqlStorage } from "./sql-database.ts"; diff --git a/packages/cloudflare/test/forget-ledger.test.ts b/packages/cloudflare/test/forget-ledger.test.ts index 2163e4a7..0c806c08 100644 --- a/packages/cloudflare/test/forget-ledger.test.ts +++ b/packages/cloudflare/test/forget-ledger.test.ts @@ -1,12 +1,27 @@ import { runInDurableObject } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import { ConfigurationError, createApp, type PersonalDataForgottenPayload } from "@bounda-dev/core"; +import { + ConfigurationError, + createApp, + FORGET_LEDGER_GRACE_MS, + ForgetLedgerUnavailableError, + type PersonalDataForgottenPayload, +} from "@bounda-dev/core"; +import type { Adapter } from "@bounda-dev/core/adapter"; import { forgetLedgerContract } from "@bounda-dev/core/adapter/testing"; import { describe, expect, it } from "vitest"; +import { durableObjectAdapter } from "../src/adapter.ts"; import { configForObject } from "../src/bounda-object.ts"; import { connect } from "../src/client.ts"; import { cloudflare } from "../src/definition.ts"; -import { createKvForgetLedger, KV_LIST_DELAY_MS, kvForgetLedger } from "../src/forget-ledger.ts"; +import { + createKvForgetLedger, + expireKvForgetLedger, + KV_FORGET_LEDGER_ROOT, + KV_LIST_DELAY_MS, + kvForgetLedger, + planKvForgetLedgerExpiry, +} from "../src/forget-ledger.ts"; import { quietRegistry } from "./app.ts"; import { clock } from "./clock.ts"; @@ -30,7 +45,13 @@ describe("the forget ledger in Workers KV", () => { kvForgetLedger({ binding: "FORGET_LEDGER" }).create({ env, tenant }); const ada = await ledgerOf("named-ada"); const bob = await ledgerOf("named-bob"); - const entry = { personId: "c-1", eventId: "e-1", forgottenAt: "2026-11-07T12:00:00.000Z" }; + const entry = { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-11-07T12:00:00.000Z", + reason: "requested" as const, + fingerprints: ["k-1"], + }; await ada.append([entry]); expect(await ada.read({ limit: 10 })).toEqual([entry]); expect(await bob.read({ limit: 10 })).toEqual([]); @@ -97,6 +118,7 @@ describe("the KV forget ledger's listing", () => { personId: `c-${index}`, eventId: `e-${String(index).padStart(4, "0")}`, forgottenAt: minute(index), + reason: "requested" as const, })); await ledger.append(entries); expect(await ledger.read({ limit: 2_000 })).toEqual(entries); @@ -110,6 +132,7 @@ describe("the KV forget ledger's listing", () => { personId: `c-${index}`, eventId: `e-${String(index).padStart(4, "0")}`, forgottenAt: minute(index), + reason: "requested" as const, })); await ledger.append(entries); const after = entries[1_002] as (typeof entries)[number]; @@ -128,9 +151,14 @@ describe("the KV forget ledger's listing", () => { const now = Date.now(); const at = (milliseconds: number): string => new Date(now + milliseconds).toISOString(); await ledger.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs - 1_000) }, - { personId: "c-2", eventId: "e-2", forgottenAt: at(-keepForMs + 30_000) }, - { personId: "c-3", eventId: "e-3", forgottenAt: at(0) }, + { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs - 1_000), reason: "requested" }, + { + personId: "c-2", + eventId: "e-2", + forgottenAt: at(-keepForMs + 30_000), + reason: "requested", + }, + { personId: "c-3", eventId: "e-3", forgottenAt: at(0), reason: "requested" }, ]); expect( [...keys.values()].map((metadata) => (metadata as { personId: string }).personId), @@ -142,7 +170,7 @@ describe("the KV forget ledger's listing", () => { expect(later).toBe(Math.ceil((Date.parse(at(0)) + keepForMs) / 1000)); const forever = countingNamespace(); await createKvForgetLedger({ namespace: forever.namespace, prefix: "p/" }).append([ - { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs * 1_000) }, + { personId: "c-1", eventId: "e-1", forgottenAt: at(-keepForMs * 1_000), reason: "requested" }, ]); expect([...forever.expirations.values()]).toEqual([undefined]); }); @@ -154,6 +182,19 @@ describe("the KV forget ledger's listing", () => { expect(await ledger.read({ limit: 10 })).toEqual([]); }); + it("reads a key written before entries kept their reason and keys as a request naming none", async () => { + const { namespace, keys } = countingNamespace(); + const ledger = createKvForgetLedger({ namespace, prefix: "p/" }); + keys.set("p/1:e-1", { personId: "c-1", forgottenAt: minute(0) }); + keys.set("p/2:e-2", { personId: "c-2", forgottenAt: minute(0), reason: "retention" }); + const read = await ledger.read({ limit: 10 }); + expect(read).toEqual([ + { personId: "c-1", eventId: "e-1", forgottenAt: minute(0), reason: "requested" }, + { personId: "c-2", eventId: "e-2", forgottenAt: minute(0), reason: "retention" }, + ]); + expect(read[0]).not.toHaveProperty("fingerprints"); + }); + it("refuses an object without a name, whose entries it could not keep apart", async () => { const { namespace } = countingNamespace(); expect(() => @@ -173,11 +214,24 @@ describe("the KV forget ledger's listing", () => { env: { ...env, LEDGER: namespace } as typeof env, tenant: "acme/eu", }); - await ledger.append([{ personId: "c-1", eventId: "e-1", forgottenAt: minute(0) }]); + await ledger.append([ + { + personId: "c-1", + eventId: "e-1", + forgottenAt: minute(0), + reason: "retention", + fingerprints: ["k-1"], + }, + { personId: "c-2", eventId: "e-2", forgottenAt: minute(0), reason: "requested" }, + ]); expect([...keys.keys()]).toEqual([ - `bounda/forget-ledger/acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-1`, + `${KV_FORGET_LEDGER_ROOT}acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-1`, + `${KV_FORGET_LEDGER_ROOT}acme%2Feu/${8_640_000_000_000_000 - Date.parse(minute(0))}:e-2`, + ]); + expect([...keys.values()]).toEqual([ + { personId: "c-1", forgottenAt: minute(0), reason: "retention", fingerprints: ["k-1"] }, + { personId: "c-2", forgottenAt: minute(0), reason: "requested" }, ]); - expect([...keys.values()]).toEqual([{ personId: "c-1", forgottenAt: minute(0) }]); }); }); @@ -219,6 +273,7 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { await store.forget("ada"); await runInDurableObject(stub, (instance) => instance.alarm()); await runInDurableObject(stub, (_instance, state) => copyTables(state.storage, true)); + clock.advance(FORGET_LEDGER_GRACE_MS); // What the object's constructor runs when it starts again. const outcome = await runInDurableObject(stub, async (_instance, state) => { @@ -246,7 +301,7 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { } }); expect(outcome).toMatchObject({ - forgotten: ["restored"], + forgotten: ["requested"], ada: { note: null }, bob: { note: "door" }, lag: { reconciled: true }, @@ -254,3 +309,196 @@ describe("restoring a Durable Object's storage with a forget ledger", () => { expect(outcome.nextDueAt).toBe(outcome.now + KV_LIST_DELAY_MS); }); }); + +describe("expiring a KV forget ledger", () => { + const DAY = 86_400_000; + const NOW = Date.UTC(2026, 10, 9); + const at = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(NOW - daysAgo * DAY).toISOString(), + }); + + it("plans an expiration for every entry kept for good or longer than keepFor, and the removal of those past it", () => { + // A second short of keepFor: KV expires a key a minute from now at the soonest. + const soonest = { ...at(0), forgottenAt: new Date(NOW - 35 * DAY + 1_000).toISOString() }; + expect( + planKvForgetLedgerExpiry({ + keys: [ + { name: "kept", metadata: at(1) }, + { name: "past", metadata: at(36) }, + { name: "just-past", metadata: at(35) }, + { name: "longer", metadata: at(2), expiration: NOW / 1000 + 100 * 86_400 }, + { name: "sooner", metadata: at(2), expiration: NOW / 1000 + 86_400 }, + { name: "due", metadata: at(2), expiration: (NOW + 33 * DAY) / 1000 }, + { name: "soonest", metadata: soonest }, + { name: "stray" }, + { name: "empty", metadata: null }, + { name: "unreadable", metadata: { forgottenAt: "not a time" } }, + { name: "untimed", metadata: { forgottenAt: 3 } }, + { name: "text", metadata: "kept" }, + ], + keepForMs: 35 * DAY, + now: NOW, + }), + ).toEqual({ + expire: [ + { key: "kept", value: "", expiration: (NOW + 34 * DAY) / 1000, metadata: at(1) }, + { key: "longer", value: "", expiration: (NOW + 33 * DAY) / 1000, metadata: at(2) }, + { + key: "soonest", + value: "", + expiration: NOW / 1000 + 60, + metadata: soonest, + }, + ], + remove: ["past", "just-past"], + }); + }); + + it("writes the plan to the namespace, every object's entries and nothing else", async () => { + const namespace = env.FORGET_LEDGER; + const now = Date.now(); + const written = (daysAgo: number) => ({ + personId: "c-1", + forgottenAt: new Date(now - daysAgo * DAY).toISOString(), + reason: "requested", + }); + await namespace.put(`${KV_FORGET_LEDGER_ROOT}expire-a/1:e-1`, "", { metadata: written(1) }); + await namespace.put(`${KV_FORGET_LEDGER_ROOT}expire-b/2:e-2`, "", { metadata: written(40) }); + await namespace.put("expire-elsewhere", "kept", { metadata: written(1) }); + const expiry = await expireKvForgetLedger({ namespace, keepForMs: 35 * DAY, now }); + expect(expiry.expire.map(({ key }) => key)).toContain(`${KV_FORGET_LEDGER_ROOT}expire-a/1:e-1`); + expect(expiry.remove).toContain(`${KV_FORGET_LEDGER_ROOT}expire-b/2:e-2`); + const listed = await namespace.list({ prefix: KV_FORGET_LEDGER_ROOT }); + expect(listed.keys.find(({ name }) => name.endsWith("expire-a/1:e-1"))).toMatchObject({ + expiration: Math.ceil((now + 34 * DAY) / 1000), + metadata: written(1), + }); + expect(listed.keys.some(({ name }) => name.endsWith("expire-b/2:e-2"))).toBe(false); + expect((await namespace.list({ prefix: "expire-elsewhere" })).keys).toEqual([ + { name: "expire-elsewhere", metadata: written(1) }, + ]); + expect(await expireKvForgetLedger({ namespace, keepForMs: 35 * DAY })).toMatchObject({ + remove: [], + }); + }); + + it("lists every page of a namespace with more keys than a page", async () => { + const { namespace, keys, pages } = countingNamespace(); + for (let index = 0; index < 1_001; index += 1) { + keys.set(`${KV_FORGET_LEDGER_ROOT}t/${index}:e-${index}`, at(0)); + } + const expiry = await expireKvForgetLedger({ namespace, keepForMs: DAY, now: NOW }); + expect(expiry.expire).toHaveLength(1_001); + expect(pages()).toBe(2); + }); +}); + +describe("a Durable Object's forgetting and the KV ledger", () => { + it("records the intention in KV before it commits, and the next start commits one whose commit never happened", async () => { + const stub = env.LEDGER_STORE.get(env.LEDGER_STORE.idFromName("intended-tenant")); + const store = connect(stub); + await store.commands.placeOrder({ orderId: "o-1", total: 7, customer: "ada", note: "bell" }); + const intention = await runInDurableObject(stub, async (_instance, state) => { + const opened = durableObjectAdapter({ storage: state.storage, options: {} }); + const evicted: Adapter = { + ...opened, + createStorage: async () => { + const storage = await opened.createStorage(); + return { + ...storage, + transact: () => Promise.reject(new Error("the object was evicted")), + }; + }, + }; + const app = await createApp({ + registry: quietRegistry, + config: { ...configForObject(config, state.storage), storage: evicted }, + env, + tenant: "intended-tenant", + clock, + }); + try { + await expect(app.forget("ada")).rejects.toThrow("the object was evicted"); + } finally { + await app.stop(); + } + const ledger = await kvForgetLedger({ binding: "FORGET_LEDGER" }).create({ + env, + tenant: "intended-tenant", + }); + const [entry] = await ledger.read({ limit: 10 }); + return entry; + }); + expect(intention).toMatchObject({ + personId: "ada", + reason: "requested", + fingerprints: [expect.any(String)], + }); + clock.advance(FORGET_LEDGER_GRACE_MS); + const outcome = await runInDurableObject(stub, async (_instance, state) => { + const app = await createApp({ + registry: quietRegistry, + config: configForObject(config, state.storage), + env, + tenant: "intended-tenant", + clock, + }); + try { + return { + forgotten: (await app.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => event.id), + ada: (await app.exportPersonalData("ada")).events[0]?.data, + }; + } finally { + await app.stop(); + } + }); + expect(outcome).toEqual({ + forgotten: [intention?.eventId], + ada: expect.objectContaining({ note: null }), + }); + }); + + it("forgets nobody, when strict, while KV does not take the intention", async () => { + const stub = env.LEDGER_STORE.get(env.LEDGER_STORE.idFromName("strict-tenant")); + const store = connect(stub); + await store.commands.placeOrder({ orderId: "o-1", total: 7, customer: "ada", note: "bell" }); + const refusal = await runInDurableObject(stub, async (_instance, state) => { + const down = { + list: (options?: KVNamespaceListOptions) => env.FORGET_LEDGER.list(options), + put: () => Promise.reject(new Error("KV is down")), + }; + const app = await createApp({ + registry: quietRegistry, + config: { + ...configForObject(config, state.storage), + runtime: { + personalData: { + forgetLedger: { store: kvForgetLedger({ binding: "DOWN" }), strict: true }, + }, + }, + }, + env: { ...env, DOWN: down } as typeof env, + tenant: "strict-tenant", + clock, + }); + try { + const error = await app.forget("ada").catch((thrown: unknown) => thrown); + return { + unavailable: error instanceof ForgetLedgerUnavailableError, + code: (error as { code?: string }).code, + ada: (await app.exportPersonalData("ada")).events[0]?.data, + }; + } finally { + await app.stop(); + } + }); + expect(refusal).toEqual({ + unavailable: true, + code: "FORGET_LEDGER_UNAVAILABLE", + ada: expect.objectContaining({ note: "bell" }), + }); + }); +}); diff --git a/packages/cloudflare/tsdown.config.ts b/packages/cloudflare/tsdown.config.ts index f560e8cb..f99e6f61 100644 --- a/packages/cloudflare/tsdown.config.ts +++ b/packages/cloudflare/tsdown.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsdown"; export default defineConfig({ - entry: ["src/index.ts"], + entry: ["src/index.ts", "src/forget-ledger.ts"], tsconfig: "tsconfig.build.json", unbundle: true, dts: true, diff --git a/packages/core/src/adapter/adapter.ts b/packages/core/src/adapter/adapter.ts index 4d36ee57..57268e7e 100644 --- a/packages/core/src/adapter/adapter.ts +++ b/packages/core/src/adapter/adapter.ts @@ -257,6 +257,10 @@ export interface IsAdapterFunction { (value: unknown): value is Adapter; } +/** + * Whether a value is an adapter with its factories, as `sqlite(...)` returns, rather than a bare + * definition: what opens a project's storage outside the app, as the CLI does. + */ export const isAdapter: IsAdapterFunction = (value): value is Adapter => typeof value === "object" && value !== null && diff --git a/packages/core/src/adapter/index.ts b/packages/core/src/adapter/index.ts index 199f2540..db2d1f48 100644 --- a/packages/core/src/adapter/index.ts +++ b/packages/core/src/adapter/index.ts @@ -3,6 +3,7 @@ export type { CommitReadModelRebuildArgs, CreateReadModelArgs, CreateReadModelRebuildArgs, + IsAdapterFunction, ReadModelRebuild, ReadModelStorage, ReadModelTransactArgs, @@ -13,6 +14,7 @@ export type { Storage, StorageTransaction, } from "./adapter.ts"; +export { isAdapter } from "./adapter.ts"; export type { AdapterDefinition } from "./adapter-definition.ts"; export type { RebuildFencing, RebuildFencingFunction } from "./rebuild-fencing.ts"; export { rebuildFencing } from "./rebuild-fencing.ts"; diff --git a/packages/core/src/adapter/sql/forget-ledger.test.ts b/packages/core/src/adapter/sql/forget-ledger.test.ts index 75f77dd4..d24c3e45 100644 --- a/packages/core/src/adapter/sql/forget-ledger.test.ts +++ b/packages/core/src/adapter/sql/forget-ledger.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; +import type { ForgetLedgerEntry } from "../storage/forget-ledger.ts"; import { postgresqlDialect, sqliteDialect } from "./dialect.ts"; -import { createSqlForgetLedger, forgetLedgerStatements } from "./forget-ledger.ts"; +import { + createSqlForgetLedger, + forgetLedgerReasonStatements, + forgetLedgerStatements, +} from "./forget-ledger.ts"; const recording = () => { const statements: (readonly [string, readonly unknown[]])[] = []; @@ -16,10 +21,12 @@ const recording = () => { return { db, statements }; }; -const entry = (index: number) => ({ +const entry = (index: number): ForgetLedgerEntry => ({ personId: `c-${index}`, eventId: `e-${index}`, forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + fingerprints: [`k-${index}`], }); describe("the SQL forget ledger", () => { @@ -34,7 +41,14 @@ describe("the SQL forget ledger", () => { ); }); - it("inserts at most 25 entries a statement, each bound to its own numbered parameters", async () => { + it("adds the reason, a request for the entries before it, and the keys, none recorded", () => { + expect(forgetLedgerReasonStatements(`"t"`)).toEqual([ + `ALTER TABLE "t" ADD COLUMN "reason" TEXT NOT NULL DEFAULT 'requested'`, + `ALTER TABLE "t" ADD COLUMN "fingerprints" TEXT`, + ]); + }); + + it("inserts at most 16 entries a statement, each bound to its own numbered parameters", async () => { const { db, statements } = recording(); const ledger = createSqlForgetLedger({ db, @@ -44,15 +58,35 @@ describe("the SQL forget ledger", () => { ensureSchema: async () => {}, close: async () => {}, }); - await ledger.append(Array.from({ length: 26 }, (_, index) => entry(index))); + const { fingerprints: _, ...unrecorded } = entry(16); + await ledger.append([ + ...Array.from({ length: 16 }, (_, index) => entry(index)), + { ...unrecorded, reason: "retention" }, + ]); expect(statements).toHaveLength(2); const [first, second] = statements; - expect(first?.[0]).toContain("($1, $2, $3, $4), ($5, $6, $7, $8)"); - expect(first?.[0]).toContain("($97, $98, $99, $100) ON CONFLICT"); - expect(first?.[1]).toHaveLength(100); - expect(first?.[1].slice(0, 4)).toEqual(["", "e-0", "c-0", "2026-01-01T00:00:00.000Z"]); - expect(second?.[0]).toContain(`VALUES ($1, $2, $3, $4) ON CONFLICT`); - expect(second?.[1]).toEqual(["", "e-25", "c-25", "2026-01-01T00:00:00.000Z"]); + expect(first?.[0]).toContain( + `INSERT INTO "t" ("tenant", "event_id", "person", "forgotten_at", "reason", "fingerprints") VALUES ($1, $2, $3, $4, $5, $6), ($7, $8, $9, $10, $11, $12)`, + ); + expect(first?.[0]).toContain("($91, $92, $93, $94, $95, $96) ON CONFLICT"); + expect(first?.[1]).toHaveLength(96); + expect(first?.[1].slice(0, 6)).toEqual([ + "", + "e-0", + "c-0", + "2026-01-01T00:00:00.000Z", + "requested", + `["k-0"]`, + ]); + expect(second?.[0]).toContain(`VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT`); + expect(second?.[1]).toEqual([ + "", + "e-16", + "c-16", + "2026-01-01T00:00:00.000Z", + "retention", + null, + ]); }); it("creates its table before the first statement, and again after that failed", async () => { diff --git a/packages/core/src/adapter/sql/forget-ledger.ts b/packages/core/src/adapter/sql/forget-ledger.ts index bcbd106e..4079bb3a 100644 --- a/packages/core/src/adapter/sql/forget-ledger.ts +++ b/packages/core/src/adapter/sql/forget-ledger.ts @@ -3,16 +3,17 @@ import type { SqlDialect } from "./dialect.ts"; import { quoteIdentifier } from "./identifiers.ts"; import type { SqlExecutor } from "./sql-table.ts"; -// Four parameters a row, within the hundred a Durable Object's SQLite binds to one statement. -const ENTRIES_PER_INSERT = 25; +const COLUMNS = ["tenant", "event_id", "person", "forgotten_at", "reason", "fingerprints"] as const; +// Within the hundred parameters a Durable Object's SQLite binds to one statement. +const ENTRIES_PER_INSERT = Math.floor(100 / COLUMNS.length); export interface ForgetLedgerStatementsFunction { (table: string, dialect: Pick): readonly string[]; } /** - * The ledger's table, `table` already quoted. PostgreSQL compares the ordering columns by code - * unit, as SQLite does, rather than by the database's collation. + * The ledger's table as its first version created it, `table` already quoted. PostgreSQL compares + * the ordering columns by code unit, as SQLite does, rather than by the database's collation. */ export const forgetLedgerStatements: ForgetLedgerStatementsFunction = (table, { name }) => { const ordered = name === "postgresql" ? ` COLLATE "C"` : ""; @@ -54,6 +55,19 @@ export const forgetLedgerMarksStatement: ForgetLedgerMarksStatementFunction = (t PRIMARY KEY ("tenant", "name") )`; +export interface ForgetLedgerReasonStatementsFunction { + (table: string): readonly string[]; +} + +/** + * The third version: each entry keeps its reason and the keys it names. An entry written before + * reads as a request that names none, which any key matches. + */ +export const forgetLedgerReasonStatements: ForgetLedgerReasonStatementsFunction = (table) => [ + `ALTER TABLE ${table} ADD COLUMN "reason" TEXT NOT NULL DEFAULT 'requested'`, + `ALTER TABLE ${table} ADD COLUMN "fingerprints" TEXT`, +]; + export interface CreateSqlForgetLedgerArgs { readonly db: SqlExecutor; /** @@ -73,11 +87,18 @@ export interface CreateSqlForgetLedgerFunction { (args: CreateSqlForgetLedgerArgs): ForgetLedger; } -const entryOf = (row: Record): ForgetLedgerEntry => ({ - personId: String(row.person), - eventId: String(row.event_id), - forgottenAt: String(row.forgotten_at), -}); +const entryOf = (row: Record): ForgetLedgerEntry => { + const entry: ForgetLedgerEntry = { + personId: String(row.person), + eventId: String(row.event_id), + forgottenAt: String(row.forgotten_at), + reason: row.reason === "retention" ? "retention" : "requested", + }; + const fingerprints: unknown = JSON.parse(String(row.fingerprints ?? null)); + return Array.isArray(fingerprints) ? { ...entry, fingerprints: fingerprints.map(String) } : entry; +}; + +const SELECTED = `"event_id", "person", "forgotten_at", "reason", "fingerprints"`; /** * The forget ledger of SQLite and PostgreSQL alike, in a table of its own that holds every @@ -107,16 +128,18 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ for (let start = 0; start < entries.length; start += ENTRIES_PER_INSERT) { const chunk = entries.slice(start, start + ENTRIES_PER_INSERT); const rows = chunk.map( - (_, index) => - `(${p(index * 4 + 1)}, ${p(index * 4 + 2)}, ${p(index * 4 + 3)}, ${p(index * 4 + 4)})`, + (_, row) => + `(${COLUMNS.map((_, column) => p(row * COLUMNS.length + column + 1)).join(", ")})`, ); await db.run( - `INSERT INTO ${table} ("tenant", "event_id", "person", "forgotten_at") VALUES ${rows.join(", ")} ON CONFLICT ("tenant", "event_id") DO NOTHING`, - chunk.flatMap(({ eventId, personId, forgottenAt }) => [ + `INSERT INTO ${table} (${COLUMNS.map((column) => `"${column}"`).join(", ")}) VALUES ${rows.join(", ")} ON CONFLICT ("tenant", "event_id") DO NOTHING`, + chunk.flatMap(({ eventId, personId, forgottenAt, reason, fingerprints }) => [ tenant, eventId, personId, forgottenAt, + reason, + fingerprints === undefined ? null : JSON.stringify(fingerprints), ]), ); } @@ -126,11 +149,11 @@ export const createSqlForgetLedger: CreateSqlForgetLedgerFunction = ({ const rows = after === undefined ? await db.all( - `SELECT "event_id", "person", "forgotten_at" FROM ${table} WHERE "tenant" = ${p(1)} ORDER BY "forgotten_at", "event_id" LIMIT ${p(2)}`, + `SELECT ${SELECTED} FROM ${table} WHERE "tenant" = ${p(1)} ORDER BY "forgotten_at", "event_id" LIMIT ${p(2)}`, [tenant, limit], ) : await db.all( - `SELECT "event_id", "person", "forgotten_at" FROM ${table} WHERE "tenant" = ${p(1)} AND ("forgotten_at", "event_id") > (${p(2)}, ${p(3)}) ORDER BY "forgotten_at", "event_id" LIMIT ${p(4)}`, + `SELECT ${SELECTED} FROM ${table} WHERE "tenant" = ${p(1)} AND ("forgotten_at", "event_id") > (${p(2)}, ${p(3)}) ORDER BY "forgotten_at", "event_id" LIMIT ${p(4)}`, [tenant, after.forgottenAt, after.eventId, limit], ); return rows.map(entryOf); diff --git a/packages/core/src/adapter/sql/index.ts b/packages/core/src/adapter/sql/index.ts index b582384a..b40d294f 100644 --- a/packages/core/src/adapter/sql/index.ts +++ b/packages/core/src/adapter/sql/index.ts @@ -24,6 +24,7 @@ export { createSqlForgetLedger, forgetLedgerMarksStatement, forgetLedgerMarksTableFor, + forgetLedgerReasonStatements, forgetLedgerStatements, } from "./forget-ledger.ts"; export { diff --git a/packages/core/src/adapter/sqlite/migrations.ts b/packages/core/src/adapter/sqlite/migrations.ts index 8fc20368..81b65840 100644 --- a/packages/core/src/adapter/sqlite/migrations.ts +++ b/packages/core/src/adapter/sqlite/migrations.ts @@ -2,7 +2,11 @@ import { ConfigurationError } from "../../contracts/errors.ts"; import type { SqlDatabase, SqlTransaction } from "../sql/database.ts"; import { resolvedAtOfEarlierLetters } from "../sql/dead-letters.ts"; import { sqliteDialect } from "../sql/dialect.ts"; -import { forgetLedgerMarksStatement, forgetLedgerStatements } from "../sql/forget-ledger.ts"; +import { + forgetLedgerMarksStatement, + forgetLedgerReasonStatements, + forgetLedgerStatements, +} from "../sql/forget-ledger.ts"; import { quoteIdentifier } from "../sql/identifiers.ts"; import { migrateSchemas, @@ -153,6 +157,12 @@ export const sqliteForgetLedgerScope: SqliteForgetLedgerScopeFunction = (table) }, }, { version: 2, migrate: (tx) => tx.run(forgetLedgerMarksStatement(table), []) }, + { + version: 3, + migrate: async (tx) => { + for (const statement of forgetLedgerReasonStatements(table)) await tx.run(statement, []); + }, + }, ], }); diff --git a/packages/core/src/adapter/storage/forget-ledger.ts b/packages/core/src/adapter/storage/forget-ledger.ts index a6ec584f..96039e84 100644 --- a/packages/core/src/adapter/storage/forget-ledger.ts +++ b/packages/core/src/adapter/storage/forget-ledger.ts @@ -1,7 +1,10 @@ +import type { PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; + /** - * One forgetting as the ledger keeps it: whom, which `PersonalDataForgotten` said so, and when. - * Ids and a time, nothing else: the ledger is a suppression list, the one thing about a forgotten - * person that has to outlive their data. + * One forgetting as the ledger keeps it: whom, which `PersonalDataForgotten` says so, when, why, + * and which keys it destroys. Ids, a time and fingerprints, nothing else: the ledger is a + * suppression list, the one thing about a forgotten person that has to outlive their data. The + * runtime writes it before the forgetting commits, so an entry may name an event no store holds. */ export interface ForgetLedgerEntry { readonly personId: string; @@ -14,6 +17,14 @@ export interface ForgetLedgerEntry { * The event's timestamp, as `Date.prototype.toISOString` writes it. */ readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + /** + * The fingerprints of the keys the person held when the forgetting was asked: the reconciliation + * commits it again only while the person holds one of them, since any other key was given to + * them after. Absent from an entry written by a Bounda that did not record them, which any key + * matches. + */ + readonly fingerprints?: readonly string[]; } /** @@ -44,14 +55,14 @@ export interface PruneForgetLedgerArgs { /** * The record of every person forgotten, kept outside the database, so restoring a copy of the * database taken before someone was forgotten can forget them again. The runtime appends each - * forgetting to it as soon as it commits, and again from the stream, and reads it when the app - * starts and every `reconcileEvery` while it runs. One ledger holds one store: a ledger built for - * a tenant sees only that tenant's entries. + * forgetting to it before the forgetting commits, and again from the stream once it has, and + * reads it when the app starts and every `reconcileEvery` while it runs. One ledger holds one + * store: a ledger built for a tenant sees only that tenant's entries. */ export interface ForgetLedger { /** - * Adds the entries, one per `eventId`: an event names one person at one time, so appending the - * same entries again leaves the ledger as it was. + * Adds the entries, one per `eventId`, each with every field it has: an event names one person + * at one time, so appending the same entries again leaves the ledger as it was. */ append(entries: readonly ForgetLedgerEntry[]): Promise; /** diff --git a/packages/core/src/adapter/testing/forget-ledger.contract.ts b/packages/core/src/adapter/testing/forget-ledger.contract.ts index 040a51d4..6936d79e 100644 --- a/packages/core/src/adapter/testing/forget-ledger.contract.ts +++ b/packages/core/src/adapter/testing/forget-ledger.contract.ts @@ -16,6 +16,8 @@ const entry = (personId: string, eventId: string, minute: number): ForgetLedgerE personId, eventId, forgottenAt: new Date(Date.UTC(2026, 10, 7, 12, minute)).toISOString(), + reason: "requested", + fingerprints: [`k-${personId}`], }); const readAll = async (ledger: ForgetLedger, limit = 2): Promise => { @@ -89,6 +91,20 @@ export const forgetLedgerContract: ForgetLedgerContractFunction = ({ create }) = ]); }); + it("keeps each entry's reason and the keys it names, none, or none recorded", async () => { + const retained: ForgetLedgerEntry = { + ...entry("c-1", "e-1", 1), + reason: "retention", + fingerprints: ["k-1", "k-2"], + }; + const keyless: ForgetLedgerEntry = { ...entry("c-2", "e-2", 2), fingerprints: [] }; + const { fingerprints: _, ...unrecorded } = entry("c-3", "e-3", 3); + await ledger.append([retained, keyless, unrecorded]); + const read = await ledger.read({ limit: 10 }); + expect(read).toEqual([retained, keyless, unrecorded]); + expect(read[2]).not.toHaveProperty("fingerprints"); + }); + it("keeps one person's forgettings apart, each under its event", async () => { await ledger.append([entry("c-1", "e-1", 1), entry("c-1", "e-2", 2)]); expect(await ledger.read({ limit: 10 })).toEqual([ diff --git a/packages/core/src/adapter/testing/inspection.contract.ts b/packages/core/src/adapter/testing/inspection.contract.ts index 410f903a..bb64691a 100644 --- a/packages/core/src/adapter/testing/inspection.contract.ts +++ b/packages/core/src/adapter/testing/inspection.contract.ts @@ -157,7 +157,13 @@ export const inspectionContract: InspectionContractFunction = ({ create }) => { const adapter = await create(); const opened = await adapter.createForgetLedger({ tenant: "t-1" }); await opened.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: "2026-11-07T12:00:00.000Z" }, + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-11-07T12:00:00.000Z", + reason: "requested", + fingerprints: [], + }, ]); const ledger = tableOf(await adapter.inspect(), "bounda_forget_ledger"); const { rows } = ledger; diff --git a/packages/core/src/adapter/testing/storage-schema.contract.ts b/packages/core/src/adapter/testing/storage-schema.contract.ts index f6fa5a94..d5b41246 100644 --- a/packages/core/src/adapter/testing/storage-schema.contract.ts +++ b/packages/core/src/adapter/testing/storage-schema.contract.ts @@ -149,18 +149,18 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ expect(versions).toEqual([ { scope: "storage", version: 0, latest: 2 }, { scope: "checkpoints", version: 0, latest: 1 }, - { scope: "forget-ledger", version: 0, latest: 2 }, + { scope: "forget-ledger", version: 0, latest: 3 }, ]); expect(await shapeOf(target, dialect)).toEqual([]); expect(await target.adapter.migrate(["checkpoints"])).toEqual([ { scope: "storage", version: 0, latest: 2 }, { scope: "checkpoints", version: 1, latest: 1 }, - { scope: "forget-ledger", version: 0, latest: 2 }, + { scope: "forget-ledger", version: 0, latest: 3 }, ]); expect(await target.adapter.migrate(["storage", "forget-ledger"])).toEqual([ { scope: "storage", version: 2, latest: 2 }, { scope: "checkpoints", version: 1, latest: 1 }, - { scope: "forget-ledger", version: 2, latest: 2 }, + { scope: "forget-ledger", version: 3, latest: 3 }, ]); const shape = await shapeOf(target, dialect); await opened(target.adapter, async () => undefined); @@ -300,7 +300,12 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ const ledger = await target.adapter.createForgetLedger({ tenant: "t-1" }); try { await ledger.append([ - { personId: "p-1", eventId: "e-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "p-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ]); } finally { await ledger.close?.(); @@ -308,6 +313,7 @@ export const storageSchemaContract: StorageSchemaContractFunction = ({ expect((await versionsOf(target)).map(({ scope, version }) => ({ scope, version }))).toEqual([ { scope: "forget-ledger", version: 1 }, { scope: "forget-ledger", version: 2 }, + { scope: "forget-ledger", version: 3 }, ]); }); diff --git a/packages/core/src/config/defaults.ts b/packages/core/src/config/defaults.ts index 0ea29566..1b654e11 100644 --- a/packages/core/src/config/defaults.ts +++ b/packages/core/src/config/defaults.ts @@ -42,6 +42,10 @@ export const DEFAULT_CATCH_UP_POLL_MS: number = 15; * The `reconcileEvery` of `runtime.personalData.forgetLedger` when it is left out. */ export const DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS: number = 5 * 60_000; +/** + * The `timeout` of `runtime.personalData.forgetLedger` when it is left out. + */ +export const DEFAULT_FORGET_LEDGER_TIMEOUT_MS: number = 10_000; /** * The share of the loads from a snapshot checked against the whole stream when `snapshot.verify` * is left out. diff --git a/packages/core/src/config/index.ts b/packages/core/src/config/index.ts index 3ee7ed4a..6cd53eb6 100644 --- a/packages/core/src/config/index.ts +++ b/packages/core/src/config/index.ts @@ -17,6 +17,7 @@ export type { MasterKeyFunction, MasterKeyOptions } from "../kernel/personal-dat export { masterKey } from "../kernel/personal-data/master-key.ts"; export { DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, DEFAULT_SNAPSHOT_VERIFY, DEFAULT_SNAPSHOT_VERSION, } from "./defaults.ts"; diff --git a/packages/core/src/config/schema.test.ts b/packages/core/src/config/schema.test.ts index b503efe5..46cbbfc2 100644 --- a/packages/core/src/config/schema.test.ts +++ b/packages/core/src/config/schema.test.ts @@ -4,7 +4,10 @@ import { ConfigurationError } from "../contracts/errors.ts"; import { defineForgetLedger } from "../kernel/personal-data/forget-ledger.ts"; import { masterKey } from "../kernel/personal-data/master-key.ts"; import { memory } from "../memory/index.ts"; -import { DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS } from "./defaults.ts"; +import { + DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, +} from "./defaults.ts"; import { resolveConfig } from "./schema.ts"; const sqlite: AdapterDefinition<"sqlite", { path: string }> = { @@ -91,7 +94,13 @@ describe("resolveConfig", () => { personalData: { retention: "30d", keyWrapper, - forgetLedger: { store: forgetLedger, reconcileEvery: "1m", strict: true, keepFor: "90d" }, + forgetLedger: { + store: forgetLedger, + reconcileEvery: "1m", + strict: true, + keepFor: "90d", + timeout: "3s", + }, }, deadLetters: { retention: "7d" }, }, @@ -128,6 +137,7 @@ describe("resolveConfig", () => { reconcileEveryMs: 60_000, strict: true, keepForMs: 7_776_000_000, + timeoutMs: 3_000, }, }); expect(resolved.runtime.deadLetters).toEqual({ retentionMs: 604_800_000 }); @@ -141,6 +151,7 @@ describe("resolveConfig", () => { reconcileEveryMs: DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, strict: false, keepForMs: null, + timeoutMs: DEFAULT_FORGET_LEDGER_TIMEOUT_MS, }); expect(defaulted?.store).toBe(ledgerDatabase); expect(resolved.readModels["users-directory"]).toBe(sqlite); @@ -284,6 +295,7 @@ describe("resolveConfig", () => { for (const [option, value] of [ ["reconcileEvery", 0], ["keepFor", "0s"], + ["timeout", 0], ] as const) { expect( message({ diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.ts index 2e30afa7..1219b61d 100644 --- a/packages/core/src/config/schema.ts +++ b/packages/core/src/config/schema.ts @@ -19,6 +19,7 @@ import { DEFAULT_COMMANDS, DEFAULT_CONCURRENCY_RETRIES, DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, + DEFAULT_FORGET_LEDGER_TIMEOUT_MS, DEFAULT_IDEMPOTENCY_KEY_LIFETIME_MS, DEFAULT_IDLE_INTERVAL_MS, DEFAULT_POLICIES, @@ -102,6 +103,7 @@ const forgetLedger = z reconcileEvery: timeout.optional(), strict: z.boolean().optional(), keepFor: timeout.optional(), + timeout: timeout.optional(), }) .refine( ({ reconcileEvery = DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, keepFor }) => @@ -226,6 +228,7 @@ const resolveForgetLedger = ( reconcileEveryMs: parsed.reconcileEvery ?? DEFAULT_FORGET_LEDGER_RECONCILE_EVERY_MS, strict: parsed.strict ?? false, keepForMs: parsed.keepFor ?? null, + timeoutMs: parsed.timeout ?? DEFAULT_FORGET_LEDGER_TIMEOUT_MS, }; const formatIssues = (issues: readonly z.core.$ZodIssue[]): string => diff --git a/packages/core/src/config/types.ts b/packages/core/src/config/types.ts index be183c87..45184718 100644 --- a/packages/core/src/config/types.ts +++ b/packages/core/src/config/types.ts @@ -155,12 +155,11 @@ export interface PersonalDataConfig { } /** - * The `runtime.personalData.forgetLedger` of the configuration. The runtime appends each - * forgetting to the ledger as soon as it commits, and copies it again from the stream in the - * background, in case the process stops in between. When the app starts, and every - * `reconcileEvery` while it runs, it reads the ledger and forgets again, with `reason: - * "restored"`, whoever it names whose forgetting the database does not hold and who still holds - * a key. + * The `runtime.personalData.forgetLedger` of the configuration. Every forgetting writes its + * intention to the ledger before it commits, and the stream copies each committed one again in + * the background. When the app starts, and every `reconcileEvery` while it runs, it reads the + * ledger and commits again each forgetting the database does not hold whose person still holds a + * key it names: one a restored backup rolled back, or one whose commit never happened. */ export interface ForgetLedgerConfig { /** @@ -179,10 +178,14 @@ export interface ForgetLedgerConfig { */ readonly reconcileEvery?: DurationInput; /** - * When `true`, the app does not start until it has read the ledger through: a ledger that cannot - * be read fails the start with `ForgetLedgerUnreconciledError`, and the start waits for the - * reconciliation however long it takes. When `false`, the default, the app starts anyway, waits - * for the ledger at most a few seconds, and reconciles in the background. + * When `true`, nothing is forgotten that the ledger has not recorded, and the app does not + * start until it has read the ledger through: a forgetting whose intention the ledger does not + * take fails with `ForgetLedgerUnavailableError` and forgets nobody, so calling again is safe; + * a ledger that cannot be read fails the start with `ForgetLedgerUnreconciledError`, and the + * start waits for the reconciliation however long it takes. When `false`, the default, a + * forgetting the ledger does not take is logged and commits all the same, for the stream to copy + * later, and the app starts after waiting for the ledger at most `timeout`, reconciling in the + * background. */ readonly strict?: boolean; /** @@ -191,6 +194,12 @@ export interface ForgetLedgerConfig { * default: entries are kept for good. */ readonly keepFor?: DurationInput; + /** + * How long each call to the ledger may take before it is given up with + * `ForgetLedgerTimeoutError`, and how long a start that is not strict waits for the + * reconciliation. Defaults to `DEFAULT_FORGET_LEDGER_TIMEOUT_MS` (ten seconds). + */ + readonly timeout?: DurationInput; } export interface ResolvedForgetLedgerConfig { @@ -201,6 +210,7 @@ export interface ResolvedForgetLedgerConfig { * `null` when entries are kept for good. */ readonly keepForMs: number | null; + readonly timeoutMs: number; } /** diff --git a/packages/core/src/contracts/duration.ts b/packages/core/src/contracts/duration.ts index e0644c55..b7315bee 100644 --- a/packages/core/src/contracts/duration.ts +++ b/packages/core/src/contracts/duration.ts @@ -33,6 +33,10 @@ export interface ParseDurationFunction { (input: LooseDurationInput): number; } +/** + * The milliseconds of a duration written as the configuration takes it, `"35d"` or a number of + * milliseconds. Throws `ValidationError` for anything else, or a negative number. + */ export const parseDuration: ParseDurationFunction = (input) => { if (typeof input === "number") { if (!Number.isFinite(input) || input < 0) { diff --git a/packages/core/src/contracts/errors.test.ts b/packages/core/src/contracts/errors.test.ts index 2f31b2dc..da6942ab 100644 --- a/packages/core/src/contracts/errors.test.ts +++ b/packages/core/src/contracts/errors.test.ts @@ -7,6 +7,8 @@ import { DeadLetterNotRetriableError, DeadLetterSettledError, DomainError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, NotFoundError, RebuildSupersededError, @@ -69,6 +71,24 @@ describe("errors", () => { expect(error.message).toContain("runtime.personalData.forgetLedger.strict"); }); + it("say a strict forget ledger refused the forgetting, with why", () => { + const cause = new Error("down"); + const error = new ForgetLedgerUnavailableError(cause); + expect(error.code).toBe("FORGET_LEDGER_UNAVAILABLE"); + expect(error.name).toBe("ForgetLedgerUnavailableError"); + expect(error.cause).toBe(cause); + expect(error.message).toContain("nothing was forgotten"); + }); + + it("name the forget ledger call that ran out of time, and the time", () => { + const error = new ForgetLedgerTimeoutError("append", 2500); + expect(error.code).toBe("FORGET_LEDGER_TIMEOUT"); + expect(error.name).toBe("ForgetLedgerTimeoutError"); + expect(error.operation).toBe("append"); + expect(error.timeoutMs).toBe(2500); + expect(error.message).toBe("The forget ledger did not answer append within 2500 ms"); + }); + it("name the read model whose rebuild was taken over", () => { const error = new RebuildSupersededError("orderSummary"); expect(error.code).toBe("REBUILD_SUPERSEDED"); diff --git a/packages/core/src/contracts/errors.ts b/packages/core/src/contracts/errors.ts index 66e99c51..718fd8cd 100644 --- a/packages/core/src/contracts/errors.ts +++ b/packages/core/src/contracts/errors.ts @@ -273,6 +273,41 @@ export class ForgetLedgerUnreconciledError extends BoundaError { } } +/** + * Thrown by `forget`, and by every other forgetting, when `runtime.personalData.forgetLedger.strict` + * is set and the forget ledger did not take the forgetting's intention: nothing was forgotten, and + * calling again tries again. `cause` is why the ledger failed, a `ForgetLedgerTimeoutError` when + * it did not answer in time. + */ +export class ForgetLedgerUnavailableError extends BoundaError { + constructor(cause: unknown) { + super( + "FORGET_LEDGER_UNAVAILABLE", + "The forget ledger did not take the forgetting, and runtime.personalData.forgetLedger.strict forgets nobody it has not recorded: nothing was forgotten, try again", + { cause }, + ); + } +} + +/** + * A call to the forget ledger did not settle within `runtime.personalData.forgetLedger.timeout`: + * the `cause` of `ForgetLedgerUnavailableError` and `ForgetLedgerUnreconciledError`, and what the + * app logs when the ledger is not strict. The call may still land after it. + */ +export class ForgetLedgerTimeoutError extends BoundaError { + readonly operation: string; + readonly timeoutMs: number; + + constructor(operation: string, timeoutMs: number) { + super( + "FORGET_LEDGER_TIMEOUT", + `The forget ledger did not answer ${operation} within ${timeoutMs} ms`, + ); + this.operation = operation; + this.timeoutMs = timeoutMs; + } +} + export interface StorageSchemaTooNewErrorArgs { readonly scope: string; readonly version: number; diff --git a/packages/core/src/contracts/index.ts b/packages/core/src/contracts/index.ts index f85a08c8..b6028c3c 100644 --- a/packages/core/src/contracts/index.ts +++ b/packages/core/src/contracts/index.ts @@ -19,8 +19,9 @@ export type { DurationString, DurationUnit, LooseDurationInput, + ParseDurationFunction, } from "./duration.ts"; -export { asDuration } from "./duration.ts"; +export { asDuration, parseDuration } from "./duration.ts"; export { BoundaError, ChainDepthExceededError, @@ -33,6 +34,8 @@ export { DeadLetterSettledError, type DeadLetterSettledErrorArgs, DomainError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, IdempotencyKeyConflictError, type IdempotencyKeyConflictErrorArgs, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b15a1f32..7a589994 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -62,6 +62,7 @@ export type { ForgetAgainEntry, ForgetLedgerReconciliationStatus, } from "./kernel/personal-data/reconciliation.ts"; +export { FORGET_LEDGER_GRACE_MS } from "./kernel/personal-data/reconciliation.ts"; export type { RetentionStatus } from "./kernel/personal-data/retention.ts"; export type { KeyRotationStatus } from "./kernel/personal-data/rotation.ts"; export type { ProcessStatus } from "./kernel/process/lifecycle.ts"; diff --git a/packages/core/src/kernel/app.ts b/packages/core/src/kernel/app.ts index e8ec3fd6..45630a10 100644 --- a/packages/core/src/kernel/app.ts +++ b/packages/core/src/kernel/app.ts @@ -2,7 +2,12 @@ import type { ObservableResult } from "@opentelemetry/api"; import { isAdapter } from "../adapter/adapter.ts"; import { checkConfigNames } from "../config/names.ts"; import { resolveConfig } from "../config/schema.ts"; -import type { Config, ResolvedConfig, RuntimeRole } from "../config/types.ts"; +import type { + Config, + ResolvedConfig, + ResolvedForgetLedgerConfig, + RuntimeRole, +} from "../config/types.ts"; import { type Clock, systemClock } from "../contracts/clock.ts"; import type { CommandRejection, DispatchResult, StoredDispatch } from "../contracts/command.ts"; import { ConfigurationError, ForgetLedgerUnreconciledError } from "../contracts/errors.ts"; @@ -35,21 +40,27 @@ import { createPersonalDataBackfill } from "./personal-data/backfill.ts"; import { exportPersonalData, type PersonalDataExport } from "./personal-data/export.ts"; import { forgetPersonalData, logForgotten } from "./personal-data/forget.ts"; import { - FORGET_LEDGER_WAIT_MS, + type BoundedForgetLedger, + boundForgetLedger, openForgetLedger, waitForLedger, } from "./personal-data/forget-ledger.ts"; -import { withLedgerAppend } from "./personal-data/ledger-append.ts"; +import { createForgetIntentions } from "./personal-data/intention.ts"; import { createLedgerOutbox } from "./personal-data/ledger-outbox.ts"; import { marksPersonalFields } from "./personal-data/marks.ts"; import { createForgetLedgerReconciliation, type ForgetAgainEntry, forgetAgain, + NEVER_DUE, } from "./personal-data/reconciliation.ts"; import { marksFingerprint } from "./personal-data/resealing.ts"; import { liftProcessingRestriction, restrictProcessing } from "./personal-data/restriction.ts"; -import { createRetentionSubscriber, createRetentionSweep } from "./personal-data/retention.ts"; +import { + createRetentionSubscriber, + createRetentionSweep, + retentionDueOf, +} from "./personal-data/retention.ts"; import { createKeyRotation } from "./personal-data/rotation.ts"; import { checkPersonId } from "./personal-data/transact.ts"; import { type PersonalDataWitness, witnessedWrapping } from "./personal-data/witness.ts"; @@ -132,7 +143,10 @@ export interface BoundaClient { * the stream `bounda:`, which every read model that projects personal fields handles. * Both happen together or not at all. Resolves like a command, so * `catchUpReadModels({ through })` waits for those read models. Rejects with - * `ValidationError` an empty id. + * `ValidationError` an empty id. With `runtime.personalData.forgetLedger`, the forgetting is in + * the ledger before it commits: one that rejects after that still forgets the person at the next + * reconciliation, and calling again is safe; a strict ledger that does not take it rejects with + * `ForgetLedgerUnavailableError`, forgetting nobody. */ forget(personId: string): Promise; /** @@ -251,13 +265,13 @@ export interface BoundaApp extends BoundaClien */ readonly deadLetters: DeadLetters; /** - * Forgets again, as `forget` does with `reason: "restored"`, each person an entry names who - * still holds a key, unless the store holds the `PersonalDataForgotten` the entry names by - * `eventId`: a store that holds it applied that forgetting, and a key the person holds was given - * to them after. What `bounda forget --from` runs, for a backup restored without - * `runtime.personalData.forgetLedger`; resolves to the ids of the persons it forgot, in order. - * Rejects with `ValidationError` an entry without a person's id, or an `actor` that `as` would - * refuse, before forgetting anyone. + * Commits again each forgetting an entry names, as a reconciliation does: unless the store holds + * the `PersonalDataForgotten` the entry names by `eventId`, and only while the person holds a key + * it names, any key when it names none; a whole entry under its own id, time and reason, a bare + * person's id as a request. What `bounda forget --from` runs, for a backup restored without + * `runtime.personalData.forgetLedger`; with one, each is recorded in it first, as `forget` does. + * Resolves to the ids of the persons it forgot, in order. Rejects with `ValidationError` an entry + * without a person's id, or an `actor` that `as` would refuse, before forgetting anyone. */ forgetAgain( entries: readonly ForgetAgainEntry[], @@ -480,22 +494,35 @@ export const assembleApp: AssembleAppFunction = async ({ opened.unshift(() => adapted.close()); const ledgerConfig = config.runtime.personalData.forgetLedger; const keepForMs = ledgerConfig?.keepForMs ?? null; - const ledger = await openForgetLedger({ - source: ledgerConfig?.store ?? null, - storage: config.storage, - env, - tenant, - keepForMs: keepForMs ?? undefined, - }); - if (ledger !== null) witness?.onForgetLedger(ledger); - if (ledger?.close !== undefined) { - const close = ledger.close.bind(ledger); - opened.unshift(close); + let forgetLedger: { + readonly config: ResolvedForgetLedgerConfig; + readonly store: BoundedForgetLedger; + } | null = null; + if (ledgerConfig !== null) { + const ledger = await openForgetLedger({ + source: ledgerConfig.store, + storage: config.storage, + env, + tenant, + keepForMs: keepForMs ?? undefined, + }); + witness?.onForgetLedger(ledger); + if (ledger.close !== undefined) opened.unshift(ledger.close.bind(ledger)); + forgetLedger = { + config: ledgerConfig, + store: boundForgetLedger({ ledger, timeoutMs: ledgerConfig.timeoutMs, clock }), + }; } - const written = - ledger === null - ? adapted - : withLedgerAppend({ storage: adapted, ledger, keepForMs, clock, logger }); + const intentions = createForgetIntentions({ + ledger: + forgetLedger === null + ? null + : { store: forgetLedger.store, strict: forgetLedger.config.strict }, + ids, + clock, + logger, + }); + const written = adapted; const aggregates = buildAggregates({ registry, ports: ports.byAggregate, config }); const processDefinitions = buildProcesses({ registry, aggregates, config }); const storage = appStorage({ @@ -558,6 +585,7 @@ export const assembleApp: AssembleAppFunction = async ({ clock, pendingRetries, logger, + intentions, }); const policies = buildPolicies({ registry, aggregates }); const policyExecutor = createPolicyExecutor({ @@ -567,6 +595,7 @@ export const assembleApp: AssembleAppFunction = async ({ ids, clock, logger, + intentions, }); const reactive = [ { @@ -607,6 +636,7 @@ export const assembleApp: AssembleAppFunction = async ({ ? undefined : createRetentionSweep({ storage: written, + intentions, retentionMs, ids, clock, @@ -627,20 +657,34 @@ export const assembleApp: AssembleAppFunction = async ({ logger, }); const { concurrencyRetries } = config.runtime.commands; - const reconciliation = - ledger === null || ledgerConfig === null - ? undefined - : createForgetLedgerReconciliation({ - ledger, - storage: written, - ids, - clock, - logger, - concurrencyRetries, - reconcileEveryMs: ledgerConfig.reconcileEveryMs, - keepForMs, - forgotten: wrapping.drop, + const retentionDue = + retentionMs === null + ? NEVER_DUE + : retentionDueOf({ + retentionMs, + batchSize: config.runtime.dispatcher.batchSize, + checkpoints: written.checkpointStore, }); + const reconciling = + forgetLedger === null + ? null + : { + config: forgetLedger.config, + reconciliation: createForgetLedgerReconciliation({ + ledger: forgetLedger.store, + intentions, + retentionDue, + storage: written, + ids, + clock, + logger, + concurrencyRetries, + reconcileEveryMs: forgetLedger.config.reconcileEveryMs, + keepForMs, + forgotten: wrapping.drop, + }), + }; + const reconciliation = reconciling?.reconciliation; const rotation = wrapping.wrapper === null ? undefined @@ -671,7 +715,7 @@ export const assembleApp: AssembleAppFunction = async ({ const restores = createRestoreWatch({ checkpoints: written.checkpointStore, events: written.eventStore, - marks: ledger, + marks: forgetLedger?.store ?? null, logger, }); const dispatcher = createDispatcher({ @@ -702,7 +746,9 @@ export const assembleApp: AssembleAppFunction = async ({ ...(retention === undefined ? [] : [createRetentionSubscriber({ persons: storage.persons })]), - ...(ledger === null ? [] : [createLedgerOutbox({ ledger, keepForMs, clock })]), + ...(forgetLedger === null + ? [] + : [createLedgerOutbox({ ledger: forgetLedger.store, keepForMs, clock })]), ], batchSize: config.runtime.dispatcher.batchSize, pollIntervalMs: config.runtime.dispatcher.pollIntervalMs, @@ -840,18 +886,17 @@ export const assembleApp: AssembleAppFunction = async ({ // the read models that show them hear of it by the time the start resolves. A ledger that // does not answer in time is left to finish behind the start, as one that fails is retried, // unless the ledger is strict, whose start waits and fails with it. - if (reconciliation !== undefined) { - const reconciling = reconciliation.reconcile(clock.now()); - const forgotten = ledgerConfig?.strict - ? await reconciling - : await waitForLedger(clock, reconciling); + if (reconciling !== null) { + const { strict, timeoutMs } = reconciling.config; + const started = reconciling.reconciliation.reconcile(clock.now()); + const forgotten = strict ? await started : await waitForLedger(clock, timeoutMs, started); if (forgotten === "late") { logger.error( "forget ledger did not answer in time; the app starts and reconciles in the background", - { timeoutMs: FORGET_LEDGER_WAIT_MS }, + { timeoutMs }, ); } else if (typeof forgotten !== "number") { - if (ledgerConfig?.strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); + if (strict) throw new ForgetLedgerUnreconciledError(forgotten.failed); } else if (forgotten > 0) { // The ledger held a forgetting the database does not: a restore, whatever the mark said. if (cold === null) { @@ -946,6 +991,7 @@ export const assembleApp: AssembleAppFunction = async ({ run: () => forgetPersonalData({ storage, + intentions, ids, clock, logger, @@ -1027,7 +1073,7 @@ export const assembleApp: AssembleAppFunction = async ({ await storage.close(); } finally { wrapping.close(); - await ledger?.close?.(); + await forgetLedger?.store.close?.(); } } } finally { @@ -1068,10 +1114,13 @@ export const assembleApp: AssembleAppFunction = async ({ for (const entry of entries) { const again = await forgetAgain({ storage: written, + intentions, ids, clock, concurrencyRetries, entry, + retentionDue, + record: true, actor, }); if (again === null) continue; diff --git a/packages/core/src/kernel/command/reaction-commands.test.ts b/packages/core/src/kernel/command/reaction-commands.test.ts index 56662937..a838b2b2 100644 --- a/packages/core/src/kernel/command/reaction-commands.test.ts +++ b/packages/core/src/kernel/command/reaction-commands.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import { pendingEvent } from "../../adapter/testing/fixtures.ts"; import { ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import type { Registry } from "../../modules/registry.ts"; +import { boundForgetLedger } from "../personal-data/forget-ledger.ts"; +import { createForgetIntentions } from "../personal-data/intention.ts"; import { createReactionCommandIds } from "../shared/idempotency-key.ts"; import { createKernelHarness, @@ -10,6 +14,7 @@ import { drained, orderRegistry, slowJob, + withoutLedger, } from "../test-support.ts"; import { createUnitOfWork } from "../unit-of-work/unit-of-work.ts"; import { scheduledCommandKey } from "./pipeline.ts"; @@ -25,6 +30,7 @@ const setUp = async (registry: Registry = orderRegistry) => { const harness = await createKernelHarness({ registry }); const unit = createUnitOfWork({ storage: harness.storage }); const { logger, entries } = createRecordingLogger(); + const ids = createSequentialIdGenerator(); const reaction = createReactionCommands({ aggregates: harness.aggregates, pipeline: harness.pipeline, @@ -32,8 +38,9 @@ const setUp = async (registry: Registry = orderRegistry) => { idempotencyKey: "key-1", within: unit, logger, - ids: createSequentialIdGenerator(), + ids, clock: harness.clock, + intentions: withoutLedger({ ids, clock: harness.clock }), maxChainDepth: 5, }); return { ...harness, reaction, unit, entries }; @@ -44,6 +51,48 @@ const forgottenEvents = async (storage: Storage) => (event) => event.type === "PersonalDataForgotten", ); +const withLedger = async () => { + const harness = await createKernelHarness({ registry: orderRegistry }); + const unit = createUnitOfWork({ storage: harness.storage }); + const recorded: ForgetLedgerEntry[] = []; + const keysAtRecord: unknown[] = []; + const ids = createSequentialIdGenerator(); + const { logger, entries } = createRecordingLogger(); + const reaction = createReactionCommands({ + aggregates: harness.aggregates, + pipeline: harness.pipeline, + context, + idempotencyKey: "key-1", + within: unit, + logger, + ids, + clock: harness.clock, + intentions: createForgetIntentions({ + ledger: { + store: boundForgetLedger({ + ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (appended) => { + recorded.push(...appended); + keysAtRecord.push(await harness.storage.personalKeys.get("c-1")); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock: harness.clock, + }), + strict: true, + }, + ids, + clock: harness.clock, + logger, + }), + maxChainDepth: 5, + }); + return { harness, unit, reaction, recorded, keysAtRecord, entries }; +}; + describe("createReactionCommands", () => { it("dispatches one causal hop deeper, with ids derived from the idempotency key", async () => { const { reaction, storage, unit } = await setUp(); @@ -360,6 +409,66 @@ describe("createReactionCommands", () => { }); }); + it("records the forgetting's intention in the ledger just before the commit's transaction", async () => { + const { harness, unit, reaction, recorded, keysAtRecord } = await withLedger(); + await harness.storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-1", kid: null }); + await reaction.forget("c-1"); + expect(recorded).toEqual([]); + await unit.commit(); + expect(keysAtRecord).toEqual([{ key: "k", fingerprint: "f-1", kid: null }]); + const [forgotten] = await forgottenEvents(harness.storage); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: forgotten?.id, + forgottenAt: forgotten?.timestamp, + reason: "requested", + fingerprints: ["f-1"], + }, + ]); + }); + + it("forgets a key given to the person after the intention too, in the same commit", async () => { + const { harness, unit, reaction, recorded } = await withLedger(); + await reaction.forget("c-1"); + unit.beforeCommit(async () => { + await harness.storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-2", kid: null }); + }); + await unit.commit(); + expect(recorded.map(({ fingerprints }) => fingerprints)).toEqual([[]]); + expect(await harness.storage.personalKeys.get("c-1")).toBeNull(); + const [forgotten] = await forgottenEvents(harness.storage); + expect(forgotten).toMatchObject({ + id: recorded[0]?.eventId, + timestamp: recorded[0]?.forgottenAt, + payload: { reason: "requested", fingerprints: ["f-2"] }, + }); + }); + + it("answers nothing more for a forgetting a reconciliation committed under the intention's id", async () => { + const { harness, unit, reaction, entries } = await withLedger(); + await reaction.forget("c-1"); + unit.beforeCommit(async () => { + await harness.storage.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + pendingEvent({ + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "id-1", + }), + ], + }); + }); + await unit.commit(); + expect((await forgottenEvents(harness.storage)).map(({ id }) => id)).toEqual(["id-1"]); + expect(entries.map(({ message }) => message)).not.toContain("personal data forgotten"); + }); + it("forgets nobody for a unit that never commits", async () => { const { reaction, storage } = await setUp(); await storage.personalKeys.add("c-1", { key: "k", fingerprint: "f", kid: null }); @@ -383,6 +492,7 @@ describe("createReactionCommands", () => { }); const harness = await setUp(); + const deepIds = createSequentialIdGenerator(); const deep = createReactionCommands({ aggregates: harness.aggregates, pipeline: harness.pipeline, @@ -390,8 +500,9 @@ describe("createReactionCommands", () => { idempotencyKey: "key-2", within: harness.unit, logger: createRecordingLogger().logger, - ids: createSequentialIdGenerator(), + ids: deepIds, clock: harness.clock, + intentions: withoutLedger({ ids: deepIds, clock: harness.clock }), maxChainDepth: 5, }); await expect(deep.forget("c-1")).rejects.toMatchObject({ diff --git a/packages/core/src/kernel/command/reaction-commands.ts b/packages/core/src/kernel/command/reaction-commands.ts index 05ef9c05..836d4a3b 100644 --- a/packages/core/src/kernel/command/reaction-commands.ts +++ b/packages/core/src/kernel/command/reaction-commands.ts @@ -9,7 +9,8 @@ import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; import type { CausationContext } from "../../contracts/metadata.ts"; import type { AggregatesRuntime } from "../aggregate/runtime.ts"; -import { forgetWithin, logForgotten } from "../personal-data/forget.ts"; +import { forgetWithin, logForgotten, STALE_INTENTION } from "../personal-data/forget.ts"; +import type { ForgetIntention, ForgetIntentions } from "../personal-data/intention.ts"; import { checkPersonId } from "../personal-data/transact.ts"; import { createReactionCommandIds } from "../shared/idempotency-key.ts"; import { errorDetails } from "../shared/retry.ts"; @@ -58,7 +59,9 @@ export interface ReactionCommands { readonly signal: AbortSignal; /** * Forgets the person when the run's attempt commits, in its transaction: once however often the - * attempt runs, never for one that does not commit, and once for a person named twice. + * attempt runs, and once for a person named twice. Its intention goes to the forget ledger just + * before the transaction opens, so an attempt whose transaction then fails leaves it for the + * next reconciliation, which commits it. */ forget(personId: string): Promise; /** @@ -94,6 +97,7 @@ export interface CreateReactionCommandsArgs { readonly logger: Logger; readonly ids: IdGenerator; readonly clock: Clock; + readonly intentions: ForgetIntentions; /** * How deep a `forget` may go, which no pipeline checks for it as it does for a command. */ @@ -120,6 +124,7 @@ export const createReactionCommands: CreateReactionCommandsFunction = ({ logger, ids, clock, + intentions, maxChainDepth, }) => { const commandIds = createReactionCommandIds(idempotencyKey); @@ -211,15 +216,32 @@ export const createReactionCommands: CreateReactionCommandsFunction = ({ if (depth > maxChainDepth) throw new ChainDepthExceededError(depth, maxChainDepth); if (forgetting.has(personId)) return; forgetting.add(personId); - within.atCommit(async (transaction) => { - const forgotten = await forgetWithin({ - transaction, - ids, - clock, + let intention: ForgetIntention | undefined; + within.beforeCommit(async (storage) => { + intention = await intentions.intend({ personId, reason: "requested", - context: { ...context, depth }, + keys: storage.personalKeys, }); + }); + within.atCommit(async (transaction) => { + const intended = intention as ForgetIntention; + const forget = (named: ForgetIntention) => + forgetWithin({ + transaction, + ids, + clock, + personId, + intention: named, + context: { ...context, depth }, + }); + // A key given after the intention goes too: the commit is one, and a run whose commit + // fails is delivered again, with an intention that names it. + let outcome = await forget(intended); + if (outcome === STALE_INTENTION) + outcome = await forget({ ...intended, fingerprints: undefined }); + if (outcome === STALE_INTENTION || "held" in outcome) return; + const forgotten = outcome; within.afterCommit(() => logForgotten(logger, forgotten)); }); }, diff --git a/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts index d339c678..93476859 100644 --- a/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts +++ b/packages/core/src/kernel/dispatch/subscriber-failures-app.test.ts @@ -340,7 +340,7 @@ describe("the status of the app", () => { { database: "storage", scope: "storage", version: 2, latest: 2 }, { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, - { database: "forgetLedger", scope: "forget-ledger", version: 2, latest: 2 }, + { database: "forgetLedger", scope: "forget-ledger", version: 3, latest: 3 }, ]); expect(status).toMatchObject({ forgetLedger: { reconciled: true, held: null }, diff --git a/packages/core/src/kernel/migrate.test.ts b/packages/core/src/kernel/migrate.test.ts index 3434d064..ca193782 100644 --- a/packages/core/src/kernel/migrate.test.ts +++ b/packages/core/src/kernel/migrate.test.ts @@ -28,7 +28,7 @@ describe("migrateTables", () => { { database: "storage", scope: "storage", version: 2, latest: 2 }, { database: "storage", scope: "checkpoints", version: 1, latest: 1 }, { database: "readModels.customers", scope: "checkpoints", version: 1, latest: 1 }, - { database: "forgetLedger", scope: "forget-ledger", version: 2, latest: 2 }, + { database: "forgetLedger", scope: "forget-ledger", version: 3, latest: 3 }, ]); expect(tablesIn(readModels.db)).toEqual(["bounda_checkpoints", "bounda_migrations"]); expect(tablesIn(ledger.db)).toEqual([ diff --git a/packages/core/src/kernel/personal-data/commands.test.ts b/packages/core/src/kernel/personal-data/commands.test.ts index 73eda185..e61d86af 100644 --- a/packages/core/src/kernel/personal-data/commands.test.ts +++ b/packages/core/src/kernel/personal-data/commands.test.ts @@ -373,6 +373,7 @@ describe("personal data in delayed commands", () => { expect(await forgottenPayload()).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [expect.any(String)], cancelledCommands: named.map((entry) => ({ commandId: entry.dedupeKey.replace(/^command:/, ""), type: entry.command.type, @@ -417,6 +418,7 @@ describe("personal data in delayed commands", () => { expect(await forgottenPayload()).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [key?.fingerprint], cancelledCommands: [], removedDeadLetters: [{ id: stored?.id, commandType: "RemindCustomer" }], }); diff --git a/packages/core/src/kernel/personal-data/forget-ledger.ts b/packages/core/src/kernel/personal-data/forget-ledger.ts index a1c75b97..6b19238a 100644 --- a/packages/core/src/kernel/personal-data/forget-ledger.ts +++ b/packages/core/src/kernel/personal-data/forget-ledger.ts @@ -2,7 +2,7 @@ import type { Adapter } from "../../adapter/adapter.ts"; import type { AdapterDefinition } from "../../adapter/adapter-definition.ts"; import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; import type { Clock } from "../../contracts/clock.ts"; -import { ConfigurationError } from "../../contracts/errors.ts"; +import { ConfigurationError, ForgetLedgerTimeoutError } from "../../contracts/errors.ts"; import type { AppEnv } from "../../register/index.ts"; export interface CreateForgetLedgerFromEnvArgs { @@ -67,12 +67,12 @@ export const isForgetLedgerSource: IsForgetLedgerSourceFunction = ( }; export interface OpenForgetLedgerArgs extends CreateForgetLedgerFromEnvArgs { - readonly source: ForgetLedgerSource | null; + readonly source: ForgetLedgerSource; readonly storage: AdapterDefinition; } export interface OpenForgetLedgerFunction { - (args: OpenForgetLedgerArgs): Promise; + (args: OpenForgetLedgerArgs): Promise; } export interface SharesStorageFunction { @@ -97,7 +97,6 @@ export const openForgetLedger: OpenForgetLedgerFunction = async ({ tenant, keepForMs, }) => { - if (source === null) return null; if (source.kind === "bounda-adapter" && sharesStorage(source, storage)) { throw new ConfigurationError( "runtime.personalData.forgetLedger.store points at the storage's own database, which a restore rolls back with it: give it another database", @@ -108,21 +107,76 @@ export const openForgetLedger: OpenForgetLedgerFunction = async ({ : source.createForgetLedger({ tenant, keepForMs }); }; -// How long the app waits for the ledger, at start or after a forgetting, before it goes on and -// lets the ledger finish behind. -export const FORGET_LEDGER_WAIT_MS = 10_000; +export interface BoundedForgetLedger extends ForgetLedger { + /** + * Whether a call given up for its timeout has not settled yet. + */ + late(): boolean; +} + +export interface BoundForgetLedgerArgs { + readonly ledger: ForgetLedger; + readonly timeoutMs: number; + readonly clock: Clock; +} + +export interface BoundForgetLedgerFunction { + (args: BoundForgetLedgerArgs): BoundedForgetLedger; +} + +/** + * The ledger whose calls reject with `ForgetLedgerTimeoutError` once `timeoutMs` passes without + * them settling. A call given up that way goes on behind, and may still land. + */ +export const boundForgetLedger: BoundForgetLedgerFunction = ({ ledger, timeoutMs, clock }) => { + let late = 0; + const bound = async (operation: string, call: () => Promise): Promise => { + const work = call(); + let cancel: () => void = () => undefined; + const timeout = new Promise((_, reject) => { + cancel = clock.after(timeoutMs, () => { + late += 1; + const settle = (): void => { + late -= 1; + }; + work.then(settle, settle); + reject(new ForgetLedgerTimeoutError(operation, timeoutMs)); + }); + }); + try { + return await Promise.race([work, timeout]); + } finally { + cancel(); + } + }; + const { prune, close } = ledger; + return { + append: (entries) => bound("append", () => ledger.append(entries)), + read: (args) => bound("read", () => ledger.read(args)), + readMark: (name) => bound("readMark", () => ledger.readMark(name)), + raiseMark: (name, value) => bound("raiseMark", () => ledger.raiseMark(name, value)), + ...(ledger.visibilityDelayMs === undefined + ? {} + : { visibilityDelayMs: ledger.visibilityDelayMs }), + ...(prune === undefined + ? {} + : { prune: (args) => bound("prune", () => prune.call(ledger, args)) }), + ...(close === undefined ? {} : { close: () => close.call(ledger) }), + late: () => late > 0, + }; +}; export interface WaitForLedgerFunction { - (clock: Clock, work: Promise): Promise; + (clock: Clock, timeoutMs: number, work: Promise): Promise; } /** - * `work`, or `"late"` once `FORGET_LEDGER_WAIT_MS` has passed without it settling. + * `work`, or `"late"` once `timeoutMs` has passed without it settling. */ -export const waitForLedger: WaitForLedgerFunction = async (clock, work) => { +export const waitForLedger: WaitForLedgerFunction = async (clock, timeoutMs, work) => { let cancel: () => void = () => undefined; const late = new Promise<"late">((resolve) => { - cancel = clock.after(FORGET_LEDGER_WAIT_MS, () => resolve("late")); + cancel = clock.after(timeoutMs, () => resolve("late")); }); try { return await Promise.race([work, late]); diff --git a/packages/core/src/kernel/personal-data/forget.test.ts b/packages/core/src/kernel/personal-data/forget.test.ts index 80f16fd0..808d5436 100644 --- a/packages/core/src/kernel/personal-data/forget.test.ts +++ b/packages/core/src/kernel/personal-data/forget.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { KeyedPerson } from "../../adapter/storage/personal-key-store.ts"; import { + pendingEvent, testCommand, testContext, testDeadLetter, @@ -12,17 +14,22 @@ import { createFixedClock } from "../../contracts/clock.ts"; import { ConcurrencyError, ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { memory } from "../../memory/index.ts"; -import { createRecordingLogger } from "../test-support.ts"; -import { forgetPersonalData } from "./forget.ts"; +import { createRecordingLogger, withoutLedger } from "../test-support.ts"; +import { forgetIntended, forgetPersonalData, forgetWithin, STALE_INTENTION } from "./forget.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; +import { createForgetIntentions } from "./intention.ts"; const setup = async (storage?: Storage) => { const live = storage ?? (await memory().createStorage()); const { logger, entries: logs } = createRecordingLogger(); + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(); const forget = (personId: string) => forgetPersonalData({ storage: live, - ids: createSequentialIdGenerator(), - clock: createFixedClock(), + intentions: withoutLedger({ ids, clock }), + ids, + clock, logger, concurrencyRetries: 2, personId, @@ -55,6 +62,7 @@ describe("forgetPersonalData", () => { payload: { personId: "c-1", reason: "requested", + fingerprints: ["f-1"], cancelledCommands: [], removedDeadLetters: [], }, @@ -131,6 +139,7 @@ describe("forgetPersonalData", () => { expect(event?.payload).toEqual({ personId: "c-1", reason: "requested", + fingerprints: [], cancelledCommands: [ { commandId: "cmd-1", type: "RemindCustomer" }, { commandId: "deadline:x", type: "RemindCustomer" }, @@ -187,3 +196,177 @@ describe("forgetPersonalData", () => { await expect(forget("c-1")).rejects.toBeInstanceOf(ConcurrencyError); }); }); + +describe("forgetIntended", () => { + const recordingIntentions = (ids = createSequentialIdGenerator()) => { + const recorded: ForgetLedgerEntry[] = []; + const clock = createFixedClock(); + const intentions = createForgetIntentions({ + ledger: { + store: boundForgetLedger({ + ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, + ids, + clock, + logger: createRecordingLogger().logger, + }); + return { intentions, recorded, ids, clock }; + }; + + it("takes a new intention that names a key given to the person between the intention and the commit", async () => { + const base = await memory().createStorage(); + await base.personalKeys.add("c-1", testPersonalKey("1")); + let raced = false; + const racing: Storage = { + ...base, + transact: async (work) => { + if (!raced) { + raced = true; + // Another forgetting committed meanwhile, and the person came back with a new key. + await base.personalKeys.remove("c-1"); + await base.personalKeys.add("c-1", testPersonalKey("2")); + } + return base.transact(work); + }, + }; + const { intentions, recorded, ids, clock } = recordingIntentions(); + const forgotten = await forgetIntended({ + storage: racing, + intentions, + ids, + clock, + concurrencyRetries: 0, + personId: "c-1", + reason: "requested", + context: testContext, + }); + expect(recorded.map(({ eventId, fingerprints }) => [eventId, fingerprints])).toEqual([ + ["id-1", ["f-1"]], + ["id-2", ["f-2"]], + ]); + expect(forgotten).toMatchObject({ + hadKey: true, + event: { id: "id-2", payload: { fingerprints: ["f-2"] } }, + }); + expect(await base.personalKeys.get("c-1")).toBeNull(); + }); + + it("forgets a person whose key went since the intention, naming none", async () => { + const base = await memory().createStorage(); + await base.personalKeys.add("c-1", testPersonalKey("1")); + let raced = false; + const racing: Storage = { + ...base, + transact: async (work) => { + if (!raced) { + raced = true; + await base.personalKeys.remove("c-1"); + } + return base.transact(work); + }, + }; + const { intentions, recorded, ids, clock } = recordingIntentions(); + const forgotten = await forgetIntended({ + storage: racing, + intentions, + ids, + clock, + concurrencyRetries: 0, + personId: "c-1", + reason: "requested", + context: testContext, + }); + expect(recorded).toHaveLength(1); + expect(forgotten).toMatchObject({ hadKey: false, payload: { fingerprints: [] } }); + }); + + it("answers with the forgetting a reconciliation committed under the intention's id meanwhile", async () => { + const { storage, logs } = await setup(); + const intended = { + eventId: "id-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested" as const, + fingerprints: [], + }; + await storage.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + { + ...pendingEvent({ + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "id-1", + }), + person: "c-1", + }, + ], + }); + const dispatch = await forgetPersonalData({ + storage, + intentions: { intend: async () => intended, record: async () => undefined }, + ids: createSequentialIdGenerator({ prefix: "other" }), + clock: createFixedClock(), + logger: createRecordingLogger().logger, + concurrencyRetries: 0, + personId: "c-1", + }); + expect(dispatch).toMatchObject({ eventIds: ["id-1"], version: 1, position: 1 }); + expect(await storage.eventStore.lastPosition()).toBe(1); + expect(logs).toEqual([]); + }); + + it("reads the held event and checks the key only when its caller has not", async () => { + const storage = await memory().createStorage(); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + let reads = 0; + const forget = (checked: boolean, fingerprints: readonly string[]) => + storage.transact((transaction) => { + const { readOne } = transaction.eventStore; + return forgetWithin({ + transaction: { + ...transaction, + eventStore: { + ...transaction.eventStore, + readOne: (args) => { + reads += 1; + return readOne.call(transaction.eventStore, args); + }, + }, + }, + ids: createSequentialIdGenerator(), + clock: createFixedClock(), + personId: "c-1", + intention: { + eventId: `e-${checked}`, + forgottenAt: "t", + reason: "requested", + fingerprints, + }, + context: testContext, + checked, + }); + }); + expect(await forget(false, ["f-other"])).toBe(STALE_INTENTION); + expect(reads).toBe(1); + expect(await forget(true, ["f-other"])).toMatchObject({ + hadKey: true, + event: { id: "e-true" }, + }); + expect(reads).toBe(1); + }); +}); diff --git a/packages/core/src/kernel/personal-data/forget.ts b/packages/core/src/kernel/personal-data/forget.ts index 53a4a8c0..71b936c9 100644 --- a/packages/core/src/kernel/personal-data/forget.ts +++ b/packages/core/src/kernel/personal-data/forget.ts @@ -14,6 +14,7 @@ import { import { scheduledCommandId } from "../command/pipeline.ts"; import { actedBy, startedBy } from "../shared/causation.ts"; import { appendSystemEvent } from "../system-events.ts"; +import type { ForgetIntention, ForgetIntentions } from "./intention.ts"; import { checkPersonId, transactRetrying } from "./transact.ts"; export interface ForgetWithinArgs { @@ -21,8 +22,13 @@ export interface ForgetWithinArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly personId: string; - readonly reason: PersonalDataForgottenReason; + readonly intention: ForgetIntention; readonly context: CausationContext; + /** + * The caller found already, in this transaction, the intention's event not held and the + * person's key named by it. + */ + readonly checked?: boolean; } export interface Forgotten { @@ -31,25 +37,59 @@ export interface Forgotten { readonly event: StoredEvent; } +/** + * The forgetting the intention names is in the store already, committed by a reconciliation + * meanwhile. + */ +export interface HeldForgetting { + readonly held: StoredEvent; +} + +/** + * The person holds a key the intention does not name: given to them after it was written. + */ +export const STALE_INTENTION: "stale" = "stale"; + +export type ForgetOutcome = Forgotten | HeldForgetting | typeof STALE_INTENTION; + export interface ForgetWithinFunction { - (args: ForgetWithinArgs): Promise; + (args: ForgetWithinArgs): Promise; } /** - * Removes the person's key and appends `PersonalDataForgotten` to their stream, on the - * transaction given: whoever reads the event finds the fields gone already. The scheduled - * commands and dead letters that hold the person's data go in the same transaction, and the event - * names them. A person without a key is forgotten all the same, since read models may hold what - * their events carried in the clear. + * Removes the person's key and appends `PersonalDataForgotten` to their stream, with the id, time + * and reason of the intention, on the transaction given: whoever reads the event finds the fields + * gone already. The scheduled commands and dead letters that hold the person's data go in the same + * transaction, and the event names them. A person without a key is forgotten all the same, since + * read models may hold what their events carried in the clear. */ export const forgetWithin: ForgetWithinFunction = async ({ transaction, ids, clock, personId, - reason, + intention, context, + checked = false, }) => { + const { fingerprints } = intention; + if (fingerprints !== undefined && !checked) { + const held = await transaction.eventStore.readOne({ + aggregateType: BOUNDA_AGGREGATE, + aggregateId: personId, + eventId: intention.eventId, + }); + if (held !== null) return { held }; + } + const key = await transaction.personalKeys.get(personId); + if ( + !checked && + key !== null && + fingerprints !== undefined && + !fingerprints.includes(key.fingerprint) + ) { + return STALE_INTENTION; + } // The key goes first: a snapshot written beside the transaction then either waits for it and // is forgotten with the rest, or finds no key and is never written. The scheduled commands go // before the dead letters: a worker giving one up writes its letter and settles its claim @@ -62,7 +102,8 @@ export const forgetWithin: ForgetWithinFunction = async ({ const letters = await transaction.deadLetterStore.forget(personId); const payload: PersonalDataForgottenPayload = { personId, - reason, + reason: intention.reason, + fingerprints: key === null ? [] : [key.fingerprint], cancelledCommands: cancelled.map(({ dedupeKey, command }) => ({ commandId: scheduledCommandId(dedupeKey) ?? dedupeKey, type: command.type, @@ -79,10 +120,53 @@ export const forgetWithin: ForgetWithinFunction = async ({ payload, context, person: personId, + identity: { id: intention.eventId, timestamp: intention.forgottenAt }, }); return { hadKey, payload, event }; }; +export interface ForgetIntendedArgs { + readonly storage: Pick; + readonly intentions: ForgetIntentions; + readonly ids: IdGenerator; + readonly clock: Clock; + readonly concurrencyRetries: number; + readonly personId: string; + readonly reason: PersonalDataForgottenReason; + readonly context: CausationContext; +} + +export interface ForgetIntendedFunction { + (args: ForgetIntendedArgs): Promise; +} + +/** + * Records the intention, then forgets in a transaction of its own. A key given to the person + * between the two takes a new intention that names it: each round needs a forgetting of the same + * person to commit in between, so the rounds end when those do. + */ +export const forgetIntended: ForgetIntendedFunction = async ({ + storage, + intentions, + ids, + clock, + concurrencyRetries, + personId, + reason, + context, +}) => { + for (;;) { + const intention = await intentions.intend({ personId, reason, keys: storage.personalKeys }); + const outcome = await transactRetrying({ + storage, + concurrencyRetries, + work: (transaction) => + forgetWithin({ transaction, ids, clock, personId, intention, context }), + }); + if (outcome !== STALE_INTENTION) return outcome; + } +}; + export interface LogForgottenFunction { (logger: Logger, forgotten: Forgotten): void; } @@ -114,6 +198,7 @@ export const forgottenDispatch: ForgottenDispatchFunction = (event) => ({ export interface ForgetPersonalDataArgs { readonly storage: Storage; + readonly intentions: ForgetIntentions; readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; @@ -127,10 +212,11 @@ export interface ForgetPersonalDataFunction { } /** - * `forget` of a client: `forgetWithin` in a transaction of its own, as a request of the person. + * `forget` of a client: `forgetIntended`, as a request of the person. */ export const forgetPersonalData: ForgetPersonalDataFunction = async ({ storage, + intentions, ids, clock, logger, @@ -140,19 +226,17 @@ export const forgetPersonalData: ForgetPersonalDataFunction = async ({ }) => { checkPersonId(personId, "forget"); const correlationId = ids.next(); - const forgotten = await transactRetrying({ + const forgotten = await forgetIntended({ storage, + intentions, + ids, + clock, concurrencyRetries, - work: (transaction) => - forgetWithin({ - transaction, - ids, - clock, - personId, - reason: "requested", - context: startedBy(correlationId, actor), - }), + personId, + reason: "requested", + context: startedBy(correlationId, actor), }); + if ("held" in forgotten) return forgottenDispatch(forgotten.held); logForgotten(logger, forgotten); return forgottenDispatch(forgotten.event); }; diff --git a/packages/core/src/kernel/personal-data/intention.test.ts b/packages/core/src/kernel/personal-data/intention.test.ts new file mode 100644 index 00000000..99e408b7 --- /dev/null +++ b/packages/core/src/kernel/personal-data/intention.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import { testPersonalKey } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock } from "../../contracts/clock.ts"; +import { ForgetLedgerUnavailableError } from "../../contracts/errors.ts"; +import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import { memory } from "../../memory/index.ts"; +import { createRecordingLogger } from "../test-support.ts"; +import { type BoundedForgetLedger, boundForgetLedger } from "./forget-ledger.ts"; +import { createForgetIntentions } from "./intention.ts"; + +const NOW = new Date("2026-03-01T00:00:00.000Z"); + +const setUp = async ({ + ledger, + strict = false, +}: { + readonly ledger: BoundedForgetLedger | null; + readonly strict?: boolean; +}) => { + const storage = await memory().createStorage(); + const { logger, entries: logs } = createRecordingLogger(); + const intentions = createForgetIntentions({ + ledger: ledger === null ? null : { store: ledger, strict }, + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + logger, + }); + return { storage, intentions, logs }; +}; + +const recordingLedger = (append: (entries: readonly ForgetLedgerEntry[]) => Promise) => + boundForgetLedger({ + ledger: { + append, + read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, + }, + timeoutMs: 1_000, + clock: createFixedClock(NOW), + }); + +describe("the intentions of forgetting", () => { + it("record each in the ledger before they resolve, naming the key the person holds now", async () => { + const appended: ForgetLedgerEntry[] = []; + const { storage, intentions } = await setUp({ + ledger: recordingLedger(async (entries) => { + appended.push(...entries); + }), + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + expect( + await intentions.intend({ personId: "c-1", reason: "retention", keys: storage.personalKeys }), + ).toEqual({ + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "retention", + fingerprints: ["f-1"], + }); + expect( + await intentions.intend({ personId: "c-2", reason: "requested", keys: storage.personalKeys }), + ).toMatchObject({ eventId: "id-2", fingerprints: [] }); + expect(appended).toEqual([ + { + personId: "c-1", + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "retention", + fingerprints: ["f-1"], + }, + { + personId: "c-2", + eventId: "id-2", + forgottenAt: NOW.toISOString(), + reason: "requested", + fingerprints: [], + }, + ]); + }); + + it("read no key and record nothing without a ledger, leaving any key to the forgetting", async () => { + const { storage, intentions } = await setUp({ ledger: null }); + let reads = 0; + const keys = { + get: async (person: string) => { + reads += 1; + return storage.personalKeys.get(person); + }, + }; + expect(await intentions.intend({ personId: "c-1", reason: "requested", keys })).toEqual({ + eventId: "id-1", + forgottenAt: NOW.toISOString(), + reason: "requested", + }); + await intentions.record({ + personId: "c-1", + eventId: "e-1", + forgottenAt: NOW.toISOString(), + reason: "requested", + }); + expect(reads).toBe(0); + }); + + it("refuse, when strict, a forgetting the ledger does not take, with why", async () => { + const down = new Error("ledger is down"); + const { storage, intentions } = await setUp({ + ledger: recordingLedger(async () => { + throw down; + }), + strict: true, + }); + const refused = intentions.intend({ + personId: "c-1", + reason: "requested", + keys: storage.personalKeys, + }); + await expect(refused).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + await expect(refused).rejects.toMatchObject({ cause: down }); + }); + + it("log a forgetting the ledger does not take, when not strict, and let it go on", async () => { + const { storage, intentions, logs } = await setUp({ + ledger: recordingLedger(async () => { + throw new Error("ledger is down"); + }), + }); + expect( + await intentions.intend({ personId: "c-1", reason: "requested", keys: storage.personalKeys }), + ).toMatchObject({ eventId: "id-1", fingerprints: [] }); + expect(logs).toEqual([ + { + level: "warn", + message: "forget ledger did not take the forgetting; it commits, and the stream copies it", + fields: expect.objectContaining({ errorName: "Error", errorMessage: "ledger is down" }), + }, + ]); + }); + + it("skip, when not strict, a ledger whose call given up for its timeout is still pending", async () => { + let late = true; + let appends = 0; + const ledger: BoundedForgetLedger = { + readMark: async () => 0, + raiseMark: async () => undefined, + append: async () => { + appends += 1; + }, + read: async () => [], + late: () => late, + }; + const loose = await setUp({ ledger }); + await loose.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: loose.storage.personalKeys, + }); + expect(appends).toBe(0); + const strict = await setUp({ ledger, strict: true }); + await strict.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: strict.storage.personalKeys, + }); + expect(appends).toBe(1); + late = false; + await loose.intentions.intend({ + personId: "c-1", + reason: "requested", + keys: loose.storage.personalKeys, + }); + expect(appends).toBe(2); + }); +}); diff --git a/packages/core/src/kernel/personal-data/intention.ts b/packages/core/src/kernel/personal-data/intention.ts new file mode 100644 index 00000000..e20ba631 --- /dev/null +++ b/packages/core/src/kernel/personal-data/intention.ts @@ -0,0 +1,105 @@ +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; +import type { PersonalKeyStore } from "../../adapter/storage/personal-key-store.ts"; +import type { Clock } from "../../contracts/clock.ts"; +import { ForgetLedgerUnavailableError } from "../../contracts/errors.ts"; +import type { IdGenerator } from "../../contracts/ids.ts"; +import type { Logger } from "../../contracts/logger.ts"; +import type { PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; +import type { BoundedForgetLedger } from "./forget-ledger.ts"; + +/** + * The forgetting about to commit, as its `PersonalDataForgotten` will carry it. + */ +export interface ForgetIntention { + readonly eventId: string; + readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + /** + * The keys the intention names, which the forgetting may destroy: one the person holds that is + * not here was given to them after the intention was written. `undefined` without a ledger, and + * for an entry that does not record them, which any key matches. + */ + readonly fingerprints?: readonly string[] | undefined; +} + +export interface IntendArgs { + readonly personId: string; + readonly reason: PersonalDataForgottenReason; + /** + * Where the key the person holds now is read, outside any transaction. + */ + readonly keys: Pick; +} + +export interface ForgetIntentions { + /** + * The intention of forgetting the person now, naming the key they hold, recorded in the ledger + * before it resolves. + */ + intend(args: IntendArgs): Promise; + /** + * Records an entry in the ledger. A strict ledger that does not take it rejects with + * `ForgetLedgerUnavailableError`; any other failure is logged, and the stream copies the + * forgetting once it commits. + */ + record(entry: ForgetLedgerEntry): Promise; +} + +export interface CreateForgetIntentionsArgs { + /** + * `null` without a ledger: nothing is recorded, and no key is read for it. + */ + readonly ledger: { + readonly store: BoundedForgetLedger; + readonly strict: boolean; + } | null; + readonly ids: IdGenerator; + readonly clock: Clock; + readonly logger: Logger; +} + +export interface CreateForgetIntentionsFunction { + (args: CreateForgetIntentionsArgs): ForgetIntentions; +} + +/** + * Every forgetting writes its intention first, so the ledger holds it whether or not the commit + * happens: an intention whose commit does not happen is committed by the next reconciliation, + * since forgetting is what was asked. While a call given up for its timeout is still pending, a + * ledger that is not strict is skipped, rather than each forgetting waiting on one that hangs. + */ +export const createForgetIntentions: CreateForgetIntentionsFunction = ({ + ledger, + ids, + clock, + logger, +}) => { + const record = async (entry: ForgetLedgerEntry): Promise => { + if (ledger === null) return; + const { store, strict } = ledger; + if (!strict && store.late()) return; + try { + await store.append([entry]); + } catch (error) { + if (strict) throw new ForgetLedgerUnavailableError(error); + logger.warn( + "forget ledger did not take the forgetting; it commits, and the stream copies it", + { + error, + }, + ); + } + }; + return { + intend: async ({ personId, reason, keys }) => { + const eventId = ids.next(); + const forgottenAt = clock.now().toISOString(); + if (ledger === null) return { eventId, forgottenAt, reason }; + const key = await keys.get(personId); + const fingerprints = key === null ? [] : [key.fingerprint]; + await record({ personId, eventId, forgottenAt, reason, fingerprints }); + return { eventId, forgottenAt, reason, fingerprints }; + }, + record, + }; +}; diff --git a/packages/core/src/kernel/personal-data/ledger-append.test.ts b/packages/core/src/kernel/personal-data/ledger-append.test.ts deleted file mode 100644 index 22ed466e..00000000 --- a/packages/core/src/kernel/personal-data/ledger-append.test.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Storage } from "../../adapter/adapter.ts"; -import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; -import { pendingEvent } from "../../adapter/testing/fixtures.ts"; -import { createFixedClock } from "../../contracts/clock.ts"; -import { type Logger, silentLogger } from "../../contracts/logger.ts"; -import { memory } from "../../memory/index.ts"; -import { FORGET_LEDGER_WAIT_MS } from "./forget-ledger.ts"; -import { withLedgerAppend } from "./ledger-append.ts"; - -const NOW = new Date("2026-01-01T00:01:00.000Z"); - -const forgetting = (personId: string, version = 1) => - pendingEvent({ - aggregateType: "bounda", - aggregateId: personId, - version, - type: "PersonalDataForgotten", - id: `f-${personId}-${version}`, - }); - -const setUp = async ({ - ledger, - keepForMs = null, - logger = silentLogger, -}: { - readonly ledger?: Partial; - readonly keepForMs?: number | null; - readonly logger?: Logger; -} = {}) => { - const appended: (readonly ForgetLedgerEntry[])[] = []; - const inner = await memory().createStorage(); - const storage: Storage = withLedgerAppend({ - storage: inner, - ledger: { - append: async (entries) => { - appended.push(entries); - }, - read: async () => [], - readMark: async () => 0, - raiseMark: async () => undefined, - ...ledger, - }, - keepForMs, - clock: createFixedClock(NOW), - logger, - }); - return { storage, appended }; -}; - -describe("the append to the forget ledger as a transaction commits", () => { - it("appends the forgettings a transaction committed, in one call, and nothing else", async () => { - const { storage, appended } = await setUp(); - const result = await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "order", - aggregateId: "o-1", - expectedVersion: 0, - events: [pendingEvent({ aggregateId: "o-1", version: 1 })], - }); - for (const person of ["c-1", "c-2"]) { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }); - } - return "done"; - }); - expect(result).toBe("done"); - expect(appended).toEqual([ - [ - { personId: "c-1", eventId: "f-c-1-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - { personId: "c-2", eventId: "f-c-2-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - ], - ]); - await storage.transact(async () => undefined); - expect(appended).toHaveLength(1); - await storage.transact((transaction) => - transaction.eventStore.reseal([{ position: 1, payload: { total: 9 }, person: "c-9" }]), - ); - expect( - await storage.eventStore.readByPerson({ person: "c-9", afterPosition: 0, limit: 1 }), - ).toMatchObject([{ payload: { total: 9 } }]); - expect(appended).toHaveLength(1); - }); - - it("appends nothing for a transaction that rolls back, nor for an earlier run of the work", async () => { - const { storage, appended } = await setUp(); - await expect( - storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }); - throw new Error("rolled back"); - }), - ).rejects.toThrow("rolled back"); - expect(appended).toEqual([]); - const inner = await memory().createStorage(); - let runs = 0; - const retrying = withLedgerAppend({ - storage: { - ...inner, - // A store that runs the work again on a fresh transaction, as a busy one may. - transact: async (work) => { - await inner.transact(work).catch(() => undefined); - return inner.transact(work); - }, - }, - ledger: { - append: async (entries) => { - appended.push(entries); - }, - read: async () => [], - readMark: async () => 0, - raiseMark: async () => undefined, - }, - keepForMs: null, - clock: createFixedClock(NOW), - logger: silentLogger, - }); - await retrying.transact(async (transaction) => { - runs += 1; - const person = `c-${runs}`; - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }); - if (runs === 1) throw new Error("busy"); - }); - expect(appended.map((entries) => entries.map(({ personId }) => personId))).toEqual([["c-2"]]); - }); - - it("leaves out a forgetting older than keepFor", async () => { - const { storage, appended } = await setUp({ keepForMs: 60_000 }); - await storage.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }), - ); - expect(appended).toEqual([]); - const { storage: fresh, appended: kept } = await setUp({ keepForMs: 60_001 }); - await fresh.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }), - ); - expect(kept).toHaveLength(1); - }); - - it("logs a ledger that refuses the forgetting and resolves what the transaction did", async () => { - const warnings: unknown[] = []; - const { storage } = await setUp({ - ledger: { - append: async () => { - throw new Error("ledger is down"); - }, - }, - logger: { ...silentLogger, warn: (message, fields) => warnings.push([message, fields]) }, - }); - const result = await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: "c-1", - expectedVersion: 0, - events: [forgetting("c-1")], - }); - return "committed"; - }); - expect(result).toBe("committed"); - expect(warnings).toEqual([ - [ - "forget ledger did not take the forgetting; the stream copies it", - expect.objectContaining({ - forgotten: 1, - error: expect.objectContaining({ message: "ledger is down" }), - }), - ], - ]); - }); - - it("leaves forgettings to the stream while an append that did not answer in time is pending", async () => { - const inner = await memory().createStorage(); - const clock = createFixedClock(NOW); - let answer: () => void = () => undefined; - const appends: string[][] = []; - const storage = withLedgerAppend({ - storage: inner, - ledger: { - append: (entries) => { - appends.push(entries.map(({ personId }) => personId)); - return appends.length === 1 - ? new Promise((resolve) => { - answer = resolve; - }) - : Promise.resolve(); - }, - read: async () => [], - readMark: async () => 0, - raiseMark: async () => undefined, - }, - keepForMs: null, - clock, - logger: silentLogger, - }); - const forget = (person: string) => - storage.transact((transaction) => - transaction.eventStore.append({ - aggregateType: "bounda", - aggregateId: person, - expectedVersion: 0, - events: [forgetting(person)], - }), - ); - let done = false; - const first = forget("c-1").then(() => { - done = true; - }); - while (!done) { - clock.advance(FORGET_LEDGER_WAIT_MS); - await new Promise((resolve) => setTimeout(resolve, 1)); - } - await first; - await forget("c-2"); - expect(appends).toEqual([["c-1"]]); - answer(); - await new Promise((resolve) => setTimeout(resolve, 1)); - await forget("c-3"); - expect(appends).toEqual([["c-1"], ["c-3"]]); - }); - - it("reads through the transaction's event store as it is", async () => { - const { storage } = await setUp(); - await storage.transact(async (transaction) => { - await transaction.eventStore.append({ - aggregateType: "order", - aggregateId: "o-1", - expectedVersion: 0, - events: [pendingEvent({ aggregateId: "o-1", version: 1 })], - }); - const loaded = await transaction.eventStore.load({ - aggregateType: "order", - aggregateId: "o-1", - }); - expect(loaded.version).toBe(1); - }); - const { eventStore } = await storage.transact(async (transaction) => transaction); - expect( - await eventStore.readOne({ - aggregateType: "order", - aggregateId: "o-1", - eventId: "order-o-1-1", - }), - ).toMatchObject({ id: "order-o-1-1" }); - expect(await eventStore.lastPosition()).toBe(1); - expect(await eventStore.readAll({ afterPosition: 0, limit: 10 })).toHaveLength(1); - expect(await eventStore.readByPerson({ afterPosition: 0, limit: 10, person: "c-1" })).toEqual( - [], - ); - }); -}); diff --git a/packages/core/src/kernel/personal-data/ledger-append.ts b/packages/core/src/kernel/personal-data/ledger-append.ts deleted file mode 100644 index acb182ca..00000000 --- a/packages/core/src/kernel/personal-data/ledger-append.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { Storage } from "../../adapter/adapter.ts"; -import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; -import type { Clock } from "../../contracts/clock.ts"; -import type { StoredEvent } from "../../contracts/event.ts"; -import type { Logger } from "../../contracts/logger.ts"; -import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; -import { FORGET_LEDGER_WAIT_MS, waitForLedger } from "./forget-ledger.ts"; - -export interface IsForgettingFunction { - (event: StoredEvent): boolean; -} - -export const isForgetting: IsForgettingFunction = (event) => - event.aggregateType === BOUNDA_AGGREGATE && event.type === PERSONAL_DATA_FORGOTTEN_EVENT; - -export interface LedgerEntriesArgs { - readonly events: readonly StoredEvent[]; - /** - * `null` keeps every entry. - */ - readonly keepForMs: number | null; - readonly now: Date; -} - -export interface LedgerEntriesFunction { - (args: LedgerEntriesArgs): readonly ForgetLedgerEntry[]; -} - -/** - * The entries of the forgettings among `events`, but those already older than `keepFor`, which a - * prune would remove anyway. - */ -export const ledgerEntries: LedgerEntriesFunction = ({ events, keepForMs, now }) => - events - .filter(isForgetting) - .filter( - (event) => keepForMs === null || Date.parse(event.timestamp) + keepForMs > now.getTime(), - ) - .map((event) => ({ - personId: event.aggregateId, - eventId: event.id, - forgottenAt: event.timestamp, - })); - -export interface WithLedgerAppendArgs { - readonly storage: Storage; - readonly ledger: ForgetLedger; - readonly keepForMs: number | null; - readonly clock: Clock; - readonly logger: Logger; -} - -export interface WithLedgerAppendFunction { - (args: WithLedgerAppendArgs): Storage; -} - -/** - * The storage whose transactions append the forgettings they commit to the ledger before they - * resolve, so a forgetting reaches the ledger unless the process stops between the commit and the - * append. The outbox copies every one again from the stream, which the ledger takes as an entry - * it holds, so a ledger that fails or does not answer in time is only logged, and while an append - * that did not answer in time is still pending, the next forgettings are left to the outbox - * rather than each waiting for a ledger that hangs. - */ -export const withLedgerAppend: WithLedgerAppendFunction = ({ - storage, - ledger, - keepForMs, - clock, - logger, -}) => { - let stalled = false; - return { - ...storage, - transact: async (work) => { - const appended: StoredEvent[] = []; - const result = await storage.transact((transaction) => { - // A store may run the work again on a fresh transaction: only the last run commits. - appended.length = 0; - const { eventStore } = transaction; - return work({ - ...transaction, - eventStore: { - append: async (args) => { - const written = await eventStore.append(args); - appended.push(...written.events); - return written; - }, - load: (args) => eventStore.load(args), - readOne: (args) => eventStore.readOne(args), - readAll: (args) => eventStore.readAll(args), - readByPerson: (args) => eventStore.readByPerson(args), - lastPosition: () => eventStore.lastPosition(), - reseal: (events) => eventStore.reseal(events), - }, - }); - }); - const entries = ledgerEntries({ events: appended, keepForMs, now: clock.now() }); - if (entries.length > 0 && !stalled) { - const appending = ledger.append(entries); - const outcome = await waitForLedger(clock, appending).catch((error: unknown) => ({ - error, - })); - if (outcome === "late") { - stalled = true; - const resume = (): void => { - stalled = false; - }; - appending.then(resume, resume); - logger.warn("forget ledger did not take the forgetting in time; the stream copies it", { - forgotten: entries.length, - timeoutMs: FORGET_LEDGER_WAIT_MS, - }); - } else if (outcome !== undefined) { - logger.warn("forget ledger did not take the forgetting; the stream copies it", { - forgotten: entries.length, - error: outcome.error, - }); - } - } - return result; - }, - }; -}; diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts index 34c417be..f53b46a3 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.test.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.test.ts @@ -1,18 +1,19 @@ import { trace } from "@opentelemetry/api"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import { pendingEvent } from "../../adapter/testing/fixtures.ts"; import { createFixedClock } from "../../contracts/clock.ts"; +import { ForgetLedgerTimeoutError } from "../../contracts/errors.ts"; import type { StoredEvent } from "../../contracts/event.ts"; import { memory } from "../../memory/index.ts"; import { + boundForgetLedger, defineForgetLedger, - FORGET_LEDGER_WAIT_MS, isForgetLedgerSource, openForgetLedger, waitForLedger, } from "./forget-ledger.ts"; -import { createLedgerOutbox, FORGET_LEDGER_SUBSCRIBER } from "./ledger-outbox.ts"; +import { createLedgerOutbox, FORGET_LEDGER_SUBSCRIBER, forgettingEntry } from "./ledger-outbox.ts"; const span = trace.getTracer("test").startSpan("test"); @@ -65,8 +66,18 @@ describe("the forget ledger's outbox", () => { expect(await outbox.process(events, span)).toBe(5); expect(appended).toEqual([ [ - { personId: "c-1", eventId: "f-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, - { personId: "c-2", eventId: "f-2", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "c-1", + eventId: "f-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, + { + personId: "c-2", + eventId: "f-2", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ], ]); expect(await outbox.process(events.slice(0, 3), span)).toBe(3); @@ -104,13 +115,63 @@ describe("the forget ledger's outbox", () => { ]; expect(await outbox.process(events, span)).toBe(3); expect(appended).toEqual([ - [{ personId: "c-3", eventId: "f-3", forgottenAt: "2026-01-01T00:01:00.001Z" }], + [ + { + personId: "c-3", + eventId: "f-3", + forgottenAt: "2026-01-01T00:01:00.001Z", + reason: "requested", + }, + ], ]); expect(await outbox.process(events.slice(0, 2), span)).toBe(2); expect(appended).toHaveLength(1); }); }); +describe("a forgetting's entry", () => { + const forgetting = (payload: unknown): StoredEvent => ({ + ...stored(4, { + aggregateType: "bounda", + aggregateId: "c-1", + version: 1, + type: "PersonalDataForgotten", + id: "f-1", + }), + payload, + }); + + it("carries the reason and the keys its event names, as its intention wrote them", () => { + expect( + forgettingEntry(forgetting({ personId: "c-1", reason: "retention", fingerprints: ["k-1"] })), + ).toEqual({ + personId: "c-1", + eventId: "f-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "retention", + fingerprints: ["k-1"], + }); + expect(forgettingEntry(forgetting({ reason: "requested", fingerprints: [] }))).toMatchObject({ + reason: "requested", + fingerprints: [], + }); + }); + + it("reads as a request naming no keys what an earlier event did not record", () => { + for (const payload of [ + {}, + null, + "text", + { reason: "restored", fingerprints: "k-1" }, + { reason: 3, fingerprints: ["k-1", 2] }, + ]) { + const entry = forgettingEntry(forgetting(payload)); + expect(entry.reason).toBe("requested"); + expect(entry).not.toHaveProperty("fingerprints"); + } + }); +}); + describe("a forget ledger's source", () => { it("is an adapter that opens ledgers or a ledger definition, nothing else", () => { const definition = defineForgetLedger(() => ({ @@ -154,13 +215,17 @@ describe("a forget ledger's source", () => { tenant: "t-1", }); await ledger?.append([ - { personId: "c-1", eventId: "e-1", forgottenAt: "2026-01-01T00:00:00.000Z" }, + { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested", + }, ]); expect( await (await adapter.createForgetLedger({ tenant: "t-1" })).read({ limit: 1 }), ).toHaveLength(1); expect(await (await adapter.createForgetLedger({})).read({ limit: 1 })).toEqual([]); - expect(await openForgetLedger({ source: null, storage: memory(), env })).toBeNull(); }); it("refuses an adapter that points at the storage's own database", async () => { @@ -195,13 +260,109 @@ describe("a forget ledger's source", () => { describe("the wait for the forget ledger", () => { it("resolves what the ledger answers in time and lets its timer go, or late once it passes", async () => { const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); - expect(await waitForLedger(clock, Promise.resolve(3))).toBe(3); + expect(await waitForLedger(clock, 500, Promise.resolve(3))).toBe(3); + expect(clock.pending()).toBe(0); + await expect(waitForLedger(clock, 500, Promise.reject(new Error("down")))).rejects.toThrow( + "down", + ); expect(clock.pending()).toBe(0); - await expect(waitForLedger(clock, Promise.reject(new Error("down")))).rejects.toThrow("down"); + let settled: unknown; + void waitForLedger(clock, 500, new Promise(() => undefined)).then((outcome) => { + settled = outcome; + }); + clock.advance(499); + await Promise.resolve(); + expect(settled).toBeUndefined(); + clock.advance(1); + await vi.waitFor(() => expect(settled).toBe("late")); + }); +}); + +describe("the forget ledger bound by its timeout", () => { + const entry = { + personId: "c-1", + eventId: "e-1", + forgottenAt: "2026-01-01T00:00:00.000Z", + reason: "requested" as const, + }; + + it("passes every call through, its delay and close included, and lets each timer go", async () => { + const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); + const calls: string[] = []; + const ledger = boundForgetLedger({ + ledger: { + visibilityDelayMs: 60_000, + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + calls.push(`append ${entries.length}`); + }, + read: async ({ limit }) => { + calls.push(`read ${limit}`); + return [entry]; + }, + prune: async ({ before }) => { + calls.push(`prune ${before}`); + }, + close: async () => { + calls.push("close"); + }, + }, + timeoutMs: 500, + clock, + }); + await ledger.append([entry]); + expect(await ledger.read({ limit: 3 })).toEqual([entry]); + await ledger.prune?.({ before: "t" }); + await ledger.close?.(); + expect(calls).toEqual(["append 1", "read 3", "prune t", "close"]); + expect(ledger.visibilityDelayMs).toBe(60_000); + expect(ledger.late()).toBe(false); expect(clock.pending()).toBe(0); - const waiting = waitForLedger(clock, new Promise(() => undefined)); - clock.advance(FORGET_LEDGER_WAIT_MS - 1); + const bare = boundForgetLedger({ + ledger: { + append: async () => {}, + read: async () => [], + readMark: async () => 0, + raiseMark: async () => undefined, + }, + timeoutMs: 500, + clock, + }); + expect(bare).not.toHaveProperty("visibilityDelayMs"); + expect(bare).not.toHaveProperty("prune"); + expect(bare).not.toHaveProperty("close"); + }); + + it("gives a call up at its timeout, and says it is late until that call settles", async () => { + const clock = createFixedClock(new Date("2026-01-01T00:00:00.000Z")); + let answer: () => void = () => undefined; + const ledger = boundForgetLedger({ + ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, + append: () => + new Promise((resolve) => { + answer = resolve; + }), + read: () => Promise.reject(new Error("down")), + prune: () => new Promise(() => undefined), + }, + timeoutMs: 500, + clock, + }); + const appending = ledger.append([entry]); + clock.advance(499); + expect(ledger.late()).toBe(false); clock.advance(1); - expect(await waiting).toBe("late"); + await expect(appending).rejects.toEqual(new ForgetLedgerTimeoutError("append", 500)); + expect(ledger.late()).toBe(true); + answer(); + await vi.waitFor(() => expect(ledger.late()).toBe(false)); + await expect(ledger.read({ limit: 1 })).rejects.toThrow("down"); + expect(ledger.late()).toBe(false); + const pruning = ledger.prune?.({ before: "t" }); + clock.advance(500); + await expect(pruning).rejects.toMatchObject({ operation: "prune", timeoutMs: 500 }); }); }); diff --git a/packages/core/src/kernel/personal-data/ledger-outbox.ts b/packages/core/src/kernel/personal-data/ledger-outbox.ts index 27a4e144..ee1784b3 100644 --- a/packages/core/src/kernel/personal-data/ledger-outbox.ts +++ b/packages/core/src/kernel/personal-data/ledger-outbox.ts @@ -1,10 +1,69 @@ -import type { ForgetLedger } from "../../adapter/storage/forget-ledger.ts"; +import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { Clock } from "../../contracts/clock.ts"; +import type { StoredEvent } from "../../contracts/event.ts"; +import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; import type { Subscriber } from "../dispatch/delivery.ts"; -import { ledgerEntries } from "./ledger-append.ts"; export const FORGET_LEDGER_SUBSCRIBER: "forget-ledger" = "forget-ledger"; +export interface IsForgettingFunction { + (event: StoredEvent): boolean; +} + +export const isForgetting: IsForgettingFunction = (event) => + event.aggregateType === BOUNDA_AGGREGATE && event.type === PERSONAL_DATA_FORGOTTEN_EVENT; + +export interface ForgettingEntryFunction { + (event: StoredEvent): ForgetLedgerEntry; +} + +/** + * The entry of a `PersonalDataForgotten`, as its intention wrote it. An event written before + * forgettings named their keys leaves `fingerprints` out, and one whose reason is no longer known + * reads as `requested`. + */ +export const forgettingEntry: ForgettingEntryFunction = (event) => { + const field = (name: string): unknown => + typeof event.payload === "object" && event.payload !== null + ? Reflect.get(event.payload, name) + : undefined; + const fingerprints = field("fingerprints"); + const entry: ForgetLedgerEntry = { + personId: event.aggregateId, + eventId: event.id, + forgottenAt: event.timestamp, + reason: field("reason") === "retention" ? "retention" : "requested", + }; + return Array.isArray(fingerprints) && fingerprints.every((value) => typeof value === "string") + ? { ...entry, fingerprints } + : entry; +}; + +export interface LedgerEntriesArgs { + readonly events: readonly StoredEvent[]; + /** + * `null` keeps every entry. + */ + readonly keepForMs: number | null; + readonly now: Date; +} + +export interface LedgerEntriesFunction { + (args: LedgerEntriesArgs): readonly ForgetLedgerEntry[]; +} + +/** + * The entries of the forgettings among `events`, but those already older than `keepFor`, which a + * prune would remove anyway. + */ +export const ledgerEntries: LedgerEntriesFunction = ({ events, keepForMs, now }) => + events + .filter(isForgetting) + .filter( + (event) => keepForMs === null || Date.parse(event.timestamp) + keepForMs > now.getTime(), + ) + .map(forgettingEntry); + export interface CreateLedgerOutboxArgs { readonly ledger: ForgetLedger; readonly keepForMs: number | null; @@ -17,10 +76,10 @@ export interface CreateLedgerOutboxFunction { /** * Follows the global stream from its first event and appends every `PersonalDataForgotten` to the - * ledger, a batch at a time: the event store is the outbox, behind the append each transaction - * makes as it commits. A batch the ledger refuses is delivered again, and so is every batch after - * a restored database rolls the checkpoint back, which the ledger takes as the entries it holds - * already. + * ledger, a batch at a time: the event store is the outbox, behind the intention each forgetting + * writes before it commits, for the forgettings whose intention a ledger that is not strict did + * not take. A batch the ledger refuses is delivered again, and so is every batch after a restored + * database rolls the checkpoint back, which the ledger takes as the entries it holds already. */ export const createLedgerOutbox: CreateLedgerOutboxFunction = ({ ledger, keepForMs, clock }) => ({ name: FORGET_LEDGER_SUBSCRIBER, diff --git a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts index a7936a96..a61dd29c 100644 --- a/packages/core/src/kernel/personal-data/reconciliation-app.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation-app.test.ts @@ -2,9 +2,14 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { Adapter } from "../../adapter/adapter.ts"; import type { ForgetLedger, ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { Table } from "../../adapter/storage/table.ts"; +import { DEFAULT_FORGET_LEDGER_TIMEOUT_MS } from "../../config/defaults.ts"; import type { ForgetLedgerConfig } from "../../config/types.ts"; import { createFixedClock } from "../../contracts/clock.ts"; -import { ForgetLedgerUnreconciledError, ValidationError } from "../../contracts/errors.ts"; +import { + ForgetLedgerUnavailableError, + ForgetLedgerUnreconciledError, + ValidationError, +} from "../../contracts/errors.ts"; import type { StoredEvent } from "../../contracts/event.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { type Logger, silentLogger } from "../../contracts/logger.ts"; @@ -18,16 +23,13 @@ import type { Registry } from "../../modules/registry.ts"; import type { FieldsArgs } from "../../modules/view.ts"; import { createTestApp } from "../../testing/index.ts"; import { createApp } from "../app.ts"; -import { - defineForgetLedger, - FORGET_LEDGER_WAIT_MS, - type ForgetLedgerSource, -} from "./forget-ledger.ts"; +import { defineForgetLedger, type ForgetLedgerSource } from "./forget-ledger.ts"; import { HOLD_MS } from "./hold.ts"; import { FORGET_LEDGER_SUBSCRIBER } from "./ledger-outbox.ts"; import { FORGET_LEDGER_ATTEMPTS, FORGET_LEDGER_CURSOR, + FORGET_LEDGER_GRACE_MS, FORGET_LEDGER_HELD_UNTIL, FORGET_LEDGER_REREAD_MS, } from "./reconciliation.ts"; @@ -185,8 +187,12 @@ const entry = (personId: string, eventId: string, at: Date): ForgetLedgerEntry = personId, eventId, forgottenAt: at.toISOString(), + reason: "requested", }); +// A forgetting the store lost, old enough at `START` for a reconciliation to commit it again. +const LOST = new Date(START.getTime() - FORGET_LEDGER_GRACE_MS - 60_000); + const opened = async (ledgerAdapter: Adapter): Promise => ledgerAdapter.createForgetLedger({}); @@ -208,18 +214,21 @@ describe("the forget ledger", () => { (event) => event.type === "PersonalDataForgotten", ); expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual( - events.map((event) => entry(event.aggregateId, event.id, new Date(event.timestamp))), + events.map((event) => ({ + ...entry(event.aggregateId, event.id, new Date(event.timestamp)), + fingerprints: [], + })), ); expect(events.map((event) => event.aggregateId)).toEqual(["c-0", "c-1"]); }); - it("forgets again at start, with reason restored, a person the ledger names whose forgetting the store does not hold", async () => { + it("commits again at start, under its own id, time and reason, a forgetting the ledger names that the store does not hold", async () => { const storage = memory(); const ledgerAdapter = memory(); const { app: before, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await register("c-2"); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); await before.stop(); const { logs, logger } = recording(); const { app, forgotten, customer } = await start({ @@ -227,13 +236,13 @@ describe("the forget ledger", () => { forgetLedger: ledgerAdapter, logger, }); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await customer("c-1")).toBeNull(); expect(await customer("c-2")).toEqual({ customerId: "c-2", name: "name of c-2" }); expect(logs).toContainEqual([ "info", "personal data forgotten", - expect.objectContaining({ personId: "c-1", reason: "restored", hadKey: true }), + expect.objectContaining({ personId: "c-1", reason: "requested", hadKey: true }), ]); const exported = await app.exportPersonalData("c-1"); expect(exported.events.map(({ type, data }) => [type, data])).toEqual([ @@ -243,17 +252,20 @@ describe("the forget ledger", () => { expect((await app.exportPersonalData("c-2")).events[0]?.data).toEqual({ name: "name of c-2" }); expect(logs).toContainEqual([ "warn", - "forgot again persons a restored backup brought back", + "committed again forgettings the database did not hold", { forgotten: 1 }, ]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); await app.runUntilIdle(); const restored = (await app.events.read({ limit: 100 })).at(-1); - expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toContainEqual({ - personId: "c-1", - eventId: restored?.id, - forgottenAt: restored?.timestamp, + expect(restored).toMatchObject({ + id: "lost-forget", + timestamp: LOST.toISOString(), + payload: { reason: "requested", fingerprints: [expect.any(String)] }, }); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual([ + entry("c-1", "lost-forget", LOST), + ]); }); it("forgets nobody on a normal start, nor a person given a key again after the forgetting the store holds", async () => { @@ -278,7 +290,7 @@ describe("the forget ledger", () => { // A start that forgot nobody leaves the read models to the background. expect(await customer("c-2")).toBeNull(); expect(logs.map(([, message]) => message)).not.toContain( - "forgot again persons a restored backup brought back", + "committed again forgettings the database did not hold", ); expect((await app.exportPersonalData("c-1")).events.at(-1)?.data).toEqual({ name: "name of c-1", @@ -311,8 +323,12 @@ describe("the forget ledger", () => { entry("c-1", "too-old", new Date(cursor - FORGET_LEDGER_REREAD_MS - 1)), entry("c-2", "in-window", new Date(cursor - FORGET_LEDGER_REREAD_MS + 1)), ]); - const { forgotten } = await start({ storage, forgetLedger: ledgerAdapter }); - expect(await forgotten()).toEqual([["c-2", "restored"]]); + const { forgotten } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: new Date(cursor), + }); + expect(await forgotten()).toEqual([["c-2", "requested"]]); expect(await written.checkpointStore.get(FORGET_LEDGER_CURSOR)).toBe(cursor); }); @@ -321,13 +337,13 @@ describe("the forget ledger", () => { const ledgerAdapter = memory(); const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); await first.stop(); const [one] = await Promise.all([ start({ storage, forgetLedger: ledgerAdapter }), start({ storage, forgetLedger: ledgerAdapter }), ]); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); }); it("starts when the ledger cannot be read, says so, and reconciles in the worker after a while", async () => { @@ -336,7 +352,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let failures = 2; const flaky = defineForgetLedger(async () => { const inner = await opened(ledgerAdapter); @@ -371,7 +387,7 @@ describe("the forget ledger", () => { expect(await app.nextDueAt()).toEqual(new Date(START.getTime() + 3 * HOLD_MS)); clock.advance(2 * HOLD_MS); expect(await app.runUntilIdle({ maxPasses: 1 })).toMatchObject({ idle: false }); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); expect(await app.nextDueAt()).toBeNull(); }); @@ -382,7 +398,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let down = true; let reads = 0; const flaky = defineForgetLedger(async () => { @@ -423,13 +439,13 @@ describe("the forget ledger", () => { down = false; one.clock.set(new Date(heldUntil)); expect(await one.app.runUntilIdle({ maxPasses: 1 })).toMatchObject({ idle: false }); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); expect(await stored()).toEqual([0, 0]); expect(await one.app.nextDueAt()).toBeNull(); other.clock.set(new Date(heldUntil)); await other.app.runUntilIdle(); expect(await other.app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); - expect(await other.forgotten()).toEqual([["c-1", "restored"]]); + expect(await other.forgotten()).toEqual([["c-1", "requested"]]); }); it("reconciles at every start, and backs off from the failures before a restart", async () => { @@ -462,7 +478,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); let answer: () => void = () => undefined; const answered = new Promise((resolve) => { answer = resolve; @@ -494,18 +510,81 @@ describe("the forget ledger", () => { }); // The start arms its timer once it reaches the ledger, which the test cannot see. while (started === undefined) { - clock.advance(FORGET_LEDGER_WAIT_MS); + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } const app = await starting; stops.push(() => app.stop()); + // Each call to the ledger is given up at the timeout, so the reconciliation fails then, and + // tries again once its hold ends. expect(logs).toContainEqual([ "error", - "forget ledger did not answer in time; the app starts and reconciles in the background", - { timeoutMs: FORGET_LEDGER_WAIT_MS }, + "forget ledger could not be reconciled; persons a restored backup brought back are not forgotten yet", + expect.objectContaining({ + errorName: "ForgetLedgerTimeoutError", + errorCode: "FORGET_LEDGER_TIMEOUT", + }), ]); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: false } }); answer(); + clock.advance(HOLD_MS); + await app.runUntilIdle(); + expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); + }); + + it("starts after the timeout when the reconciliation takes longer, each call to the ledger within it", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); + for (const person of ["c-1", "c-2", "c-3"]) await register(person); + await first.stop(); + await (await opened(ledgerAdapter)).append( + ["c-1", "c-2", "c-3"].map((person) => entry(person, `lost-${person}`, LOST)), + ); + const clock = createFixedClock(START); + let reconciling = false; + // Each forgetting the reconciliation commits again takes most of the timeout. + const slow: Adapter = { + ...storage, + createStorage: async () => { + const opened = await storage.createStorage(); + return { + ...opened, + transact: async (work) => { + if (reconciling) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS / 2); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return opened.transact(work); + }, + }; + }, + }; + const watching = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + readMark: async () => 0, + raiseMark: async () => undefined, + append: (entries) => inner.append(entries), + read: (args) => { + reconciling = true; + return inner.read(args); + }, + }; + }); + const { logs, logger } = recording(); + const app = await createApp({ + registry, + config: { storage: slow, runtime: { personalData: { forgetLedger: { store: watching } } } }, + clock, + logger, + }); + stops.push(() => app.stop()); + expect(logs).toContainEqual([ + "error", + "forget ledger did not answer in time; the app starts and reconciles in the background", + { timeoutMs: DEFAULT_FORGET_LEDGER_TIMEOUT_MS }, + ]); await vi.waitFor(async () => expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }), ); @@ -526,11 +605,11 @@ describe("the forget ledger", () => { await (await opened(ledgerAdapter)).append([entry("c-1", "shown-late", START)]); clock.advance(60_000); await app.runUntilIdle(); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect(await app.nextDueAt()).toBeNull(); }); - it("copies a forgetting from the stream once the ledger takes it, when it refused it at commit", async () => { + it("copies a forgetting from the stream once the ledger takes it, when it refused its intention", async () => { const storage = memory(); const ledgerAdapter = memory(); let refusals = 2; @@ -555,8 +634,8 @@ describe("the forget ledger", () => { expect(refusals).toBe(1); expect(logs).toContainEqual([ "warn", - "forget ledger did not take the forgetting; the stream copies it", - expect.objectContaining({ forgotten: 1, errorName: "Error" }), + "forget ledger did not take the forgetting; it commits, and the stream copies it", + expect.objectContaining({ errorName: "Error" }), ]); await app.runUntilIdle({ maxPasses: 1 }); expect(refusals).toBe(0); @@ -572,7 +651,7 @@ describe("the forget ledger", () => { ).toEqual(["c-1"]); }); - it("appends a forgetting to the ledger before forget resolves, and the stream copies it once", async () => { + it("records a forgetting in the ledger before it commits, and the stream copies it once", async () => { const storage = memory(); const ledgerAdapter = memory(); const { app, register } = await start({ storage, forgetLedger: ledgerAdapter }); @@ -581,7 +660,9 @@ describe("the forget ledger", () => { const [forgetting] = (await app.events.read({ limit: 100 })).filter( (event) => event.type === "PersonalDataForgotten", ); - const appended = [entry("c-1", forgetting?.id ?? "", START)]; + const appended = [ + { ...entry("c-1", forgetting?.id ?? "", START), fingerprints: [expect.any(String)] }, + ]; expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual(appended); await app.runUntilIdle(); expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toEqual(appended); @@ -592,14 +673,22 @@ describe("the forget ledger", () => { const ledgerAdapter = memory(); const { app, register, forgotten } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); + await register("c-2"); await app.forgetAgain([{ personId: "c-1" }]); - expect(await forgotten()).toEqual([["c-1", "restored"]]); + expect(await forgotten()).toEqual([["c-1", "requested"]]); expect( (await (await opened(ledgerAdapter)).read({ limit: 10 })).map(({ personId }) => personId), ).toEqual(["c-1"]); + const elsewhere = entry("c-2", "from-elsewhere", LOST); + await app.forgetAgain([elsewhere]); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toContainEqual(elsewhere); + expect(await forgotten()).toEqual([ + ["c-1", "requested"], + ["c-2", "requested"], + ]); }); - it("appends what the retention forgets as it commits, before the stream copies it", async () => { + it("records what the retention forgets before it commits, and the stream copies it", async () => { const ledgerAdapter = memory(); const appends: string[] = []; const watching = defineForgetLedger(async () => { @@ -637,11 +726,11 @@ describe("the forget ledger", () => { (event) => event.type === "PersonalDataForgotten", ), ).toHaveLength(1); - // Once as the forgetting committed, once more from the stream. + // Once as the forgetting's intention, once more from the stream. expect(appends).toEqual(["c-1", "c-1"]); }); - it("resolves forget once the ledger has not taken the forgetting in time, and says so", async () => { + it("commits a forgetting once the ledger has not taken its intention in time, says so, and waits for it no more while it hangs", async () => { const storage = memory(); const ledgerAdapter = memory(); const hanging = defineForgetLedger(async () => { @@ -659,17 +748,25 @@ describe("the forget ledger", () => { const forgetting = app.forget("c-1").then(() => { done = true; }); - // The append arms its timer once the forgetting commits, which the test cannot see. + // The append arms its timer once it is called, which the test cannot see. while (!done) { - clock.advance(FORGET_LEDGER_WAIT_MS); + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } await forgetting; expect(logs).toContainEqual([ "warn", - "forget ledger did not take the forgetting in time; the stream copies it", - { forgotten: 1, timeoutMs: FORGET_LEDGER_WAIT_MS }, + "forget ledger did not take the forgetting; it commits, and the stream copies it", + expect.objectContaining({ + errorName: "ForgetLedgerTimeoutError", + errorCode: "FORGET_LEDGER_TIMEOUT", + }), ]); + await app.forget("c-2"); + const forgotten = (await app.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ); + expect(forgotten.map((event) => event.aggregateId)).toEqual(["c-1", "c-2"]); }); it("reconciles again every reconcileEvery while the app runs, one instance a period", async () => { @@ -694,7 +791,7 @@ describe("the forget ledger", () => { await one.register("c-1"); expect(reads).toBe(2); // As if the database had been restored, while the app runs, to before c-1 was forgotten. - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); expect(await one.app.nextDueAt()).toBeNull(); one.clock.advance(59_999); other.clock.advance(59_999); @@ -708,7 +805,7 @@ describe("the forget ledger", () => { // ledger at once too. await other.app.runUntilIdle(); expect(reads).toBe(4); - expect(await one.forgotten()).toEqual([["c-1", "restored"]]); + expect(await one.forgotten()).toEqual([["c-1", "requested"]]); expect(await one.customer("c-1")).toBeNull(); other.clock.advance(60_000); await other.app.runUntilIdle(); @@ -820,7 +917,7 @@ describe("the forget ledger", () => { const { app: first, register } = await start({ storage, forgetLedger: ledgerAdapter }); await register("c-1"); await first.stop(); - await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", START)]); + await (await opened(ledgerAdapter)).append([entry("c-1", "lost-forget", LOST)]); const down = new Error("ledger is down"); let closed = 0; const broken = defineForgetLedger(async () => ({ @@ -858,26 +955,44 @@ describe("the forget ledger", () => { }; }); const clock = createFixedClock(START); - let started = false; + let settled = false; const starting = createApp({ registry, config: { storage, - runtime: { personalData: { forgetLedger: { store: slow, strict: true } } }, + runtime: { + personalData: { forgetLedger: { store: slow, strict: true, timeout: "1m" } }, + }, }, clock, - }).then((app) => { - started = true; - stops.push(() => app.stop()); - return app; + }).finally(() => { + settled = true; }); - for (let round = 0; round < 3; round += 1) { - clock.advance(FORGET_LEDGER_WAIT_MS); + void starting.catch(() => undefined); + // Waits past the default timeout, then gives the read up at its own. + for (let round = 0; round < 5; round += 1) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); await new Promise((resolve) => setTimeout(resolve, 5)); } - expect(started).toBe(false); + expect(settled).toBe(false); + while (!settled) { + clock.advance(DEFAULT_FORGET_LEDGER_TIMEOUT_MS); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await expect(starting).rejects.toMatchObject({ + code: "FORGET_LEDGER_UNRECONCILED", + cause: { code: "FORGET_LEDGER_TIMEOUT", operation: "read", timeoutMs: 60_000 }, + }); answer(); - const app = await starting; + const app = await createApp({ + registry, + config: { + storage, + runtime: { personalData: { forgetLedger: { store: slow, strict: true } } }, + }, + clock, + }); + stops.push(() => app.stop()); expect(await app.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); const events = await app.events.read({ limit: 100 }); expect(events.filter((event) => event.type === "PersonalDataForgotten")).toHaveLength(1); @@ -932,6 +1047,301 @@ describe("the forget ledger", () => { }); }); +// The memory adapter whose storage transactions fail while `failing` says so, as a process that +// stops between the intention and the commit leaves the store. +const crashing = (storage: Adapter) => { + const state = { failing: false }; + const adapter: Adapter = { + ...storage, + createStorage: async () => { + const opened = await storage.createStorage(); + return { + ...opened, + transact: async (work) => { + if (state.failing) throw new Error("the process stopped"); + return opened.transact(work); + }, + }; + }, + }; + return { adapter, state }; +}; + +describe("the intention a forgetting records first", () => { + it("is committed by the next start when the process stops before the commit, under its id, time and reason", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app, register } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + await register("c-1"); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + const [intention] = await (await opened(ledgerAdapter)).read({ limit: 10 }); + expect(intention).toEqual({ + personId: "c-1", + eventId: expect.any(String), + forgottenAt: START.toISOString(), + reason: "requested", + fingerprints: [expect.any(String)], + }); + expect(await app.exportPersonalData("c-1")).toMatchObject({ + events: [{ data: { name: "name of c-1" } }], + }); + await app.stop(); + state.failing = false; + const later = new Date(START.getTime() + FORGET_LEDGER_GRACE_MS); + const { app: next, customer } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: later, + }); + const forgetting = (await next.events.read({ limit: 100 })).at(-1); + expect(forgetting).toMatchObject({ + id: intention?.eventId, + timestamp: START.toISOString(), + payload: { personId: "c-1", reason: "requested", fingerprints: intention?.fingerprints }, + }); + expect(await customer("c-1")).toBeNull(); + }); + + it("waits for its grace before a reconciliation commits it, and is due once that ends", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app, register } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + await register("c-1"); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + await app.stop(); + state.failing = false; + const { + app: next, + clock, + forgotten, + } = await start({ + storage, + forgetLedger: ledgerAdapter, + }); + expect(await forgotten()).toEqual([]); + expect(await next.nextDueAt()).toEqual(new Date(START.getTime() + FORGET_LEDGER_GRACE_MS)); + clock.advance(FORGET_LEDGER_GRACE_MS); + await next.runUntilIdle(); + expect(await forgotten()).toEqual([["c-1", "requested"]]); + expect(await next.nextDueAt()).toBeNull(); + }); + + it("leaves a key given to the person after it, when its commit never happened", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const { adapter, state } = crashing(storage); + const { app } = await start({ storage: adapter, forgetLedger: ledgerAdapter }); + state.failing = true; + await expect(app.forget("c-1")).rejects.toThrow("the process stopped"); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toMatchObject([ + { personId: "c-1", fingerprints: [] }, + ]); + state.failing = false; + await app.stop(); + const later = new Date(START.getTime() + FORGET_LEDGER_GRACE_MS); + const { register } = await start({ storage, forgetLedger: ledgerAdapter, now: later }); + await register("c-1"); + const { forgotten, customer } = await start({ + storage, + forgetLedger: ledgerAdapter, + now: later, + }); + expect(await forgotten()).toEqual([]); + expect(await customer("c-1")).toEqual({ customerId: "c-1", name: "name of c-1" }); + }); + + it("forgets nobody, when strict, while the ledger does not take it, so calling again is safe", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + let down = true; + const flaky = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + if (down) throw new Error("ledger is down"); + await inner.append(entries); + }, + }; + }); + const { app, register, forgotten, customer } = await start({ + storage, + forgetLedger: flaky, + options: { strict: true }, + }); + await register("c-1"); + const refused = app.forget("c-1"); + await expect(refused).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + await expect(refused).rejects.toMatchObject({ + code: "FORGET_LEDGER_UNAVAILABLE", + cause: new Error("ledger is down"), + }); + expect(await forgotten()).toEqual([]); + expect(await customer("c-1")).toEqual({ customerId: "c-1", name: "name of c-1" }); + expect(await app.forgetAgain([{ personId: "c-1" }]).catch((error: unknown) => error)).toEqual( + expect.any(ForgetLedgerUnavailableError), + ); + expect(await forgotten()).toEqual([]); + down = false; + await app.forget("c-1"); + expect(await forgotten()).toEqual([["c-1", "requested"]]); + expect(await (await opened(ledgerAdapter)).read({ limit: 10 })).toHaveLength(1); + }); + + it("forgets nobody, when strict, once the ledger has not taken it within its timeout", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const hanging = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, + append: () => new Promise(() => undefined), + }; + }); + const { app, clock, register, forgotten } = await start({ + storage, + forgetLedger: hanging, + options: { strict: true, timeout: "2s" }, + }); + await register("c-1"); + let outcome: unknown; + const forgetting = app.forget("c-1").then( + () => "forgotten", + (error: unknown) => error, + ); + void forgetting.then((settled) => { + outcome = settled; + }); + while (outcome === undefined) { + clock.advance(1_000); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(outcome).toBeInstanceOf(ForgetLedgerUnavailableError); + expect(outcome).toMatchObject({ + cause: { code: "FORGET_LEDGER_TIMEOUT", operation: "append", timeoutMs: 2_000 }, + }); + expect(await forgotten()).toEqual([]); + }); + + it("keeps a person past the retention, when strict, while the ledger does not take it", async () => { + let down = true; + const ledgerAdapter = memory(); + const flaky = defineForgetLedger(async () => { + const inner = await opened(ledgerAdapter); + return { + read: (args) => inner.read(args), + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + if (down) throw new Error("ledger is down"); + await inner.append(entries); + }, + }; + }); + const clock = createFixedClock(START); + const { logs, logger } = recording(); + const app = await createApp({ + registry, + config: { + storage: memory(), + runtime: { + personalData: { retention: "1h", forgetLedger: { store: flaky, strict: true } }, + }, + }, + clock, + logger, + }); + stops.push(() => app.stop()); + const commands = app.commands as unknown as Record< + string, + (payload: { customerId: string; name: string }) => Promise + >; + await commands.registerCustomer?.({ customerId: "c-1", name: "Ada" }); + await app.runUntilIdle(); + clock.advance(3_600_000); + await app.runUntilIdle(); + const forgettings = async () => + (await app.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ); + expect(await forgettings()).toEqual([]); + expect(logs).toContainEqual([ + "error", + "person past the retention could not be forgotten", + expect.objectContaining({ personId: "c-1", errorCode: "FORGET_LEDGER_UNAVAILABLE" }), + ]); + down = false; + clock.advance(HOLD_MS); + await app.runUntilIdle(); + expect((await forgettings()).map((event) => event.payload)).toMatchObject([ + { personId: "c-1", reason: "retention" }, + ]); + }); + + it("commits again a forgetting the retention made only while the retention would still forget the person", async () => { + const storage = memory(); + const ledgerAdapter = memory(); + const config = (retention?: "1h" | "2h") => ({ + storage, + runtime: { + personalData: { + ...(retention === undefined ? {} : { retention }), + forgetLedger: { store: ledgerAdapter }, + }, + }, + }); + const clock = createFixedClock(START); + const first = await createApp({ registry, config: config("1h"), clock }); + const commands = first.commands as unknown as Record< + string, + (payload: { customerId: string; name: string }) => Promise + >; + await commands.registerCustomer?.({ customerId: "c-1", name: "Ada" }); + await commands.registerCustomer?.({ customerId: "c-2", name: "Bob" }); + await first.runUntilIdle(); + await first.stop(); + const retained = (person: string): ForgetLedgerEntry => ({ + ...entry(person, `retained-${person}`, LOST), + reason: "retention", + }); + await (await opened(ledgerAdapter)).append([retained("c-1"), retained("c-2")]); + clock.advance(90 * 60_000); + const later = await createApp({ registry, config: config("2h"), clock }); + stops.push(() => later.stop()); + expect( + (await later.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ), + ).toEqual([]); + await later.stop(); + const without = await createApp({ registry, config: config(), clock }); + expect( + (await without.events.read({ limit: 100 })).filter( + (event) => event.type === "PersonalDataForgotten", + ), + ).toEqual([]); + await without.stop(); + const due = await createApp({ registry, config: config("1h"), clock }); + stops.push(() => due.stop()); + expect( + (await due.events.read({ limit: 100 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => [event.id, (event.payload as PersonalDataForgottenPayload).reason]), + ).toEqual([ + ["retained-c-1", "retention"], + ["retained-c-2", "retention"], + ]); + }); +}); + describe("forgetAgain", () => { it("forgets again whom an entry names unless the store holds its forgetting or they hold no key", async () => { const storage = memory(); @@ -962,8 +1372,8 @@ describe("forgetAgain", () => { expect(logs.filter(([, message]) => message === "personal data forgotten")).toHaveLength(3); expect(await forgotten()).toEqual([ ["c-4", "requested"], - ["c-1", "restored"], - ["c-2", "restored"], + ["c-1", "requested"], + ["c-2", "requested"], ]); }); @@ -983,7 +1393,7 @@ describe("forgetAgain", () => { expect(logs).toContainEqual([ "info", "personal data forgotten", - expect.objectContaining({ personId: "c-1", reason: "restored", actor: "operator-1" }), + expect.objectContaining({ personId: "c-1", reason: "requested", actor: "operator-1" }), ]); }); diff --git a/packages/core/src/kernel/personal-data/reconciliation.test.ts b/packages/core/src/kernel/personal-data/reconciliation.test.ts index 2152b225..9b46f630 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.test.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.test.ts @@ -7,20 +7,25 @@ import type { ReadForgetLedgerArgs, } from "../../adapter/storage/forget-ledger.ts"; import { createFixedClock } from "../../contracts/clock.ts"; -import { ValidationError } from "../../contracts/errors.ts"; +import { ForgetLedgerUnavailableError, ValidationError } from "../../contracts/errors.ts"; import { createSequentialIdGenerator } from "../../contracts/ids.ts"; import { type LogFields, type Logger, silentLogger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; +import { withoutLedger } from "../test-support.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; import { HOLD_MS } from "./hold.ts"; +import { createForgetIntentions } from "./intention.ts"; import { createForgetLedgerReconciliation, FORGET_LEDGER_ATTEMPTS, FORGET_LEDGER_CURSOR, + FORGET_LEDGER_GRACE_MS, FORGET_LEDGER_HELD_UNTIL, FORGET_LEDGER_LAST_RUN, FORGET_LEDGER_REREAD_MS, type ForgetAgainEntry, forgetAgain, + NEVER_DUE, } from "./reconciliation.ts"; // Each error line as its message, marked when it carries no error. @@ -81,6 +86,8 @@ const setUp = async ({ await keyed(storage, persons); const { ledger, reads } = recordingLedger(entries, visibilityDelayMs); const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ ids: createSequentialIdGenerator(), clock: createFixedClock(NOW) }), + retentionDue: NEVER_DUE, ledger, storage, ids: createSequentialIdGenerator(), @@ -98,9 +105,12 @@ const entry = (personId: string, eventId: string, forgottenAt: string): ForgetLe personId, eventId, forgottenAt, + reason: "requested", }); -const at = (minute: number): string => new Date(NOW.getTime() + minute * 60_000).toISOString(); +// Minutes into the hour before `NOW`: every entry is past the grace a reconciliation gives. +const at = (minute: number): string => + new Date(NOW.getTime() - 60 * 60_000 + minute * 60_000).toISOString(); describe("the forget ledger's reconciliation", () => { it("reads every page of the ledger, and stops at the first short one", async () => { @@ -139,7 +149,7 @@ describe("the forget ledger's reconciliation", () => { }); await storage.checkpointStore.set( FORGET_LEDGER_CURSOR, - NOW.getTime() + FORGET_LEDGER_REREAD_MS, + Date.parse(at(0)) + FORGET_LEDGER_REREAD_MS, ); expect(await reconciliation.reconcile(NOW)).toBe(1); }); @@ -200,6 +210,11 @@ describe("the forget ledger's reconciliation", () => { const inner = await memory().createForgetLedger({}); await inner.append([entry("c-1", "e-1", at(0))]); const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { readMark: async () => 0, raiseMark: async () => undefined, @@ -246,6 +261,11 @@ describe("the forget ledger's reconciliation", () => { const errors: unknown[] = []; let reads = 0; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { readMark: async () => 0, raiseMark: async () => undefined, @@ -296,6 +316,11 @@ describe("the forget ledger's reconciliation", () => { const errors: unknown[] = []; const create = () => createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([]).ledger, storage: { ...storage, checkpointStore: checkpoints }, ids: createSequentialIdGenerator(), @@ -322,6 +347,11 @@ describe("the forget ledger's reconciliation", () => { const storage = await memory().createStorage(); const asked: string[] = []; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([], 1).ledger, storage: { ...storage, @@ -355,6 +385,11 @@ describe("the forget ledger's reconciliation", () => { await storage.checkpointStore.set(FORGET_LEDGER_HELD_UNTIL, NOW.getTime() + 60_000); const told: string[] = []; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: recordingLedger([entry("c-1", "e-1", at(0))]).ledger, storage, ids: createSequentialIdGenerator(), @@ -401,6 +436,11 @@ describe("the forget ledger's periodic reconciliation", () => { const { ledger, reads } = recordingLedger([]); let left = failures; const reconciliation = createForgetLedgerReconciliation({ + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, ledger: { ...ledger, read: async (args) => { @@ -574,15 +614,142 @@ describe("the forget ledger's periodic reconciliation", () => { }); }); +describe("what the reconciliation commits again", () => { + const named = (fingerprints: readonly string[] | undefined, minute = 0): ForgetLedgerEntry => ({ + ...entry("c-1", `e-${String(fingerprints)}`, at(minute)), + ...(fingerprints === undefined ? {} : { fingerprints }), + }); + + it("commits a forgetting while the person holds a key it names, or any key when it names none recorded", async () => { + for (const [fingerprints, forgotten] of [ + [["f-c-1"], 1], + [["f-other", "f-c-1"], 1], + [undefined, 1], + [["f-other"], 0], + [[], 0], + ] as const) { + const { reconciliation, storage } = await setUp({ + entries: [named(fingerprints)], + persons: ["c-1"], + }); + expect(await reconciliation.reconcile(NOW)).toBe(forgotten); + expect((await storage.personalKeys.get("c-1")) === null).toBe(forgotten === 1); + } + }); + + it("commits it under its own id, time and reason, naming the key it destroyed", async () => { + const retained = named(["f-c-1"], 7); + const { reconciliation, storage } = await setUp({ entries: [retained], persons: ["c-1"] }); + await reconciliation.reconcile(NOW); + expect(await storage.eventStore.readAll({ afterPosition: 0, limit: 10 })).toMatchObject([ + { + id: retained.eventId, + timestamp: retained.forgottenAt, + aggregateType: "bounda", + aggregateId: "c-1", + person: "c-1", + payload: { personId: "c-1", reason: "requested", fingerprints: ["f-c-1"] }, + }, + ]); + }); + + it("commits a forgetting the retention made only while the retention would forget the person now", async () => { + const due: string[] = []; + for (const [answer, forgotten] of [ + [false, 0], + [true, 1], + ] as const) { + const storage = await memory().createStorage(); + await keyed(storage, ["c-1"]); + const clock = createFixedClock(NOW); + const reconciliation = createForgetLedgerReconciliation({ + ledger: recordingLedger([{ ...named(["f-c-1"]), reason: "retention" }]).ledger, + intentions: withoutLedger({ ids: createSequentialIdGenerator(), clock }), + retentionDue: async (_, personId, now) => { + due.push(`${personId} ${now.toISOString()}`); + return answer; + }, + storage, + ids: createSequentialIdGenerator(), + clock, + logger: silentLogger, + concurrencyRetries: 3, + reconcileEveryMs: EVERY, + keepForMs: null, + }); + expect(await reconciliation.reconcile(NOW)).toBe(forgotten); + } + // Once before the transaction, and once more in it for the one it forgot. + expect(due).toEqual([ + `c-1 ${NOW.toISOString()}`, + `c-1 ${NOW.toISOString()}`, + `c-1 ${NOW.toISOString()}`, + ]); + }); + + it("leaves an entry younger than its grace for a reconciliation due once the youngest comes of age", async () => { + const young = new Date(NOW.getTime() - FORGET_LEDGER_GRACE_MS + 1).toISOString(); + const younger = new Date(NOW.getTime() - FORGET_LEDGER_GRACE_MS + 2).toISOString(); + const { reconciliation, storage } = await setUp({ + entries: [ + entry("c-1", "e-1", at(0)), + entry("c-2", "e-2", young), + entry("c-3", "e-3", younger), + ], + persons: ["c-1", "c-2", "c-3"], + }); + expect(await reconciliation.reconcile(NOW)).toBe(1); + expect(await storage.personalKeys.get("c-2")).not.toBeNull(); + expect(reconciliation.nextDueAt()).toEqual( + new Date(Date.parse(younger) + FORGET_LEDGER_GRACE_MS), + ); + expect(await storage.checkpointStore.get(FORGET_LEDGER_CURSOR)).toBe(Date.parse(younger)); + const due = new Date(Date.parse(younger) + FORGET_LEDGER_GRACE_MS); + expect(await reconciliation.sweep(due)).toBe(2); + expect(reconciliation.nextDueAt()).toBeNull(); + }); + + it("commits a young entry at once after a restore, and gives the grace again after that", async () => { + const young = new Date(NOW.getTime() - 1).toISOString(); + const entries: ForgetLedgerEntry[] = [entry("c-1", "e-1", young)]; + const { reconciliation, storage } = await setUp({ entries, persons: ["c-1", "c-2"] }); + await reconciliation.restored(false); + expect(await reconciliation.reconcile(NOW)).toBe(1); + entries.push(entry("c-2", "e-2", young)); + expect(await reconciliation.reconcile(NOW)).toBe(0); + expect(await storage.personalKeys.get("c-2")).not.toBeNull(); + }); + + it("is due at the sooner of a ledger's delay and an entry's grace", async () => { + const young = new Date(NOW.getTime() - 1).toISOString(); + const late = await setUp({ entries: [entry("c-1", "e-1", young)], visibilityDelayMs: 1_000 }); + await late.reconciliation.reconcile(NOW); + expect(late.reconciliation.nextDueAt()).toEqual(new Date(NOW.getTime() + 1_000)); + const slow = await setUp({ + entries: [entry("c-1", "e-1", young)], + visibilityDelayMs: 10 * FORGET_LEDGER_GRACE_MS, + }); + await slow.reconciliation.reconcile(NOW); + expect(slow.reconciliation.nextDueAt()).toEqual( + new Date(Date.parse(young) + FORGET_LEDGER_GRACE_MS), + ); + }); +}); + describe("forgetAgain", () => { it("refuses an entry without a person's id", async () => { const storage = await memory().createStorage(); + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); const refused = forgetAgain({ storage, - ids: createSequentialIdGenerator(), - clock: createFixedClock(NOW), + intentions: withoutLedger({ ids, clock }), + ids, + clock, concurrencyRetries: 3, entry: { personId: "" }, + retentionDue: NEVER_DUE, + record: true, }); await expect(refused).rejects.toBeInstanceOf(ValidationError); await expect(refused).rejects.toThrow("forgetting again needs the id of the person"); @@ -598,13 +765,18 @@ describe("forgetAgain", () => { return storage.transact(work); }, }; + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); const run = (entry: ForgetAgainEntry) => forgetAgain({ storage: counted, - ids: createSequentialIdGenerator(), - clock: createFixedClock(NOW), + intentions: withoutLedger({ ids, clock }), + ids, + clock, concurrencyRetries: 3, entry, + retentionDue: NEVER_DUE, + record: true, }); expect(await run({ personId: "c-1", eventId: "e-1" })).toBeNull(); expect(transactions).toBe(0); @@ -612,6 +784,209 @@ describe("forgetAgain", () => { expect(await run({ personId: "c-1", eventId: "e-1" })).not.toBeNull(); expect(transactions).toBe(1); }); + + const ledgered = async () => { + const base = await memory().createStorage(); + let transactions = 0; + const beforeTransaction: (() => Promise)[] = []; + const storage: Storage = { + ...base, + transact: async (work) => { + transactions += 1; + await beforeTransaction.shift()?.(); + return base.transact(work); + }, + }; + const recorded: ForgetLedgerEntry[] = []; + const ids = createSequentialIdGenerator(); + const clock = createFixedClock(NOW); + let down = false; + const intentions = createForgetIntentions({ + ledger: { + store: boundForgetLedger({ + ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + if (down) throw new Error("ledger is down"); + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, + ids, + clock, + logger: silentLogger, + }); + const run = (entry: ForgetAgainEntry) => + forgetAgain({ + storage, + intentions, + ids, + clock, + concurrencyRetries: 3, + entry, + retentionDue: NEVER_DUE, + record: true, + }); + return { + storage, + recorded, + run, + intentions, + ids, + clock, + transactions: () => transactions, + beforeTransaction: (step: () => Promise) => beforeTransaction.push(step), + fail: () => { + down = true; + }, + }; + }; + + it("records a whole entry from elsewhere in the ledger as it is, then commits it under its id", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1", "c-2"]); + const whole: ForgetLedgerEntry = { ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] }; + const unrecorded = entry("c-2", "e-2", at(4)); + expect(await run(whole)).toMatchObject({ event: { id: "e-1", timestamp: at(3) } }); + expect(await run(unrecorded)).toMatchObject({ event: { id: "e-2" } }); + expect(recorded).toEqual([whole, unrecorded]); + expect(recorded[1]).not.toHaveProperty("fingerprints"); + }); + + it("takes an intention of its own for a bare id, naming the key the person holds, and none for one who holds none", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1"]); + expect(await run({ personId: "c-2" })).toBeNull(); + expect(await run({ personId: "c-1", eventId: "elsewhere" })).toMatchObject({ + event: { id: "id-2", payload: { reason: "requested", fingerprints: ["f-c-1"] } }, + }); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: "id-2", + forgottenAt: NOW.toISOString(), + reason: "requested", + fingerprints: ["f-c-1"], + }, + ]); + }); + + it("forgets nobody, when strict, while the ledger does not take the entry", async () => { + const { storage, run, fail } = await ledgered(); + await keyed(storage, ["c-1"]); + fail(); + await expect(run({ ...entry("c-1", "e-1", at(3)) })).rejects.toBeInstanceOf( + ForgetLedgerUnavailableError, + ); + await expect(run({ personId: "c-1" })).rejects.toBeInstanceOf(ForgetLedgerUnavailableError); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + expect(await storage.eventStore.lastPosition()).toBe(0); + }); + + it("leaves a key the entry does not name, and a person a store holds the forgetting of", async () => { + const { storage, run } = await ledgered(); + await keyed(storage, ["c-1"]); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-other"] })).toBeNull(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).not.toBeNull(); + await keyed(storage, ["c-1"]); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).toBeNull(); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); + + it("leaves a key the entry does not name without a transaction", async () => { + const { storage, run, transactions } = await ledgered(); + await keyed(storage, ["c-1"]); + const keying = transactions(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-other"] })).toBeNull(); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: [] })).toBeNull(); + expect(transactions()).toBe(keying); + }); + + it("leaves an entry naming no keys whose forgetting the store holds, whatever key the person holds", async () => { + const { storage, run } = await ledgered(); + await keyed(storage, ["c-1"]); + const legacy = entry("c-1", "e-1", at(3)); + expect(await run(legacy)).not.toBeNull(); + await keyed(storage, ["c-1"]); + expect(await run(legacy)).toBeNull(); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); + + it("takes an intention of its own for an entry missing its id, its time or its reason", async () => { + const { storage, recorded, run } = await ledgered(); + await keyed(storage, ["c-1", "c-2", "c-3"]); + const { forgottenAt, eventId } = entry("c-1", "e-1", at(3)); + await run({ personId: "c-1", eventId, reason: "requested" }); + await run({ personId: "c-2", forgottenAt, reason: "requested" }); + await run({ personId: "c-3", eventId: "e-3", forgottenAt }); + expect( + recorded.map(({ personId, eventId: id, forgottenAt: at, reason }) => [ + personId, + id, + at, + reason, + ]), + ).toEqual([ + ["c-1", "id-2", NOW.toISOString(), "requested"], + ["c-2", "id-4", NOW.toISOString(), "requested"], + ["c-3", "id-6", NOW.toISOString(), "requested"], + ]); + }); + + it("takes a new intention for a bare id when the person is given a key between the intention and the commit", async () => { + const { storage, recorded, run, beforeTransaction } = await ledgered(); + await keyed(storage, ["c-1"]); + beforeTransaction(async () => { + await storage.personalKeys.remove("c-1"); + await storage.personalKeys.add("c-1", { key: "k-2", fingerprint: "f-2", kid: null }); + }); + expect(await run({ personId: "c-1" })).toMatchObject({ + event: { id: "id-3", payload: { fingerprints: ["f-2"] } }, + }); + expect(recorded.map(({ eventId, fingerprints }) => [eventId, fingerprints])).toEqual([ + ["id-2", ["f-c-1"]], + ["id-3", ["f-2"]], + ]); + }); + + it("records nothing as it reconciles: the entries are the ledger's own", async () => { + const { storage, recorded, intentions, ids, clock } = await ledgered(); + await keyed(storage, ["c-1"]); + const reconciliation = createForgetLedgerReconciliation({ + ledger: recordingLedger([{ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] }]).ledger, + intentions, + retentionDue: NEVER_DUE, + storage, + ids, + clock, + logger: silentLogger, + concurrencyRetries: 3, + reconcileEveryMs: EVERY, + keepForMs: null, + }); + expect(await reconciliation.reconcile(NOW)).toBe(1); + expect(recorded).toEqual([]); + }); + + it("never commits again a forgetting the retention made without a retention", async () => { + expect(await NEVER_DUE(await memory().createStorage(), "c-1", NOW)).toBe(false); + }); + + it("forgets nobody whose key went between its check and its transaction", async () => { + const { storage, run, beforeTransaction } = await ledgered(); + await keyed(storage, ["c-1"]); + beforeTransaction(async () => { + await storage.personalKeys.remove("c-1"); + }); + expect(await run({ ...entry("c-1", "e-1", at(3)), fingerprints: ["f-c-1"] })).toBeNull(); + expect(await storage.eventStore.lastPosition()).toBe(0); + }); }); describe("the reconciliation's status and restores", () => { @@ -671,6 +1046,11 @@ describe("the reconciliation's status and restores", () => { const logged: unknown[][] = []; const reconciliation = createForgetLedgerReconciliation({ ledger: recordingLedger([]).ledger, + intentions: withoutLedger({ + ids: createSequentialIdGenerator(), + clock: createFixedClock(NOW), + }), + retentionDue: NEVER_DUE, storage: { ...storage, checkpointStore: { diff --git a/packages/core/src/kernel/personal-data/reconciliation.ts b/packages/core/src/kernel/personal-data/reconciliation.ts index 7b511554..6440189b 100644 --- a/packages/core/src/kernel/personal-data/reconciliation.ts +++ b/packages/core/src/kernel/personal-data/reconciliation.ts @@ -1,4 +1,4 @@ -import type { Storage } from "../../adapter/adapter.ts"; +import type { Storage, StorageTransaction } from "../../adapter/adapter.ts"; import type { ForgetLedger, ForgetLedgerEntry, @@ -8,10 +8,11 @@ import { type Hold, isReleasedBy } from "../../adapter/storage/hold.ts"; import type { Clock } from "../../contracts/clock.ts"; import type { IdGenerator } from "../../contracts/ids.ts"; import type { Logger } from "../../contracts/logger.ts"; -import { BOUNDA_AGGREGATE } from "../../modules/bounda-events.ts"; +import { BOUNDA_AGGREGATE, type PersonalDataForgottenReason } from "../../modules/bounda-events.ts"; import { startedBy } from "../shared/causation.ts"; -import { type Forgotten, forgetWithin, logForgotten } from "./forget.ts"; +import { type Forgotten, forgetWithin, logForgotten, STALE_INTENTION } from "./forget.ts"; import { holdAfter } from "./hold.ts"; +import type { ForgetIntention, ForgetIntentions } from "./intention.ts"; import { checkPersonId, transactRetrying } from "./transact.ts"; /** @@ -45,21 +46,61 @@ export const FORGET_LEDGER_LAST_RUN = "forget-ledger:last-run"; // KV does, lists again for every page. export const FORGET_LEDGER_PAGE = 1_000; +/** + * How old a forget ledger's entry must be before a reconciliation commits it again: a younger one + * may be the intention of a forgetting still committing. A reconciliation that leaves one is due + * again once it is that old. + */ +export const FORGET_LEDGER_GRACE_MS: number = 60_000; + +/** + * A forgetting to commit again: a ledger's entry, or a line of `bounda forgotten`, or a bare + * person's id. + */ export interface ForgetAgainEntry { readonly personId: string; /** * The `PersonalDataForgotten` the person was forgotten by: a store that holds it has applied the - * forgetting already, and a key the person holds now was given to them after. + * forgetting already. With `forgottenAt` and `reason`, it is committed again under the same id, + * time and reason. */ readonly eventId?: string; + readonly forgottenAt?: string; + readonly reason?: PersonalDataForgottenReason; + /** + * The keys the forgetting named: a key the person holds that is not here was given to them + * after, and is kept. Any key counts when absent. + */ + readonly fingerprints?: readonly string[]; } +/** + * Whether the retention would forget the person now, read on the stores given: a forgetting the + * retention made is committed again only then. + */ +export interface RetentionDueFunction { + ( + stores: Pick, + personId: string, + now: Date, + ): Promise; +} + +export const NEVER_DUE: RetentionDueFunction = async () => false; + export interface ForgetAgainArgs { - readonly storage: Pick; + readonly storage: Pick; + readonly intentions: ForgetIntentions; readonly ids: IdGenerator; readonly clock: Clock; readonly concurrencyRetries: number; readonly entry: ForgetAgainEntry; + readonly retentionDue: RetentionDueFunction; + /** + * Whether the entry is written to the ledger first, as it is when it comes from elsewhere than + * the ledger. + */ + readonly record: boolean; readonly actor?: string | undefined; } @@ -67,23 +108,31 @@ export interface ForgetAgainFunction { (args: ForgetAgainArgs): Promise; } +type DueStores = Pick; + /** - * Forgets the person again, with `reason: "restored"`, when the store does not hold the event the - * entry names and the person still holds a key, deciding both again in the transaction that - * forgets them, so of two instances doing it at once, the one whose append conflicts finds the key - * gone on its retry; an entry that reads as applied takes no transaction. Resolves `null` when it - * forgot nobody. `storage` is the store as written: nothing here reads a personal field. + * Commits the forgetting again when the store does not hold the event the entry names, the person + * still holds a key it names, and, for one the retention made, the retention would forget them + * now; deciding all of it again in the transaction that forgets them, so of two instances doing + * it at once, the one whose append conflicts finds it done on its retry. An entry that reads as + * applied takes no transaction. A whole entry is committed under its own id, time and reason; any + * other takes an intention of its own, as a request. Resolves `null` when it forgot nobody. + * `storage` is the store as written: nothing here reads a personal field. */ export const forgetAgain: ForgetAgainFunction = async ({ storage, + intentions, ids, clock, concurrencyRetries, - entry: { personId, eventId }, + entry, + retentionDue, + record, actor, }) => { + const { personId, eventId, forgottenAt, reason, fingerprints } = entry; checkPersonId(personId, "forgetting again"); - const isDue = async (stores: Pick): Promise => { + const isDue = async (stores: DueStores, named: readonly string[] | undefined) => { if (eventId !== undefined) { const applied = await stores.eventStore.readOne({ aggregateType: BOUNDA_AGGREGATE, @@ -92,25 +141,45 @@ export const forgetAgain: ForgetAgainFunction = async ({ }); if (applied !== null) return false; } - return (await stores.personalKeys.get(personId)) !== null; + const key = await stores.personalKeys.get(personId); + if (key === null || (named !== undefined && !named.includes(key.fingerprint))) return false; + return reason !== "retention" || retentionDue(stores, personId, clock.now()); }; - if (!(await isDue(storage))) return null; + if (!(await isDue(storage, fingerprints))) return null; + const whole = eventId !== undefined && forgottenAt !== undefined && reason !== undefined; + if (whole && record) { + const recorded = { personId, eventId, forgottenAt, reason }; + await intentions.record(fingerprints === undefined ? recorded : { ...recorded, fingerprints }); + } const correlationId = ids.next(); - return transactRetrying({ - storage, - concurrencyRetries, - work: async (transaction) => { - if (!(await isDue(transaction))) return null; - return forgetWithin({ - transaction, - ids, - clock, - personId, - reason: "restored", - context: startedBy(correlationId, actor), - }); - }, - }); + // A whole entry is due only while the person holds a key it names, so it is never stale; an + // intention of its own takes another that names a key given meanwhile, as `forget` does. + for (;;) { + const intention: ForgetIntention = whole + ? { eventId, forgottenAt, reason, fingerprints } + : await intentions.intend({ + personId, + reason: reason ?? "requested", + keys: storage.personalKeys, + }); + const outcome = await transactRetrying({ + storage, + concurrencyRetries, + work: async (transaction) => + (await isDue(transaction, fingerprints)) + ? forgetWithin({ + transaction, + ids, + clock, + personId, + intention, + context: startedBy(correlationId, actor), + checked: whole, + }) + : null, + }); + if (outcome !== STALE_INTENTION) return outcome === null || "held" in outcome ? null : outcome; + } }; export interface ReconciliationFailed { @@ -139,9 +208,10 @@ export interface ForgetLedgerReconciliationStatus { export interface ForgetLedgerReconciliation { /** - * Reads the ledger past the database's cursor and forgets again whom `forgetAgain` finds; then - * moves the cursor. Resolves to how many persons it forgot, or to why it could not read the - * ledger through; never rejects: a failure is logged and the worker tries again later. + * Reads the ledger past the database's cursor and commits again what `forgetAgain` finds due, + * leaving entries younger than `FORGET_LEDGER_GRACE_MS` to a reconciliation due once they are + * not; then moves the cursor. Resolves to how many persons it forgot, or to why it could not read + * the ledger through; never rejects: a failure is logged and the worker tries again later. */ reconcile(now: Date): Promise; /** @@ -152,8 +222,9 @@ export interface ForgetLedgerReconciliation { */ sweep(now: Date): Promise; /** - * When the next reconciliation after a failure, or for a ledger's delay, is due; `null` when - * none is. The periodic one is left out: a host that only wakes for it would never sleep. + * When the next reconciliation after a failure, for a ledger's delay or for an entry left for + * being young, is due; `null` when none is. The periodic one is left out: a host that only wakes + * for it would never sleep. */ nextDueAt(): Date | null; /** @@ -164,18 +235,20 @@ export interface ForgetLedgerReconciliation { status(): Promise; /** * After the database went back to a copy: due at once, since the copy may hold persons - * forgotten after it was taken; when `first`, clears the hold the copy brought back. Never - * rejects. + * forgotten after it was taken, and without the grace for young entries, which the copy cannot + * be committing; when `first`, clears the hold the copy brought back. Never rejects. */ restored(first: boolean): Promise; } export interface CreateForgetLedgerReconciliationArgs { readonly ledger: ForgetLedger; + readonly intentions: ForgetIntentions; + readonly retentionDue: RetentionDueFunction; /** * The store as written. */ - readonly storage: Pick; + readonly storage: ForgetAgainArgs["storage"] & Pick; readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; @@ -211,6 +284,8 @@ export interface CreateForgetLedgerReconciliationFunction { */ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationFunction = ({ ledger, + intentions, + retentionDue, storage, ids, clock, @@ -223,6 +298,8 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF }) => { const { checkpointStore } = storage; let done = false; + // A restore was found since the last reconciliation that read the ledger through. + let restoredSince = false; let dueAt: number | null = null; // When this instance next looks whether the periodic reconciliation is due. let periodicAt: number = clock.now().getTime() + reconcileEveryMs; @@ -237,30 +314,53 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF } }; - const run = async (): Promise => { + // Resolves how many it forgot, and when the youngest entry it left comes of age; the cursor + // moves past that entry, which the next read takes again, since the grace is shorter than the + // stretch read again. + const run = async (at: number): Promise<{ forgotten: number; young: number | null }> => { const cursor = await checkpointStore.get(FORGET_LEDGER_CURSOR); let after: ForgetLedgerPosition | undefined = cursor === 0 ? undefined : { forgottenAt: new Date(cursor - FORGET_LEDGER_REREAD_MS).toISOString(), eventId: "" }; let forgotten = 0; + let youngest: number | null = null; + const ended = () => ({ + forgotten, + young: youngest === null ? null : youngest + FORGET_LEDGER_GRACE_MS, + }); for (;;) { const page: readonly ForgetLedgerEntry[] = await ledger.read({ ...(after === undefined ? {} : { after }), limit: pageSize, }); for (const entry of page) { - const again = await forgetAgain({ storage, ids, clock, concurrencyRetries, entry }); + const time = Date.parse(entry.forgottenAt); + if (!restoredSince && time + FORGET_LEDGER_GRACE_MS > at) { + // Entries come in the order of their time: the last one left is the youngest. + youngest = time; + continue; + } + const again = await forgetAgain({ + storage, + intentions, + ids, + clock, + concurrencyRetries, + entry, + retentionDue, + record: false, + }); if (again === null) continue; logForgotten(logger, again); onForgotten?.(entry.personId); forgotten += 1; } const last = page.at(-1); - if (last === undefined) return forgotten; + if (last === undefined) return ended(); const time = Date.parse(last.forgottenAt); if (Number.isFinite(time)) await advance(FORGET_LEDGER_CURSOR, time); - if (page.length < pageSize) return forgotten; + if (page.length < pageSize) return ended(); after = last; } }; @@ -302,8 +402,9 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF const reconcile = async (now: Date): Promise => { const at = now.getTime(); let forgotten: number; + let young: number | null; try { - forgotten = await run(); + ({ forgotten, young } = await run(at)); } catch (error) { done = false; logger.error( @@ -315,10 +416,13 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF ); return { failed: error }; } - if (forgotten > 0) - logger.warn("forgot again persons a restored backup brought back", { forgotten }); + if (forgotten > 0) { + logger.warn("committed again forgettings the database did not hold", { forgotten }); + } done = true; - dueAt = settled ? null : at + (ledger.visibilityDelayMs ?? 0); + restoredSince = false; + const delayed = settled ? null : at + (ledger.visibilityDelayMs ?? 0); + dueAt = delayed === null || young === null ? (delayed ?? young) : Math.min(delayed, young); settled = true; periodicAt = at + reconcileEveryMs; await release().catch((error: unknown) => @@ -412,6 +516,7 @@ export const createForgetLedgerReconciliation: CreateForgetLedgerReconciliationF }, restored: async (first) => { done = false; + restoredSince = true; dueAt = clock.now().getTime(); // A hold the copy brought back is not waited for: what it held was failing before the copy. local = null; diff --git a/packages/core/src/kernel/personal-data/retention-app.test.ts b/packages/core/src/kernel/personal-data/retention-app.test.ts index 168555bf..8059135f 100644 --- a/packages/core/src/kernel/personal-data/retention-app.test.ts +++ b/packages/core/src/kernel/personal-data/retention-app.test.ts @@ -282,7 +282,13 @@ describe("forget in a reaction", () => { expect(await forgotten()).toEqual([]); await advance(2); expect(await forgotten()).toEqual([ - { personId: "c-1", reason: "requested", cancelledCommands: [], removedDeadLetters: [] }, + { + personId: "c-1", + reason: "requested", + fingerprints: [expect.any(String)], + cancelledCommands: [], + removedDeadLetters: [], + }, ]); expect(await storage.personalKeys.get("c-1")).toBeNull(); expect(await storage.personalKeys.get("c-2")).not.toBeNull(); diff --git a/packages/core/src/kernel/personal-data/retention.test.ts b/packages/core/src/kernel/personal-data/retention.test.ts index 13eb0bdd..03914ac9 100644 --- a/packages/core/src/kernel/personal-data/retention.test.ts +++ b/packages/core/src/kernel/personal-data/retention.test.ts @@ -1,16 +1,21 @@ import { describe, expect, it } from "vitest"; import type { Storage } from "../../adapter/adapter.ts"; +import type { ForgetLedgerEntry } from "../../adapter/storage/forget-ledger.ts"; import type { StoredPerson } from "../../adapter/storage/person-store.ts"; -import { createFixedClock } from "../../contracts/clock.ts"; +import { testPersonalKey } from "../../adapter/testing/fixtures.ts"; +import { createFixedClock, type FixedClock } from "../../contracts/clock.ts"; import type { StoredEvent } from "../../contracts/event.ts"; -import { createSequentialIdGenerator } from "../../contracts/ids.ts"; +import { createSequentialIdGenerator, type IdGenerator } from "../../contracts/ids.ts"; +import { silentLogger } from "../../contracts/logger.ts"; import { memory } from "../../memory/index.ts"; import { createKeptPersonStore } from "../../memory/person-store.ts"; import { LEASE_MS } from "../shared/lease.ts"; import { tracer } from "../telemetry.ts"; -import { createRecordingLogger } from "../test-support.ts"; +import { createRecordingLogger, withoutLedger } from "../test-support.ts"; import { forgetPersonalData } from "./forget.ts"; +import { boundForgetLedger } from "./forget-ledger.ts"; import { HOLD_MAX_MS, HOLD_MS } from "./hold.ts"; +import { createForgetIntentions } from "./intention.ts"; import { restrictProcessing } from "./restriction.ts"; import { createRetentionSubscriber, @@ -18,6 +23,7 @@ import { RETENTION_LEASE, RETENTION_SUBSCRIBER, RETENTION_SWEEP_LIMIT, + retentionDueOf, } from "./retention.ts"; const DAY = 86_400_000; @@ -126,6 +132,7 @@ const setUp = async (storage?: Storage) => { const sweepOver = (storage: Storage) => createRetentionSweep({ storage, + intentions: withoutLedger({ ids, clock }), retentionMs: 30 * DAY, ids, clock, @@ -251,6 +258,7 @@ describe("createRetentionSweep", () => { await touch("c-1"); await forgetPersonalData({ storage, + intentions: withoutLedger({ ids, clock }), ids, clock, logger, @@ -283,6 +291,7 @@ describe("createRetentionSweep", () => { await setUp(); const other = createRetentionSweep({ storage, + intentions: withoutLedger({ ids, clock }), retentionMs: 30 * DAY, ids, clock, @@ -685,6 +694,7 @@ describe("createRetentionSweep", () => { let lists = 0; const { touch, record, advance, clock, sweepOver } = await setUp(live); const told: string[] = []; + const toldIds = createSequentialIdGenerator(); const sweep = createRetentionSweep({ storage: { ...live, @@ -696,8 +706,9 @@ describe("createRetentionSweep", () => { }, }, }, + intentions: withoutLedger({ ids: toldIds, clock }), retentionMs: 30 * DAY, - ids: createSequentialIdGenerator(), + ids: toldIds, clock, logger: createRecordingLogger().logger, concurrencyRetries: 1, @@ -728,10 +739,12 @@ describe("createRetentionSweep", () => { const { storage, record, touch, clock, sweepOver } = await setUp(); await touch("c-1"); await record(); + const longerIds = createSequentialIdGenerator(); const longer = createRetentionSweep({ storage, + intentions: withoutLedger({ ids: longerIds, clock }), retentionMs: 60 * DAY, - ids: createSequentialIdGenerator(), + ids: longerIds, clock, logger: createRecordingLogger().logger, concurrencyRetries: 0, @@ -766,6 +779,215 @@ describe("createRetentionSweep", () => { }); }); +describe("the retention and the forget ledger", () => { + const recordingIntentions = (ids: IdGenerator, clock: FixedClock) => { + const recorded: ForgetLedgerEntry[] = []; + const intentions = createForgetIntentions({ + ledger: { + store: boundForgetLedger({ + ledger: { + readMark: async () => 0, + raiseMark: async () => undefined, + append: async (entries) => { + recorded.push(...entries); + }, + read: async () => [], + }, + timeoutMs: 1_000, + clock, + }), + strict: true, + }, + ids, + clock, + logger: silentLogger, + }); + return { intentions, recorded }; + }; + + it("records the intention of each forgetting before it commits, and none for a person it skips", async () => { + const { record, touch, advance, clock, storage, ids, logger } = await setUp(); + const { intentions, recorded } = recordingIntentions(ids, clock); + const sweep = createRetentionSweep({ + storage, + intentions, + retentionMs: 30 * DAY, + ids, + clock, + logger, + concurrencyRetries: 1, + batchSize: 2, + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + await touch("c-1"); + await touch("c-2"); + await record(); + await storage.persons.restrict("c-2", new Date(day(0))); + advance(31); + expect(await sweep.sweep(clock.now())).toBe(1); + const [forgetting] = ( + await storage.eventStore.readAll({ afterPosition: 0, limit: 100 }) + ).filter((event) => event.type === "PersonalDataForgotten"); + expect(recorded).toEqual([ + { + personId: "c-1", + eventId: forgetting?.id, + forgottenAt: forgetting?.timestamp, + reason: "retention", + fingerprints: ["f-1"], + }, + ]); + }); + + it("says a person is due as a sweep would forget them now, by their row and what it has not recorded", async () => { + const { record, touch, advance, clock, storage } = await setUp(); + const due = retentionDueOf({ + retentionMs: 30 * DAY, + batchSize: 2, + checkpoints: storage.checkpointStore, + }); + const isDue = (person: string) => due(storage, person, clock.now()); + for (const person of ["c-1", "c-2", "c-3", "c-4"]) await touch(person); + await record(); + expect(await isDue("c-1")).toBe(false); + advance(30); + expect(await isDue("c-1")).toBe(true); + expect(await isDue("c-9")).toBe(false); + await storage.persons.restrict("c-2", new Date(day(30))); + expect(await isDue("c-2")).toBe(false); + await touch("c-3"); + expect(await isDue("c-3")).toBe(false); + await storage.persons.record({ person: "c-4", position: 99, forgottenAt: day(30) }); + expect(await isDue("c-4")).toBe(false); + }); + + // A sweep through a store that runs `meanwhile` once, between the list and the person's forget, + // or before the forget's transaction. + const racing = async ( + when: "listed" | "transaction", + meanwhile: (storage: Storage) => Promise, + ) => { + const setup = await setUp(); + const { storage, ids, clock, logger } = setup; + let pending: ((storage: Storage) => Promise) | undefined = meanwhile; + const once = async () => { + const step = pending; + pending = undefined; + await step?.(storage); + }; + const raced: Storage = { + ...storage, + persons: { + ...storage.persons, + listIdle: async (args) => { + const listed = await storage.persons.listIdle(args); + if (when === "listed" && listed.length > 0) await once(); + return listed; + }, + }, + transact: async (work) => { + if (when === "transaction") await once(); + return storage.transact(work); + }, + }; + const { intentions, recorded } = recordingIntentions(ids, clock); + const sweep = createRetentionSweep({ + storage: raced, + intentions, + retentionMs: 30 * DAY, + ids, + clock, + logger, + concurrencyRetries: 1, + batchSize: 2, + }); + await storage.personalKeys.add("c-1", testPersonalKey("1")); + await setup.touch("c-1"); + await setup.record(); + setup.advance(31); + return { ...setup, sweep, recorded }; + }; + + it("records no intention for a person restricted between the list and their forget", async () => { + const { sweep, clock, recorded, forgottenFor } = await racing("listed", (storage) => + storage.persons.restrict("c-1", new Date(day(31))).then(() => undefined), + ); + expect(await sweep.sweep(clock.now())).toBe(0); + expect(recorded).toEqual([]); + expect(await forgottenFor()).toEqual([]); + }); + + it("forgets nobody whose transaction finds them restricted after the intention, nor given a key since, and holds them", async () => { + const restricted = await racing("transaction", (storage) => + storage.persons.restrict("c-1", new Date(day(31))).then(() => undefined), + ); + expect(await restricted.sweep.sweep(restricted.clock.now())).toBe(0); + expect(restricted.recorded).toHaveLength(1); + expect(await restricted.forgottenFor()).toEqual([]); + + const rekeyed = await racing("transaction", async (storage) => { + await storage.personalKeys.remove("c-1"); + await storage.personalKeys.add("c-1", testPersonalKey("2")); + }); + expect(await rekeyed.sweep.sweep(rekeyed.clock.now())).toBe(0); + expect(rekeyed.entries.filter(({ level }) => level === "error")).toEqual([]); + expect(await rekeyed.forgottenFor()).toEqual([]); + expect(await rekeyed.storage.personalKeys.get("c-1")).toEqual(testPersonalKey("2")); + const [row] = await rekeyed.storage.persons.listIdle({ limit: 1 }); + expect(row?.hold).not.toBeNull(); + expect(await rekeyed.sweep.sweep(rekeyed.clock.now())).toBe(0); + }); + + it("leaves a person a reconciliation forgot under the intention's id meanwhile", async () => { + let intended = ""; + const { sweep, clock, recorded, forgottenFor, storage } = await racing( + "transaction", + async (store) => { + intended = recorded[0]?.eventId ?? ""; + await store.eventStore.append({ + aggregateType: "bounda", + aggregateId: "c-1", + expectedVersion: 0, + events: [ + { + ...forgotten(99, "c-1"), + aggregateId: "c-1", + id: intended, + payload: { personId: "c-1", reason: "retention" }, + }, + ], + }); + }, + ); + expect(await sweep.sweep(clock.now())).toBe(0); + expect((await forgottenFor()).map(([person]) => person)).toEqual(["c-1"]); + expect(intended).not.toBe(""); + expect(await storage.personalKeys.get("c-1")).not.toBeNull(); + }); + + it("says a person whose row never saw them active is not due", async () => { + const { storage, clock } = await setUp(); + await storage.persons.record({ person: "c-5", position: 1, forgottenAt: null }); + const due = retentionDueOf({ + retentionMs: DAY, + batchSize: 2, + checkpoints: storage.checkpointStore, + }); + expect(await due(storage, "c-5", new Date(clock.now().getTime() + 2 * DAY))).toBe(false); + }); + + it("records no intention for a person active since their row was recorded", async () => { + const { sweep, clock, recorded, forgottenFor, touch } = await racing( + "listed", + async () => undefined, + ); + await touch("c-1"); + expect(await sweep.sweep(clock.now())).toBe(0); + expect(recorded).toEqual([]); + expect(await forgottenFor()).toEqual([]); + }); +}); + describe("the retention across instances", () => { // A store that runs `after` once each forget commits. const committing = (live: Storage, after: () => Promise): Storage => ({ diff --git a/packages/core/src/kernel/personal-data/retention.ts b/packages/core/src/kernel/personal-data/retention.ts index c33ed5c5..f3e5ca3c 100644 --- a/packages/core/src/kernel/personal-data/retention.ts +++ b/packages/core/src/kernel/personal-data/retention.ts @@ -1,4 +1,5 @@ import type { Storage } from "../../adapter/adapter.ts"; +import type { CheckpointStore } from "../../adapter/storage/checkpoint-store.ts"; import type { EventStore } from "../../adapter/storage/event-store.ts"; import { isReleasedBy } from "../../adapter/storage/hold.ts"; import type { @@ -13,8 +14,10 @@ import type { Logger } from "../../contracts/logger.ts"; import { BOUNDA_AGGREGATE, PERSONAL_DATA_FORGOTTEN_EVENT } from "../../modules/bounda-events.ts"; import type { Subscriber } from "../dispatch/delivery.ts"; import { createLease } from "../shared/lease.ts"; -import { forgetWithin, logForgotten } from "./forget.ts"; +import { forgetWithin, logForgotten, STALE_INTENTION } from "./forget.ts"; import { createPause, holdAfter } from "./hold.ts"; +import type { ForgetIntentions } from "./intention.ts"; +import type { RetentionDueFunction } from "./reconciliation.ts"; import { transactRetrying } from "./transact.ts"; export const RETENTION_SUBSCRIBER: "retention" = "retention"; @@ -49,6 +52,55 @@ const forgettingOf = (event: StoredEvent & { readonly person: string }): PersonA forgottenAt: event.timestamp, }); +const rowIsCurrent = async ( + eventStore: EventStore, + person: string, + through: number, + batchSize: number, +): Promise => { + for (let after = through; ; ) { + const events = await eventStore.readByPerson({ + person, + afterPosition: after, + limit: batchSize, + }); + if (events.some((event) => isActivity(event) || isForget(event))) return false; + const last = events[batchSize - 1]; + if (last === undefined) return true; + after = last.position; + } +}; + +const isIdle = (row: StoredPerson | null): row is StoredPerson => + row !== null && row.restrictedSince === null && row.forgottenAt === null; + +export interface RetentionDueArgs { + readonly retentionMs: number; + readonly batchSize: number; + /** + * Where the retention subscriber keeps how far its rows reach. + */ + readonly checkpoints: CheckpointStore; +} + +export interface RetentionDueOfFunction { + (args: RetentionDueArgs): RetentionDueFunction; +} + +/** + * Whether the retention would forget the person at `now`, as a sweep decides it in the transaction + * that forgets them. + */ +export const retentionDueOf: RetentionDueOfFunction = + ({ retentionMs, batchSize, checkpoints }) => + async (stores, personId, now) => { + const row = await stores.persons.get(personId); + if (!isIdle(row) || row.lastActiveAt === null) return false; + if (Date.parse(row.lastActiveAt) + retentionMs > now.getTime()) return false; + const through = await checkpoints.get(RETENTION_SUBSCRIBER); + return rowIsCurrent(stores.eventStore, personId, through, batchSize); + }; + export interface CreateRetentionSubscriberArgs { readonly persons: PersonStore; } @@ -116,6 +168,7 @@ export interface RetentionSweep { export interface CreateRetentionSweepArgs { readonly storage: Storage; + readonly intentions: ForgetIntentions; readonly retentionMs: number; readonly ids: IdGenerator; readonly clock: Clock; @@ -142,6 +195,7 @@ export interface CreateRetentionSweepFunction { */ export const createRetentionSweep: CreateRetentionSweepFunction = ({ storage, + intentions, retentionMs, ids, clock, @@ -150,53 +204,46 @@ export const createRetentionSweep: CreateRetentionSweepFunction = ({ batchSize, forgotten: onForgotten, }) => { - const rowIsCurrent = async ( - eventStore: EventStore, - person: string, - through: number, - ): Promise => { - for (let after = through; ; ) { - const events = await eventStore.readByPerson({ - person, - afterPosition: after, - limit: batchSize, - }); - if (events.some((event) => isActivity(event) || isForget(event))) return false; - const last = events[batchSize - 1]; - if (last === undefined) return true; - after = last.position; - } - }; - - // Resolves `undone` for a person whose row is behind, who is then left for a later pass, and - // `skipped` for one restricted, or forgotten by another instance, since the list was read. + // Resolves `undone` for a person whose row is behind, or who was given a key since the + // intention, who is then left for a later pass, and `skipped` for one restricted, or forgotten by + // another instance, since the list was read. Both are decided once before the intention, so a + // person plainly left leaves none, and again in the transaction; an intention it then leaves is + // committed again only if the retention would still forget the person. const forget = async ( personId: string, through: number, ): Promise<"forgotten" | "skipped" | "undone"> => { + if (!isIdle(await storage.persons.get(personId))) return "skipped"; + if (!(await rowIsCurrent(storage.eventStore, personId, through, batchSize))) return "undone"; + const intention = await intentions.intend({ + personId, + reason: "retention", + keys: storage.personalKeys, + }); const correlationId = ids.next(); const forgotten = await transactRetrying({ storage, concurrencyRetries, work: async (transaction) => { - const row = await transaction.persons.get(personId); - if (row === null || row.restrictedSince !== null || row.forgottenAt !== null) { - return "skipped"; + if (!isIdle(await transaction.persons.get(personId))) return "skipped"; + if (!(await rowIsCurrent(transaction.eventStore, personId, through, batchSize))) { + return "undone"; } - if (!(await rowIsCurrent(transaction.eventStore, personId, through))) return "undone"; - const forgotten = await forgetWithin({ + const outcome = await forgetWithin({ transaction, ids, clock, personId, - reason: "retention", + intention, context: { correlationId, causationId: correlationId, depth: 0 }, }); + if (outcome === STALE_INTENTION) return "undone"; + if ("held" in outcome) return "skipped"; // Recorded here as the subscriber will record it, so the row leaves the idle list, its // hold with it, before any pass can take the person again; the subscriber's record of the // same event then changes nothing. - await transaction.persons.record(forgettingOf({ ...forgotten.event, person: personId })); - return forgotten; + await transaction.persons.record(forgettingOf({ ...outcome.event, person: personId })); + return outcome; }, }); if (typeof forgotten === "string") return forgotten; diff --git a/packages/core/src/kernel/policy/executor.ts b/packages/core/src/kernel/policy/executor.ts index 2a77cc83..5866c295 100644 --- a/packages/core/src/kernel/policy/executor.ts +++ b/packages/core/src/kernel/policy/executor.ts @@ -7,6 +7,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import { createReactionCommands } from "../command/reaction-commands.ts"; import { openedUnder } from "../personal-data/event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { actedBy, causedBy } from "../shared/causation.ts"; import { deriveIdempotencyKey } from "../shared/idempotency-key.ts"; import { withTimeout } from "../shared/timeout.ts"; @@ -51,6 +52,7 @@ export interface CreatePolicyExecutorArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreatePolicyExecutorFunction { @@ -64,6 +66,7 @@ export const createPolicyExecutor: CreatePolicyExecutorFunction = ({ ids, clock, logger, + intentions, }) => ({ run: async ({ policy, event, attempt, retryId, actor, within }) => { within.read(openedUnder(event)); @@ -83,6 +86,7 @@ export const createPolicyExecutor: CreatePolicyExecutorFunction = ({ logger, ids, clock, + intentions, maxChainDepth: config.forAggregate(policy.aggregate).policies.maxChainDepth, }); const handled = traced({ diff --git a/packages/core/src/kernel/process/handlers.ts b/packages/core/src/kernel/process/handlers.ts index 33e8bdc1..e0e6e6f8 100644 --- a/packages/core/src/kernel/process/handlers.ts +++ b/packages/core/src/kernel/process/handlers.ts @@ -10,6 +10,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import { createReactionCommands, type ReactionCommands } from "../command/reaction-commands.ts"; import { openedUnder } from "../personal-data/event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { checkStatePerson } from "../personal-data/process-state.ts"; import { refuseRestricted } from "../personal-data/refusal.ts"; import { actedBy, causedBy } from "../shared/causation.ts"; @@ -89,6 +90,7 @@ export interface CreateProcessHandlersArgs { readonly ids: IdGenerator; readonly clock: Clock; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreateProcessHandlersFunction { @@ -102,6 +104,7 @@ export const createProcessHandlers: CreateProcessHandlersFunction = ({ ids, clock, logger, + intentions, }) => { const reactionFor = ( process: ProcessRuntime, @@ -118,6 +121,7 @@ export const createProcessHandlers: CreateProcessHandlersFunction = ({ logger, ids, clock, + intentions, maxChainDepth: config.forAggregate(process.aggregate).policies.maxChainDepth, }); diff --git a/packages/core/src/kernel/process/runner.ts b/packages/core/src/kernel/process/runner.ts index 9ea95ffd..93b307a6 100644 --- a/packages/core/src/kernel/process/runner.ts +++ b/packages/core/src/kernel/process/runner.ts @@ -6,6 +6,7 @@ import type { AggregatesRuntime } from "../aggregate/runtime.ts"; import type { CommandPipeline } from "../command/pipeline.ts"; import type { Subscriber } from "../dispatch/dispatcher.ts"; import type { AppStorage } from "../events/app-event-store.ts"; +import type { ForgetIntentions } from "../personal-data/intention.ts"; import { deliverInOrder } from "../shared/in-order.ts"; import type { PendingRetries } from "../shared/pending-retries.ts"; import type { ProcessesRuntime } from "./build-processes.ts"; @@ -33,6 +34,7 @@ export interface CreateProcessRunnerArgs { readonly clock: Clock; readonly pendingRetries: PendingRetries; readonly logger: Logger; + readonly intentions: ForgetIntentions; } export interface CreateProcessRunnerFunction { @@ -53,6 +55,7 @@ export const createProcessRunner: CreateProcessRunnerFunction = ({ clock, pendingRetries, logger, + intentions, }) => { const units = createProcessUnits({ storage, config, ids, clock }); const failures = createProcessFailures({ @@ -69,6 +72,7 @@ export const createProcessRunner: CreateProcessRunnerFunction = ({ ids, clock, logger, + intentions, }); const schedule = createDeadlineSchedule({ storage, units }); const deadlineStep = createDeadlineStep({ units, handlers, ids }); diff --git a/packages/core/src/kernel/reactive-harness.ts b/packages/core/src/kernel/reactive-harness.ts index 940a987f..fdfec034 100644 --- a/packages/core/src/kernel/reactive-harness.ts +++ b/packages/core/src/kernel/reactive-harness.ts @@ -14,6 +14,7 @@ import { createRestrictedRetries } from "./dead-letters/restricted.ts"; import { createDeadLetterRetry, type DeadLetterRetry } from "./dead-letters/retry.ts"; import { createDispatcher, type Dispatcher } from "./dispatch/dispatcher.ts"; import type { AppStorage } from "./events/app-event-store.ts"; +import { createForgetIntentions } from "./personal-data/intention.ts"; import { buildPolicies, type PoliciesRuntime } from "./policy/build-policies.ts"; import { createDelayedPolicies } from "./policy/delayed.ts"; import { createPolicyExecutor, type PolicyExecutor } from "./policy/executor.ts"; @@ -104,6 +105,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ logger, }); const policies = buildPolicies({ registry, aggregates }); + const intentions = createForgetIntentions({ ledger: null, ids, clock, logger }); const policyExecutor = createPolicyExecutor({ aggregates, pipeline, @@ -111,6 +113,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ ids, clock, logger, + intentions, }); const processes = createProcessRunner({ processes: buildProcesses({ registry, aggregates, config }), @@ -122,6 +125,7 @@ export const createReactiveHarness: CreateReactiveHarnessFunction = async ({ clock, pendingRetries: ignoredRetries, logger, + intentions, }); const retries = createDeadLetterRetry({ storage, diff --git a/packages/core/src/kernel/shared/restores-app.test.ts b/packages/core/src/kernel/shared/restores-app.test.ts index 86388166..df8174d4 100644 --- a/packages/core/src/kernel/shared/restores-app.test.ts +++ b/packages/core/src/kernel/shared/restores-app.test.ts @@ -16,6 +16,7 @@ import { type BoundaApp, createApp } from "../app.ts"; import { defineForgetLedger } from "../personal-data/forget-ledger.ts"; import { HOLD_MAX_MS } from "../personal-data/hold.ts"; import { masterKey } from "../personal-data/master-key.ts"; +import { FORGET_LEDGER_GRACE_MS } from "../personal-data/reconciliation.ts"; import { ROTATION_CHECK_INTERVAL_MS } from "../personal-data/rotation.ts"; import { advanceUntilWaiting } from "../test-support.ts"; import { GENERATION_INTERVAL_MS } from "./restores.ts"; @@ -115,9 +116,10 @@ const start = async (args: { readonly runtime?: RuntimeConfig; readonly env?: Record; readonly logger?: Logger; + readonly now?: Date; }): Promise<{ app: BoundaApp; clock: ReturnType }> => { started += 1; - const clock = createFixedClock(START); + const clock = createFixedClock(args.now ?? START); const app = await createApp({ registry, config: { @@ -269,7 +271,7 @@ describe("a database restored while the app runs", () => { const reasons = (await app.events.read({ limit: 100 })) .filter((event) => event.type === "PersonalDataForgotten") .map((event) => (event.payload as PersonalDataForgottenPayload).reason); - expect(reasons).toEqual(["restored"]); + expect(reasons).toEqual(["requested"]); // Found once, though the reconciliation of the same pass forgot someone again. expect((await app.status()).restoresFound).toBe(1); }); @@ -354,7 +356,7 @@ describe("a database restored while the app was stopped", () => { const reasons = (await after.app.events.read({ limit: 100 })) .filter((event) => event.type === "PersonalDataForgotten") .map((event) => (event.payload as PersonalDataForgottenPayload).reason); - expect(reasons).toEqual(["restored"]); + expect(reasons).toEqual(["requested"]); expect((await after.app.status()).restoresFound).toBe(1); await after.app.stop(); const again = recordingLogger(); @@ -384,7 +386,9 @@ describe("a database restored while the app was stopped", () => { await before.app.stop(); restore(store.db, copy); const logger = recordingLogger(); - const after = await start({ adapter: store.adapter, runtime, logger }); + // A restore takes longer than the grace a reconciliation gives a forgetting still committing. + const now = new Date(START.getTime() + FORGET_LEDGER_GRACE_MS); + const after = await start({ adapter: store.adapter, runtime, logger, now }); expect(restarts(logger).map(({ fields }) => fields)).toEqual([{ first: true }]); expect((await after.app.status()).restoresFound).toBe(1); expect((await after.app.getLag()).forgetLedger).toEqual({ reconciled: true }); diff --git a/packages/core/src/kernel/shared/runtime-text.ts b/packages/core/src/kernel/shared/runtime-text.ts index d7498d80..51e18a11 100644 --- a/packages/core/src/kernel/shared/runtime-text.ts +++ b/packages/core/src/kernel/shared/runtime-text.ts @@ -3,6 +3,8 @@ import { ClaimLostError, ConcurrencyError, DeadLetterSettledError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, ProcessingRestrictedError, RebuildSupersededError, @@ -18,6 +20,8 @@ const COMPOSED: readonly (abstract new (...args: never[]) => Error)[] = [ ClaimLostError, ConcurrencyError, DeadLetterSettledError, + ForgetLedgerTimeoutError, + ForgetLedgerUnavailableError, ForgetLedgerUnreconciledError, HandlerTimeoutError, ProcessingRestrictedError, diff --git a/packages/core/src/kernel/system-events.test.ts b/packages/core/src/kernel/system-events.test.ts index 61e7baee..a246e948 100644 --- a/packages/core/src/kernel/system-events.test.ts +++ b/packages/core/src/kernel/system-events.test.ts @@ -68,6 +68,23 @@ describe("appendSystemEvent", () => { ).toEqual([appended]); }); + it("takes the id and timestamp set before it, when given", async () => { + const storage = await memory().createStorage(); + const appended = await appendSystemEvent({ + eventStore: storage.eventStore, + ids: createSequentialIdGenerator(), + clock: createFixedClock(), + aggregateType: "bounda", + aggregateId: "c-1", + type: "PersonalDataForgotten", + payload: {}, + context, + identity: { id: "intended", timestamp: "2026-01-02T03:04:05.000Z" }, + }); + expect(appended.id).toBe("intended"); + expect(appended.timestamp).toBe("2026-01-02T03:04:05.000Z"); + }); + it("lets a stream that moved fail, for the unit of work to run again", async () => { const { storage, append } = await setup(); let appends = 0; diff --git a/packages/core/src/kernel/system-events.ts b/packages/core/src/kernel/system-events.ts index d25c4832..26db33b4 100644 --- a/packages/core/src/kernel/system-events.ts +++ b/packages/core/src/kernel/system-events.ts @@ -30,6 +30,11 @@ export interface AppendSystemEventArgs { * The person the event is about, on their own stream. */ readonly person?: string; + /** + * The event's id and timestamp when they were set before, as a forgetting's intention sets them; + * from `ids` and `clock` when omitted. + */ + readonly identity?: { readonly id: string; readonly timestamp: string }; } export interface AppendSystemEventFunction { @@ -51,6 +56,7 @@ export const appendSystemEvent: AppendSystemEventFunction = async ({ payload, context, person = null, + identity, }) => { const { version } = await eventStore.load({ aggregateType, @@ -63,13 +69,13 @@ export const appendSystemEvent: AppendSystemEventFunction = async ({ expectedVersion: version, events: [ { - id: ids.next(), + id: identity?.id ?? ids.next(), aggregateType, aggregateId, version: version + 1, type, payload, - timestamp: clock.now().toISOString(), + timestamp: identity?.timestamp ?? clock.now().toISOString(), metadata: { ...context, schemaVersion: 1, system: true }, person, }, diff --git a/packages/core/src/kernel/test-support.ts b/packages/core/src/kernel/test-support.ts index a33e1f96..9273b0b0 100644 --- a/packages/core/src/kernel/test-support.ts +++ b/packages/core/src/kernel/test-support.ts @@ -6,7 +6,7 @@ import { resolveConfig } from "../config/schema.ts"; import type { Config, ErrorText, PortsConfig, ResolvedConfig } from "../config/types.ts"; import { type Clock, createFixedClock, type FixedClock } from "../contracts/clock.ts"; import type { CommandRejection } from "../contracts/command.ts"; -import { createSequentialIdGenerator } from "../contracts/ids.ts"; +import { createSequentialIdGenerator, type IdGenerator } from "../contracts/ids.ts"; import { type LogFields, type Logger, silentLogger } from "../contracts/logger.ts"; import { memory } from "../memory/index.ts"; import type { RejectFunction } from "../modules/command.ts"; @@ -27,6 +27,7 @@ import { toBase64, withRawKey, } from "./personal-data/crypto.ts"; +import { createForgetIntentions, type ForgetIntentions } from "./personal-data/intention.ts"; import { createPersonalKeyWrapping, type PersonalKeys } from "./personal-data/wrapped-keys.ts"; import { createPorts, type ModulePorts } from "./ports/ports.ts"; import { guardedLogger } from "./shared/guarded-logger.ts"; @@ -485,3 +486,14 @@ export const openedKey: OpenedKeyFunction = async ({ key, fingerprint }) => ({ fingerprint, cryptoKey: await importRawKey({ raw: fromBase64(key), extractable: false }), }); + +export interface WithoutLedgerArgs { + readonly ids: IdGenerator; + readonly clock: Clock; +} + +/** + * The intentions of an app without a forget ledger: ids and times, nothing recorded. + */ +export const withoutLedger = ({ ids, clock }: WithoutLedgerArgs): ForgetIntentions => + createForgetIntentions({ ledger: null, ids, clock, logger: silentLogger }); diff --git a/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts b/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts index 82f9db56..d5cb63d8 100644 --- a/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts +++ b/packages/core/src/kernel/unit-of-work/unit-of-work.test.ts @@ -331,6 +331,47 @@ describe("createUnitOfWork", () => { expect(transactions).toBe(1); }); + it("runs the steps before the commit's transaction, in order, on the store, and writes nothing when one throws", async () => { + const storage = await memory().createStorage(); + await storage.personalKeys.add("c-1", { key: "k", fingerprint: "f-1", kid: null }); + const transact = storage.transact.bind(storage); + const ran: string[] = []; + storage.transact = (work) => { + ran.push("transaction"); + return transact(work); + }; + const unit = createUnitOfWork({ storage }); + unit.beforeCommit(async ({ personalKeys }) => { + ran.push(`first ${(await personalKeys.get("c-1"))?.fingerprint}`); + }); + unit.beforeCommit(async () => { + ran.push("second"); + }); + unit.atCommit(async () => { + ran.push("at commit"); + }); + expect(ran).toEqual([]); + await unit.commit(); + expect(ran).toEqual(["first f-1", "second", "transaction", "at commit"]); + + const refused = createUnitOfWork({ storage }); + await refused.scheduler.schedule(entry("command:refused")); + refused.beforeCommit(async () => { + throw new Error("ledger is down"); + }); + refused.afterCommit(() => ran.push("after")); + await expect(refused.commit()).rejects.toThrow("ledger is down"); + expect(await storage.scheduler.list()).toEqual([]); + expect(ran).toEqual(["first f-1", "second", "transaction", "at commit"]); + + const idle = createUnitOfWork({ storage }); + idle.beforeCommit(async () => { + ran.push("idle"); + }); + await idle.commit(); + expect(ran).not.toContain("idle"); + }); + it("commits nothing when a stream moved since the unit loaded it", async () => { const storage = await memory().createStorage(); const unit = createUnitOfWork({ storage }); diff --git a/packages/core/src/kernel/unit-of-work/unit-of-work.ts b/packages/core/src/kernel/unit-of-work/unit-of-work.ts index 01209253..2fc46b25 100644 --- a/packages/core/src/kernel/unit-of-work/unit-of-work.ts +++ b/packages/core/src/kernel/unit-of-work/unit-of-work.ts @@ -11,7 +11,14 @@ import { asError } from "../shared/retry.ts"; */ export type UnitStores = Pick< UnitOfWork, - "eventStore" | "scheduler" | "idempotencyStore" | "persons" | "atCommit" | "afterCommit" | "read" + | "eventStore" + | "scheduler" + | "idempotencyStore" + | "persons" + | "beforeCommit" + | "atCommit" + | "afterCommit" + | "read" >; /** @@ -26,6 +33,12 @@ export type UnitStores = Pick< */ export interface UnitOfWork extends Omit { commit(): Promise; + /** + * Runs `step` before the commit's transaction opens, in the order given, for what has to happen + * outside the store first, as recording a forgetting's intention does: what it throws fails the + * commit, and nothing is written. Never for a unit that does not commit. + */ + beforeCommit(step: (storage: Pick) => Promise): void; /** * Runs `write` on the commit's transaction, after the unit's events and other writes, in the * order given: for a write that needs what the transaction's stores answer, as forgetting a @@ -59,6 +72,7 @@ export interface CreateUnitOfWorkFunction { export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { const staged = createStagedEventStore(storage.eventStore); const { stores, flush, pending } = deferWrites(storage); + const beforeCommit: ((storage: Pick) => Promise)[] = []; const atCommit: ((transaction: StorageTransaction) => Promise)[] = []; const committed: (() => void)[] = []; const read: KeyedPerson[] = []; @@ -72,6 +86,7 @@ export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { const batches = staged.batches(); const reseals = staged.reseals(); if (batches.length > 0 || reseals.length > 0 || pending() || atCommit.length > 0) { + for (const step of beforeCommit) await step(storage); await storage.transact(async (transaction) => { for (const batch of batches) await transaction.eventStore.append(batch); if (reseals.length > 0) await transaction.eventStore.reseal(reseals); @@ -81,6 +96,9 @@ export const createUnitOfWork: CreateUnitOfWorkFunction = ({ storage }) => { } for (const callback of committed.splice(0)) callback(); }, + beforeCommit: (step) => { + beforeCommit.push(step); + }, atCommit: (write) => { atCommit.push(write); }, diff --git a/packages/core/src/modules/bounda-events.ts b/packages/core/src/modules/bounda-events.ts index 6e74fb8c..71227588 100644 --- a/packages/core/src/modules/bounda-events.ts +++ b/packages/core/src/modules/bounda-events.ts @@ -12,7 +12,8 @@ export const BOUNDA_AGGREGATE: "bounda" = "bounda"; * a policy or process that forgets them, or the retention: every event's personal field of theirs * reads as `null` from then on, and every read model that projects such fields handles it. It * names the stored commands that held the person's data and went with the key, since no other - * event records them. + * event records them. With a forget ledger, the reconciliation may append it again, with the same + * id, time and reason, to a store that lost it. */ export const PERSONAL_DATA_FORGOTTEN_EVENT: "PersonalDataForgotten" = "PersonalDataForgotten"; @@ -33,16 +34,22 @@ export interface RemovedDeadLetter { } /** - * Why a person was forgotten: `requested` by `forget`, from a client or a policy or process, - * `retention`, past `runtime.personalData.retention` without activity, or `restored`, again, - * because a backup restored from before they were forgotten brought their key back, as - * `runtime.personalData.forgetLedger` or `bounda forget --from` found. + * Why a person was forgotten: `requested` by `forget`, from a client or a policy or process, or + * by `bounda forget --from` for a bare id, or `retention`, past `runtime.personalData.retention` + * without activity. A forgetting committed again from `runtime.personalData.forgetLedger`, after + * a restore or a commit that never happened, keeps the reason it was asked with. */ -export type PersonalDataForgottenReason = "requested" | "retention" | "restored"; +export type PersonalDataForgottenReason = "requested" | "retention"; export interface PersonalDataForgottenPayload { readonly personId: string; readonly reason: PersonalDataForgottenReason; + /** + * The fingerprints of the keys the forgetting destroyed, none when the person held none: a key + * the person holds whose fingerprint is not here was given to them after. Absent from a + * forgetting recorded before Bounda 0.4.0. + */ + readonly fingerprints?: readonly string[]; readonly cancelledCommands: readonly CancelledCommand[]; readonly removedDeadLetters: readonly RemovedDeadLetter[]; } diff --git a/packages/core/src/modules/policy.ts b/packages/core/src/modules/policy.ts index 1eea8fde..f1304147 100644 --- a/packages/core/src/modules/policy.ts +++ b/packages/core/src/modules/policy.ts @@ -40,7 +40,9 @@ export type PolicyHandlerArgs; diff --git a/packages/core/src/testing/expect-forgotten.test.ts b/packages/core/src/testing/expect-forgotten.test.ts index 58099b98..f7eac456 100644 --- a/packages/core/src/testing/expect-forgotten.test.ts +++ b/packages/core/src/testing/expect-forgotten.test.ts @@ -803,6 +803,7 @@ describe("expectForgotten", () => { personId: `p-${index}`, eventId: `ledger-event-${String(index).padStart(4, "0")}`, forgottenAt: new Date(Date.UTC(2025, 0, 1, 0, 0, index)).toISOString(), + reason: "requested" as const, })), ); const { app } = await createTestApp({ diff --git a/packages/core/test-types/config.test-d.ts b/packages/core/test-types/config.test-d.ts index 90315e74..696f49ef 100644 --- a/packages/core/test-types/config.test-d.ts +++ b/packages/core/test-types/config.test-d.ts @@ -1,3 +1,4 @@ +import type { PersonalDataForgottenReason } from "@bounda-dev/core"; import { type DurationInput, ERROR_TEXT_VARIABLE, @@ -406,7 +407,13 @@ describe("defineConfig", () => { ports, runtime: { personalData: { - forgetLedger: { store: own, reconcileEvery: "1m", strict: true, keepFor: "35d" }, + forgetLedger: { + store: own, + reconcileEvery: "1m", + strict: true, + keepFor: "35d", + timeout: "5s", + }, }, }, }); @@ -415,11 +422,14 @@ describe("defineConfig", () => { readonly reconcileEvery?: DurationInput; readonly strict?: boolean; readonly keepFor?: DurationInput; + readonly timeout?: DurationInput; }>(); expectTypeOf().toEqualTypeOf<{ readonly personId: string; readonly eventId: string; readonly forgottenAt: string; + readonly reason: PersonalDataForgottenReason; + readonly fingerprints?: readonly string[]; }>(); expectTypeOf().toEqualTypeOf<{ readonly before: string }>(); expectTypeOf().toEqualTypeOf(); diff --git a/packages/core/test-types/inference.test-d.ts b/packages/core/test-types/inference.test-d.ts index 8f6f3ad9..cd198cf9 100644 --- a/packages/core/test-types/inference.test-d.ts +++ b/packages/core/test-types/inference.test-d.ts @@ -734,15 +734,14 @@ describe("personal data", () => { expectTypeOf().toEqualTypeOf<{ readonly personId: string; readonly reason: PersonalDataForgottenReason; + readonly fingerprints?: readonly string[]; readonly cancelledCommands: readonly CancelledCommand[]; readonly removedDeadLetters: readonly RemovedDeadLetter[]; }>(); - expectTypeOf().toEqualTypeOf< - "requested" | "retention" | "restored" - >(); + expectTypeOf().toEqualTypeOf<"requested" | "retention">(); expectTypeOf< AuditOnPersonalDataForgotten.HandlerArgs["event"]["payload"]["reason"] - >().toEqualTypeOf<"requested" | "retention" | "restored">(); + >().toEqualTypeOf<"requested" | "retention">(); expectTypeOf().toEqualTypeOf<{ readonly commandId: string; readonly type: string; diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 80f5bb81..3af2fb93 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "core", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], typecheck: { enabled: true, include: ["test-types/**/*.test-d.ts"], diff --git a/packages/create-bounda/vitest.config.ts b/packages/create-bounda/vitest.config.ts index ed0ae200..e54e2b3a 100644 --- a/packages/create-bounda/vitest.config.ts +++ b/packages/create-bounda/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "create-bounda", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], testTimeout: 60_000, }, diff --git a/packages/postgresql/src/migrations.ts b/packages/postgresql/src/migrations.ts index 9be8b4f9..f0039802 100644 --- a/packages/postgresql/src/migrations.ts +++ b/packages/postgresql/src/migrations.ts @@ -2,6 +2,7 @@ import { ConfigurationError } from "@bounda-dev/core"; import type { SchemaVersion } from "@bounda-dev/core/adapter"; import { forgetLedgerMarksStatement, + forgetLedgerReasonStatements, forgetLedgerStatements, migrateSchemas, postgresqlDialect, @@ -181,6 +182,12 @@ export const postgresqlForgetLedgerScope: PostgresqlForgetLedgerScopeFunction = }, }, { version: 2, migrate: (tx) => tx.run(forgetLedgerMarksStatement(table), []) }, + { + version: 3, + migrate: async (tx) => { + for (const statement of forgetLedgerReasonStatements(table)) await tx.run(statement, []); + }, + }, ], }); diff --git a/packages/react-router/vitest.config.ts b/packages/react-router/vitest.config.ts index 2d5a24ea..f46b5cc8 100644 --- a/packages/react-router/vitest.config.ts +++ b/packages/react-router/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ }, test: { name: "react-router", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/packages/sqlite/src/database.ts b/packages/sqlite/src/database.ts index db2937ff..7aebbbee 100644 --- a/packages/sqlite/src/database.ts +++ b/packages/sqlite/src/database.ts @@ -17,8 +17,16 @@ export interface CreateSqliteDatabaseOptions { readonly wal: boolean; } +export interface LibsqlConnection extends SqliteConnection { + /** + * Resolves once no statement or transaction it started is still running: closing the client + * under one frees what libSQL's native code is still using. + */ + settled(): Promise; +} + export interface CreateSqliteDatabaseFunction { - (client: Client, options: CreateSqliteDatabaseOptions): SqliteConnection; + (client: Client, options: CreateSqliteDatabaseOptions): LibsqlConnection; } const toRows = (result: ResultSet): readonly Record[] => @@ -52,6 +60,15 @@ const QUEUED_METHODS: ReadonlySet = new Set(["execute", "batch", "e export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { local, wal }) => { const serially = createSerialQueue(); + const running = new Set>(); + const tracked = (work: Promise): Promise => { + running.add(work); + const done = (): void => { + running.delete(work); + }; + work.then(done, done); + return work; + }; let ready: Promise | undefined; const prepared = (task: () => Promise): Promise => { if (ready === undefined && wal) { @@ -66,7 +83,7 @@ export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { loc return ready === undefined ? task() : ready.then(task); }; const outside = (task: () => Promise): Promise => - prepared(() => (local ? serially(task) : task())); + tracked(prepared(() => (local ? serially(task) : task()))); const direct = executorOf(client); const raw = local ? new Proxy(client, { @@ -83,24 +100,29 @@ export const createSqliteDatabase: CreateSqliteDatabaseFunction = (client, { loc return { raw, isBusy, + settled: async () => { + while (running.size > 0) await Promise.allSettled([...running]); + }, db: { run: (sql, params) => outside(() => direct.run(sql, params)), all: (sql, params) => outside(() => direct.all(sql, params)), // One write transaction at a time within the process, so it never waits on the engine's busy // handler for another transaction of its own. write: (work) => - prepared(() => - serially(async () => { - const transaction = await client.transaction("write"); - try { - const result = await work({ ...executorOf(transaction), raw: transaction }); - await transaction.commit(); - return result; - } catch (error) { - await transaction.rollback(); - throw error; - } - }), + tracked( + prepared(() => + serially(async () => { + const transaction = await client.transaction("write"); + try { + const result = await work({ ...executorOf(transaction), raw: transaction }); + await transaction.commit(); + return result; + } catch (error) { + await transaction.rollback(); + throw error; + } + }), + ), ), }, }; diff --git a/packages/sqlite/src/forget-ledger.test.ts b/packages/sqlite/src/forget-ledger.test.ts index 67c085ba..b909366f 100644 --- a/packages/sqlite/src/forget-ledger.test.ts +++ b/packages/sqlite/src/forget-ledger.test.ts @@ -4,13 +4,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type BoundaApp, + type Clock, createApp, type EventPayloadArgs, type PayloadArgs, type PersonalDataForgottenPayload, type Registry, + systemClock, } from "@bounda-dev/core"; -import type { ForgetLedger } from "@bounda-dev/core/adapter"; +import type { Adapter, ForgetLedger } from "@bounda-dev/core/adapter"; import { forgetLedgerContract } from "@bounda-dev/core/adapter/testing"; import { createClient } from "@libsql/client"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -92,13 +94,24 @@ interface Paths { readonly ledger: string; } -const boot = (paths: Paths): Promise => +// How far ahead a start after a restore runs: past the grace a reconciliation gives an intention, +// as a restore takes longer than that. +const LATER_MS = 2 * 60_000; + +// The system's clock, `LATER_MS` ahead. +const later: Clock = { + now: () => new Date(Date.now() + LATER_MS), + after: (milliseconds, callback, options) => systemClock.after(milliseconds, callback, options), +}; + +const boot = (paths: Paths, clock: Clock = systemClock): Promise => createApp({ registry, config: { storage: sqlite({ path: paths.store }), runtime: { personalData: { forgetLedger: { store: sqlite({ path: paths.ledger }) } } }, }, + clock, }); const register = async (app: BoundaApp, customerId: string): Promise => { @@ -129,7 +142,7 @@ const startElsewhere = (paths: Paths): Promise => [ "--input-type=module", "-e", - `import { createApp } from "@bounda-dev/core"; + `import { createApp, systemClock } from "@bounda-dev/core"; import { sqlite } from "./index.ts"; const app = await createApp({ registry: { aggregates: {}, readModels: {} }, @@ -137,6 +150,10 @@ const startElsewhere = (paths: Paths): Promise => storage: sqlite({ path: ${JSON.stringify(paths.store)} }), runtime: { personalData: { forgetLedger: { store: sqlite({ path: ${JSON.stringify(paths.ledger)} }) } } }, }, + clock: { + now: () => new Date(Date.now() + ${LATER_MS}), + after: (milliseconds, callback, options) => systemClock.after(milliseconds, callback, options), + }, }); await app.stop();`, ], @@ -180,15 +197,15 @@ describe("restoring a backup with a forget ledger", () => { await before.stop(); const restoredPaths = { ...paths, store: await restore(backup) }; - const restored = await boot(restoredPaths); - expect(await forgotten(restored)).toEqual([["c-1", "restored"]]); + const restored = await boot(restoredPaths, later); + expect(await forgotten(restored)).toEqual([["c-1", "requested"]]); expect(await nameOf(restored, "c-1")).toBeNull(); expect(await nameOf(restored, "c-2")).toBe("name of c-2"); await restored.runUntilIdle(); await restored.stop(); - const again = await boot(restoredPaths); - expect(await forgotten(again)).toEqual([["c-1", "restored"]]); + const again = await boot(restoredPaths, later); + expect(await forgotten(again)).toEqual([["c-1", "requested"]]); expect(await again.getLag()).toMatchObject({ forgetLedger: { reconciled: true } }); await again.stop(); }); @@ -210,11 +227,65 @@ describe("restoring a backup with a forget ledger", () => { await client.execute("PRAGMA journal_mode = WAL"); client.close(); await Promise.all([startElsewhere(restoredPaths), startElsewhere(restoredPaths)]); - const after = await boot(restoredPaths); - expect(await forgotten(after)).toEqual([["c-1", "restored"]]); + const after = await boot(restoredPaths, later); + expect(await forgotten(after)).toEqual([["c-1", "requested"]]); await after.stop(); }); + it("commits on the restored copy the forgettings after the backup, a pending intention's included, each under its id", async () => { + const paths = fresh(); + const backup = pathOf(`backup-${databases}`); + const before = await boot(paths); + await register(before, "c-1"); + await register(before, "c-2"); + await register(before, "c-3"); + await backUp(paths.store, backup); + await before.forget("c-1"); + await before.stop(); + // The process stops between the intention and the commit of c-2's forgetting. + let crashing = true; + const store = sqlite({ path: paths.store }); + const stopped: Adapter = { + ...store, + createStorage: async () => { + const opened = await store.createStorage(); + return { + ...opened, + transact: (work) => + crashing ? Promise.reject(new Error("the process stopped")) : opened.transact(work), + }; + }, + }; + const stopping = await createApp({ + registry, + config: { + storage: stopped, + runtime: { personalData: { forgetLedger: { store: sqlite({ path: paths.ledger }) } } }, + }, + }); + await expect(stopping.forget("c-2")).rejects.toThrow("the process stopped"); + crashing = false; + await stopping.stop(); + + const ledger = await sqlite({ path: paths.ledger }).createForgetLedger({}); + opened.push(ledger); + const intentions = await ledger.read({ limit: 10 }); + expect(intentions.map(({ personId, reason }) => [personId, reason])).toEqual([ + ["c-1", "requested"], + ["c-2", "requested"], + ]); + const restored = await boot({ ...paths, store: await restore(backup) }, later); + expect( + (await restored.events.read({ limit: 1000 })) + .filter((event) => event.type === "PersonalDataForgotten") + .map((event) => [event.aggregateId, event.id]), + ).toEqual(intentions.map(({ personId, eventId }) => [personId, eventId])); + expect(await nameOf(restored, "c-1")).toBeNull(); + expect(await nameOf(restored, "c-2")).toBeNull(); + expect(await nameOf(restored, "c-3")).toBe("name of c-3"); + await restored.stop(); + }); + it("brings the person back without a ledger, which bounda forget --from then forgets again", async () => { const store = fresh().store; const backup = pathOf(`backup-${databases}`); diff --git a/packages/sqlite/src/index.ts b/packages/sqlite/src/index.ts index 49c81ac3..e18687d3 100644 --- a/packages/sqlite/src/index.ts +++ b/packages/sqlite/src/index.ts @@ -1,13 +1,9 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import type { Adapter } from "@bounda-dev/core/adapter"; -import { - createSqliteAdapter, - SQLITE_BUSY_TIMEOUT_MS, - type SqliteConnection, -} from "@bounda-dev/core/adapter/sqlite"; +import { createSqliteAdapter, SQLITE_BUSY_TIMEOUT_MS } from "@bounda-dev/core/adapter/sqlite"; import { type Client, createClient } from "@libsql/client"; -import { createSqliteDatabase } from "./database.ts"; +import { createSqliteDatabase, type LibsqlConnection } from "./database.ts"; import { resolveSqliteOptions, type SqliteOptions } from "./options.ts"; /** @@ -20,7 +16,7 @@ export interface SqliteFunction { (options: SqliteOptions): SqliteAdapter; } -interface Connection extends SqliteConnection { +interface Connection extends LibsqlConnection { readonly client: Client; uses: number; } @@ -44,21 +40,25 @@ export const sqlite: SqliteFunction = (options) => { ...(authToken === undefined ? {} : { authToken }), ...(location === "file" ? { timeout: SQLITE_BUSY_TIMEOUT_MS } : {}), }); - const { db, raw } = createSqliteDatabase(client, { + const opened = createSqliteDatabase(client, { local: location !== "remote", wal: location === "file", }); - connection = { client, db, raw, uses: 0 }; + connection = { ...opened, client, uses: 0 }; } connection.uses += 1; return connection; }; + // The client closes once what it runs has settled, unless it was handed out again meanwhile. const release = async (): Promise => { - if (connection === null) return; - connection.uses -= 1; - if (connection.uses > 0) return; - connection.client.close(); + const releasing = connection; + if (releasing === null) return; + releasing.uses -= 1; + if (releasing.uses > 0) return; + await releasing.settled(); + if (releasing.uses > 0 || connection !== releasing) return; + releasing.client.close(); connection = null; }; diff --git a/packages/sqlite/src/sqlite.test.ts b/packages/sqlite/src/sqlite.test.ts index 86cdc414..81671ad2 100644 --- a/packages/sqlite/src/sqlite.test.ts +++ b/packages/sqlite/src/sqlite.test.ts @@ -2,7 +2,6 @@ import { spawn } from "node:child_process"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { Worker } from "node:worker_threads"; import { ConfigurationError, type FieldsArgs, @@ -113,6 +112,57 @@ describe("sqlite adapter in memory, under concurrent use", () => { }); }); +const OPENING_PROCESSES = 6; + +// Opens the file's storage from processes of their own, each loaded first and then let go at +// once, so the first statements of every connection meet on the file: processes, as instances +// on several machines are, and as libSQL's native code wants, since a worker thread that ends +// under it can bring the whole process down. Resolves what each said. +const openTogether = async (path: string): Promise => { + const children = Array.from({ length: OPENING_PROCESSES }, () => + spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { sqlite } from "./index.ts"; + process.stdout.write("loaded\\n"); + process.stdin.once("data", async () => { + try { + await (await sqlite({ path: ${JSON.stringify(path)} }).createStorage()).close(); + process.stdout.write("opened\\n"); + } catch (error) { + process.stdout.write(String(error) + "\\n"); + } + process.stdin.destroy(); + });`, + ], + { cwd: import.meta.dirname, stdio: ["pipe", "pipe", "inherit"] }, + ), + ); + const said = children.map((child) => { + const lines: string[] = []; + let loaded: () => void = () => undefined; + const ready = new Promise((resolve) => { + loaded = resolve; + }); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + for (const line of chunk.split("\n").filter(Boolean)) { + if (line === "loaded") loaded(); + else lines.push(line); + } + }); + const exited = new Promise((resolve) => { + child.once("exit", (code) => resolve(lines.join(" ") || `exited with ${code}`)); + }); + return { ready, exited }; + }); + await Promise.all(said.map(({ ready }) => ready)); + for (const child of children) child.stdin.write("go\n"); + return Promise.all(said.map(({ exited }) => exited)); +}; + describe("sqlite adapter on a file", () => { let directory: string; let counter = 0; @@ -171,6 +221,47 @@ describe("sqlite adapter on a file", () => { locking: "single-writer", }); + it("closes the file only once what runs on it has settled, a transaction in flight included", async () => { + const path = freshPath(); + const adapter = sqlite({ path }); + const storage = await adapter.createStorage(); + const ledger = await adapter.createForgetLedger({}); + // Its table made, so its read below is one statement, queued behind the transaction. + await ledger.read({ limit: 1 }); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + let entered: () => void = () => undefined; + const inside = new Promise((resolve) => { + entered = resolve; + }); + const writing = storage.transact(async (tx) => { + entered(); + await held; + await tx.eventStore.append({ + aggregateType: "order", + aggregateId: "1", + expectedVersion: 0, + events: [pendingEvent({ aggregateId: "1", version: 1 })], + }); + }); + await inside; + const reading = ledger.read({ limit: 1 }); + let closed = false; + const closing = Promise.all([storage.close(), ledger.close?.()]).then(() => { + closed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(closed).toBe(false); + release(); + await writing; + expect(await reading).toEqual([]); + await closing; + const reopened = await openFile(path); + expect(await reopened.eventStore.lastPosition()).toBe(1); + }); + it("lets its own statements wait for its own write transaction, without blocking the process", async () => { const storage = await openFile(); let release: () => void = () => undefined; @@ -236,22 +327,7 @@ describe("sqlite adapter on a file", () => { await readModel.close(); }); - it("opens a file not in WAL yet, such as one just restored, from several connections at once", async () => { - const connections = 6; - // Each worker loads the adapter, waits at the gate with the others, then opens its storage: - // the first statements of every connection meet on the file at the same moment. - const worker = ` - const { workerData, parentPort } = require("node:worker_threads"); - import(workerData.module).then(async ({ sqlite }) => { - parentPort.postMessage("loaded"); - Atomics.wait(new Int32Array(workerData.gate), 0, 0); - try { - await (await sqlite({ path: workerData.path }).createStorage()).close(); - parentPort.postMessage("opened"); - } catch (error) { - parentPort.postMessage(String(error)); - } - });`; + it("opens a file not in WAL yet, such as one just restored, from several processes at once", async () => { for (let round = 0; round < 3; round += 1) { const sourcePath = freshPath(); const source = await openFile(sourcePath); @@ -269,86 +345,14 @@ describe("sqlite adapter on a file", () => { expect((await restored.execute("PRAGMA journal_mode")).rows[0]?.[0]).toBe("delete"); restored.close(); - const gate = new SharedArrayBuffer(4); - const workers = Array.from( - { length: connections }, - () => - new Worker(worker, { - eval: true, - workerData: { module: new URL("./index.ts", import.meta.url).href, path, gate }, - }), - ); - const loaded = workers.map( - (each) => - new Promise((resolve) => { - each.once("message", () => resolve()); - each.once("error", () => resolve()); - }), - ); - const outcomes = workers.map( - (each) => - new Promise((resolve) => { - each.on("message", (message: string) => { - if (message !== "loaded") resolve(message); - }); - each.on("error", (error) => resolve(String(error))); - }), - ); - await Promise.all(loaded); - Atomics.store(new Int32Array(gate), 0, 1); - Atomics.notify(new Int32Array(gate), 0); - expect(await Promise.all(outcomes)).toEqual(Array(connections).fill("opened")); - await Promise.all(workers.map((each) => each.terminate())); + expect(await openTogether(path)).toEqual(Array(OPENING_PROCESSES).fill("opened")); expect(await (await openFile(path)).eventStore.lastPosition()).toBe(1); } }); it("migrates a new file once when processes open it at the same moment", async () => { - const connections = 6; const path = freshPath(); - const worker = ` - const { workerData, parentPort } = require("node:worker_threads"); - import(workerData.module).then(async ({ sqlite }) => { - parentPort.postMessage("loaded"); - Atomics.wait(new Int32Array(workerData.gate), 0, 0); - try { - await (await sqlite({ path: workerData.path }).createStorage()).close(); - parentPort.postMessage("opened"); - } catch (error) { - parentPort.postMessage(String(error)); - } - });`; - const gate = new SharedArrayBuffer(4); - const workers = Array.from( - { length: connections }, - () => - new Worker(worker, { - eval: true, - workerData: { module: new URL("./index.ts", import.meta.url).href, path, gate }, - }), - ); - const outcomes = workers.map( - (each) => - new Promise((resolve) => { - each.on("message", (message: string) => { - if (message !== "loaded") resolve(message); - }); - each.on("error", (error) => resolve(String(error))); - }), - ); - await Promise.all( - workers.map( - (each) => - new Promise((resolve) => { - each.once("message", () => resolve()); - each.once("error", () => resolve()); - }), - ), - ); - Atomics.store(new Int32Array(gate), 0, 1); - Atomics.notify(new Int32Array(gate), 0); - expect(await Promise.all(outcomes)).toEqual(Array(connections).fill("opened")); - await Promise.all(workers.map((each) => each.terminate())); + expect(await openTogether(path)).toEqual(Array(OPENING_PROCESSES).fill("opened")); const client = createClient({ url: `file:${path}` }); const { rows } = await client.execute( 'SELECT "scope", "version" FROM "bounda_migrations" ORDER BY "scope", "version"', @@ -558,6 +562,10 @@ describe("sqlite adapter connection sharing", () => { const stillOpen = await adapter.createStorage(); expect(await stillOpen.eventStore.lastPosition()).toBe(1); await stillOpen.close(); + const client = readModel.client.raw as Client; + expect(client.closed).toBe(false); + await readModel.close(); + expect(client.closed).toBe(true); await readModel.close(); const fresh = await adapter.createStorage(); diff --git a/packages/sqlite/vitest.config.ts b/packages/sqlite/vitest.config.ts index 05541e08..88ce18a3 100644 --- a/packages/sqlite/vitest.config.ts +++ b/packages/sqlite/vitest.config.ts @@ -1,8 +1,10 @@ +import { join } from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ test: { name: "sqlite", + setupFiles: [join(import.meta.dirname, "../../vitest.trace.ts")], include: ["src/**/*.test.ts"], }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c9188374..29a95011 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -462,6 +462,9 @@ importers: specifier: 'catalog:' version: 15.0.0 devDependencies: + '@bounda-dev/cloudflare': + specifier: workspace:* + version: link:../cloudflare '@bounda-dev/core': specifier: workspace:* version: link:../core @@ -483,6 +486,9 @@ importers: vitest: specifier: 'catalog:' version: 5.1.0(@opentelemetry/api@1.10.0)(@types/node@26.6.6)(@vitest/coverage-v8@5.1.0)(vite@8.3.6(@types/node@26.6.6)(esbuild@0.28.2)(yaml@2.9.1)) + wrangler: + specifier: 'catalog:' + version: 4.157.0(@cloudflare/workers-types@5.20261103.1)(@types/node@26.6.6) packages/cloudflare: devDependencies: diff --git a/vitest.trace.ts b/vitest.trace.ts new file mode 100644 index 00000000..94038c3c --- /dev/null +++ b/vitest.trace.ts @@ -0,0 +1,11 @@ +import { writeSync } from "node:fs"; +import { beforeEach } from "vitest"; + +// With BOUNDA_TEST_TRACE set, names each test as it starts, straight on the process's stderr: a +// test whose process dies in native code leaves its name behind, which the reporters, fed by the +// worker over RPC, lose with it. +if (process.env.BOUNDA_TEST_TRACE) { + beforeEach(({ task }) => { + writeSync(2, `[test] ${task.file.name} > ${task.name}\n`); + }); +}