From d0245adb2875a9c827d1ced1c7ba3be65cccda75 Mon Sep 17 00:00:00 2001 From: Dawid Kisielwski <91346346+dawidreedsy@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:01:32 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20Add=20`retry`=20middleware=20ac?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a commit loses the race to another op, `SubmitRequest.retry()` re-runs `submit()` on the same request. That re-fetches the snapshot and resets fixups, but keeps any properties middleware set on the request during the failed attempt, so they can describe a stale snapshot. The new `retry` action is triggered before each retry, giving middleware a place to reset that state. An error passed to `next` rejects the op. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/middleware/actions.md | 8 +++ docs/middleware/op-submission.md | 20 +++++++ lib/backend.js | 2 + lib/submit-request.js | 6 +- test/middleware.js | 98 ++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) diff --git a/docs/middleware/actions.md b/docs/middleware/actions.md index 3d34dc6a..8f59eb6d 100644 --- a/docs/middleware/actions.md +++ b/docs/middleware/actions.md @@ -223,6 +223,14 @@ An operation was applied to a snapshot, and is about to be committed to the data This action has the same additional `context` properties as [`'submit'`](#submit). +## `'retry'` + +An operation lost the race to commit, because another op was committed to the same document first. The op is about to be retried: the snapshot will be fetched again, and the op transformed against the newer ops before passing through [`'apply'`](#apply) and [`'commit'`](#commit) again. [`'submit'`](#submit) is not triggered again. + +This is the place to reset any state that middleware stored on the request during the failed attempt. + +This action has the same additional `context` properties as [`'submit'`](#submit). + ## `'afterWrite'` An operation and its updated snapshot were successfully written to the database. diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index 10f6e2b0..8ca22e76 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -18,6 +18,7 @@ When an op is submitted, it will pass through a number of middleware hooks as it - [**Submit**](#submit) -- an op has been received by the server - [**Apply**](#apply) -- an op is about to be applied to the snapshot - [**Commit**](#commit) -- an op and its updated snapshot are about to be committed to the database + - [**Retry**](#retry) -- an op lost the race to commit, and is about to be submitted again - [**After write**](#after-write) -- an op and its updated snapshot have successfully been committed to the database - [**Submit request end**](#submit-request-end) -- an op submission has finished (this is an _event_, **not** a middleware hook) @@ -88,6 +89,25 @@ backend.use('commit', (context, next) => { }) ``` +### Retry + +The [`retry`]({{ site.baseurl }}{% link middleware/actions.md %}#retry) hook is triggered when another client wins the race to commit, before the op is submitted again. The op then passes through the [apply](#apply) and [commit](#commit) hooks again, against the newer snapshot. The [submit](#submit) hook is not triggered again. + +The retry reuses the same `context`, so any properties set on it during the failed attempt are still there. This hook is the place to reset them. + +```js +backend.use('apply', (context, next) => { + context.snapshotBeforeApply ??= structuredClone(context.snapshot) + next() +}) + +backend.use('retry', (context, next) => { + // Capture the newer snapshot on the next attempt + delete context.snapshotBeforeApply + next() +}) +``` + ### After write The [`afterWrite`]({{ site.baseurl }}{% link middleware/actions.md %}#afterwrite) hook is triggered after the op and updated snapshot have been successfully written to the database. This is the earliest that you know the op and snapshot are canonical. diff --git a/lib/backend.js b/lib/backend.js index 17a409df..ca105edf 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -89,6 +89,8 @@ Backend.prototype.MIDDLEWARE_ACTIONS = { receivePresence: 'receivePresence', // About to send presence information to a client sendPresence: 'sendPresence', + // An operation lost the race to commit and is about to be retried against a newer snapshot + retry: 'retry', // An operation is about to be submitted to the database submit: 'submit' }; diff --git a/lib/submit-request.js b/lib/submit-request.js index d2cac8d9..e9145f32 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -262,7 +262,11 @@ SubmitRequest.prototype.retry = function(callback) { } this._resetFixups(); this.backend.emit('timing', 'submit.retry', Date.now() - this.start, this); - this.submit(callback); + var request = this; + this.backend.trigger(this.backend.MIDDLEWARE_ACTIONS.retry, this.agent, this, function(err) { + if (err) return callback(err); + request.submit(callback); + }); }; // The fixups must be undone before the op is transformed forward again diff --git a/test/middleware.js b/test/middleware.js index 77cb80a5..38a19dbe 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -423,6 +423,104 @@ describe('middleware', function() { }); }); + describe('retry', function() { + var backend; + var doc; + var remoteDoc; + + beforeEach(function(done) { + backend = this.backend; + doc = backend.connect().get('dogs', 'fido'); + remoteDoc = backend.connect().get('dogs', 'fido'); + doc.create({age: 3}, function(error) { + if (error) return done(error); + remoteDoc.fetch(done); + }); + }); + + // Holds both ops in 'commit' until they've been applied to the same snapshot, + // then commits the remote op first, so the local op loses the race and retries + function forceRetry(callback) { + var commitDoc; + var commitRemoteDoc; + backend.use('commit', function(request, next) { + if (commitDoc && commitRemoteDoc) return next(); + if (request.op.src === doc.connection.id) commitDoc = next; + else commitRemoteDoc = next; + if (commitDoc && commitRemoteDoc) commitRemoteDoc(); + }); + + remoteDoc.submitOp([{p: ['age'], na: 7}], function(error) { + if (error) return callback(error); + commitDoc(); + }); + doc.submitOp([{p: ['age'], na: 2}], callback); + } + + it('is triggered between a lost commit and the next attempt', function(done) { + var actions = []; + ['submit', 'apply', 'commit', 'retry', 'afterWrite'].forEach(function(action) { + backend.use(action, function(request, next) { + if (request.op.src === doc.connection.id) actions.push(action); + next(); + }); + }); + + forceRetry(function(error) { + if (error) return done(error); + expect(actions).to.eql(['submit', 'apply', 'commit', 'retry', 'apply', 'commit', 'afterWrite']); + expect(doc.data).to.eql({age: 12}); + done(); + }); + }); + + it('has the number of retries so far', function(done) { + var retries = []; + backend.use('retry', function(request, next) { + retries.push(request.retries); + next(); + }); + + forceRetry(function(error) { + if (error) return done(error); + expect(retries).to.eql([1]); + done(); + }); + }); + + it('is not triggered if the op commits first time', function(done) { + var retry = sinon.spy(function(_request, next) { + next(); + }); + backend.use('retry', retry); + + doc.submitOp([{p: ['age'], na: 2}], function(error) { + if (error) return done(error); + expect(retry).not.to.have.been.called; + done(); + }); + }); + + it('is not triggered once the op has used up its retries', function(done) { + backend.maxSubmitRetries = 0; + var retry = sinon.spy(function(_request, next) { + next(); + }); + backend.use('retry', retry); + + forceRetry(function(error) { + expect(error.code).to.equal(ERROR_CODE.ERR_MAX_SUBMIT_RETRIES_EXCEEDED); + expect(retry).not.to.have.been.called; + done(); + }); + }); + + it('rejects the op with an error passed to next', function(done) { + backend.use('retry', passError); + forceRetry(getErrorTest(done)); + }); + }); + describe('$fixup', function() { var connection; var backend; From 2850978de2648bb0ff207dbf717de18b6e76d116 Mon Sep 17 00:00:00 2001 From: Dawid Kisielwski <91346346+dawidreedsy@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:07:49 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=9D=20Clarify=20`retry`=20hook=20d?= =?UTF-8?q?ocs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Explain why the example captures the snapshot first-write-wins - Note that snapshot and channels are still the failed attempt's in the hook - Say "retried" rather than "submitted again" - Point the 'apply' re-run warning at the retry hook Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/middleware/actions.md | 2 ++ docs/middleware/op-submission.md | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/middleware/actions.md b/docs/middleware/actions.md index 8f59eb6d..a928de2e 100644 --- a/docs/middleware/actions.md +++ b/docs/middleware/actions.md @@ -229,6 +229,8 @@ An operation lost the race to commit, because another op was committed to the sa This is the place to reset any state that middleware stored on the request during the failed attempt. +The `snapshot` and `channels` are still those of the failed attempt, so the `snapshot` already has the op applied. The snapshot is fetched again and the channels reset after this action, before [`'apply'`](#apply). + This action has the same additional `context` properties as [`'submit'`](#submit). ## `'afterWrite'` diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index 8ca22e76..b6ef7315 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -18,7 +18,7 @@ When an op is submitted, it will pass through a number of middleware hooks as it - [**Submit**](#submit) -- an op has been received by the server - [**Apply**](#apply) -- an op is about to be applied to the snapshot - [**Commit**](#commit) -- an op and its updated snapshot are about to be committed to the database - - [**Retry**](#retry) -- an op lost the race to commit, and is about to be submitted again + - [**Retry**](#retry) -- an op lost the race to commit, and is about to be retried against a newer snapshot - [**After write**](#after-write) -- an op and its updated snapshot have successfully been committed to the database - [**Submit request end**](#submit-request-end) -- an op submission has finished (this is an _event_, **not** a middleware hook) @@ -67,7 +67,7 @@ backend.use('apply', (context, next) => { ``` {: .warn :} -The `'apply'` hook may be triggered more than once for a single submission. If another client wins the race to commit, the op is transformed over the winning op and applied again to the newer snapshot, re-triggering the hook. Be careful, therefore, with any side effects that assume the hook only runs once per op. +The `'apply'` hook may be triggered more than once for a single submission. If another client wins the race to commit, the op is transformed over the winning op and applied again to the newer snapshot, re-triggering the hook. Be careful, therefore, with any side effects that assume the hook only runs once per op. Use the [retry](#retry) hook to reset any state stored on the `context` during the failed attempt. ### Commit @@ -91,12 +91,15 @@ backend.use('commit', (context, next) => { ### Retry -The [`retry`]({{ site.baseurl }}{% link middleware/actions.md %}#retry) hook is triggered when another client wins the race to commit, before the op is submitted again. The op then passes through the [apply](#apply) and [commit](#commit) hooks again, against the newer snapshot. The [submit](#submit) hook is not triggered again. +The [`retry`]({{ site.baseurl }}{% link middleware/actions.md %}#retry) hook is triggered when another client wins the race to commit, before the op is retried. The op then passes through the [apply](#apply) and [commit](#commit) hooks again, against the newer snapshot. The [submit](#submit) hook is not triggered again. The retry reuses the same `context`, so any properties set on it during the failed attempt are still there. This hook is the place to reset them. +In the example below, several `'apply'` middlewares share one lazily-taken snapshot clone, so only the first one that needs it pays for the copy. Without a reset, the retry would keep the failed attempt's clone. A plain `=` assignment, as in [Comparing old snapshot version with new version](#comparing-old-snapshot-version-with-new-version), re-captures on every attempt and doesn't need this hook. + ```js backend.use('apply', (context, next) => { + // Any of several 'apply' middlewares may take the clone first context.snapshotBeforeApply ??= structuredClone(context.snapshot) next() }) From de10c67d90f0eab37e23df33f096663898a6bc25 Mon Sep 17 00:00:00 2001 From: Dawid Kisielwski <91346346+dawidreedsy@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:07:49 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=85=20Test=20that=20`retry`=20resets?= =?UTF-8?q?=20state=20before=20the=20next=20apply?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) --- test/middleware.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/middleware.js b/test/middleware.js index 38a19dbe..d39e7e45 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -488,6 +488,28 @@ describe('middleware', function() { }); }); + it('lets middleware reset state before the next attempt', function(done) { + var snapshotsBeforeApply = []; + backend.use('apply', function(request, next) { + if (request.op.src !== doc.connection.id) return next(); + if (!request.snapshotBeforeApply) { + request.snapshotBeforeApply = JSON.parse(JSON.stringify(request.snapshot.data)); + } + snapshotsBeforeApply.push(request.snapshotBeforeApply); + next(); + }); + backend.use('retry', function(request, next) { + delete request.snapshotBeforeApply; + next(); + }); + + forceRetry(function(error) { + if (error) return done(error); + expect(snapshotsBeforeApply).to.eql([{age: 3}, {age: 10}]); + done(); + }); + }); + it('is not triggered if the op commits first time', function(done) { var retry = sinon.spy(function(_request, next) { next();