From fef291c0c011fd2cb26b11e5bf00d3b9df6eab87 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 20:44:16 +0300 Subject: [PATCH 01/83] refactor: share the backoff, thrown errors, fingerprint and path checks The queue to come computes the delay of an attempt and classifies what a transport throws as ./retry does, and fingerprints a message as ./idempotency does, so these move out of the two decorators into src/internal/, with no change of behavior: computeBackoff, retryAfterOf and isReceiptError (backoff.js); isProgrammerError and receiptOfThrown (thrown.js); fingerprintOf (fingerprint.js, moved from ./idempotency); and isWithin (path.js), which the file loader of ./templates shares with fromPath. The existing suites are the gate. Co-Authored-By: Claude Opus 5.5 --- bench/bench.js | 2 +- src/attachments/path.js | 9 +- src/idempotency/idempotency-transport.js | 2 +- src/internal/backoff.js | 58 ++++++++++++ src/{idempotency => internal}/fingerprint.js | 0 src/internal/path.js | 22 +++++ src/internal/thrown.js | 64 +++++++++++++ src/retry/retry-transport.js | 95 ++------------------ 8 files changed, 155 insertions(+), 97 deletions(-) create mode 100644 src/internal/backoff.js rename src/{idempotency => internal}/fingerprint.js (100%) create mode 100644 src/internal/path.js create mode 100644 src/internal/thrown.js diff --git a/bench/bench.js b/bench/bench.js index 4a1a33d..1c34894 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -30,7 +30,7 @@ const { const { composeMessage } = require('../src/mime/index.js'); const { inlineDataUrls } = require('../src/attachments/index.js'); const { htmlToText } = require('../src/render/index.js'); -const { fingerprintOf } = require('../src/idempotency/fingerprint.js'); +const { fingerprintOf } = require('../src/internal/fingerprint.js'); const { gatedSendMany } = require('../src/internal/gate.js'); const { createMailtrapWebhookVerifier, diff --git a/src/attachments/path.js b/src/attachments/path.js index 9d18fac..3f9e007 100644 --- a/src/attachments/path.js +++ b/src/attachments/path.js @@ -13,17 +13,10 @@ const fsp = require('node:fs/promises'); const path = require('node:path'); +const { isWithin } = require('../internal/path.js'); const { checkMaxBytes, createAttachment } = require('./attachment.js'); const { typeOfFilename } = require('./mime-types.js'); -// Whether a resolved path is the directory or under it; a name such as -// `..data` under it is, `../data` is not. -const isWithin = (file, directory) => { - const relative = path.relative(directory, file); - if (path.isAbsolute(relative)) return false; - return relative !== '..' && !relative.startsWith(`..${path.sep}`); -}; - const tooLong = (filePath, maxBytes) => new RangeError(`${JSON.stringify(filePath)} holds more than ${maxBytes} bytes.`); diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index 5a5d8a3..5660af3 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -25,7 +25,7 @@ const { createLog } = require('../internal/log.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createIdempotencyConfig } = require('./config.js'); -const { fingerprintOf } = require('./fingerprint.js'); +const { fingerprintOf } = require('../internal/fingerprint.js'); const STORE_FAILED = 'MAILER_IDEMPOTENCY_STORE_FAILED'; diff --git a/src/internal/backoff.js b/src/internal/backoff.js new file mode 100644 index 0000000..7a46ca6 --- /dev/null +++ b/src/internal/backoff.js @@ -0,0 +1,58 @@ +// Not ported: the delay between two attempts, shared by ./retry, which waits +// it in the process, and ./queue, which hands it to the queue to schedule. + +/** + * Whether a value is a `ReceiptError`: a message, a code, a category and + * whether it is retryable. + * + * @internal + */ +const isReceiptError = (value) => + typeof value === 'object' && + value !== null && + typeof value.message === 'string' && + typeof value.code === 'string' && + typeof value.category === 'string' && + typeof value.retryable === 'boolean'; + +/** + * The delay a failure asks to be waited before the next attempt: the + * `retryAfterMilliseconds` of a receipt error, or the first one among the + * errors of a failed receipt. + * + * @param failure A failed receipt, a receipt error, or nothing. + * @returns The delay in milliseconds, or `undefined` when none is asked. + * @internal + */ +const retryAfterOf = (failure) => { + if (failure == null) return undefined; + if (isReceiptError(failure)) return failure.retryAfterMilliseconds; + if (failure.errors == null) return undefined; + for (const error of failure.errors) { + if (error.retryAfterMilliseconds != null) return error.retryAfterMilliseconds; + } + return undefined; +}; + +/** + * The exponential delay before the attempt after `attempt`: the base delay + * multiplied by the factor once per attempt made before it, at most the + * longest delay, and with full jitter a random part of it. + * + * @param attempt The attempt that failed, from 1. + * @param backoff `baseDelayMilliseconds`, `maxDelayMilliseconds` and `factor`. + * @param jitter `'full'`, or `'none'` or `false` for none. + * @param random A function giving a number in [0, 1). + * @returns The delay in milliseconds. + * @internal + */ +const computeBackoff = (attempt, backoff, jitter, random) => { + const computedDelay = Math.min( + backoff.baseDelayMilliseconds * Math.pow(backoff.factor, attempt - 1), + backoff.maxDelayMilliseconds, + ); + if (jitter === false || jitter === 'none') return computedDelay; + return Math.floor(random() * computedDelay); +}; + +module.exports = { computeBackoff, isReceiptError, retryAfterOf }; diff --git a/src/idempotency/fingerprint.js b/src/internal/fingerprint.js similarity index 100% rename from src/idempotency/fingerprint.js rename to src/internal/fingerprint.js diff --git a/src/internal/path.js b/src/internal/path.js new file mode 100644 index 0000000..fe3a722 --- /dev/null +++ b/src/internal/path.js @@ -0,0 +1,22 @@ +// Not ported: whether a file is inside a directory, for ./attachments and the +// file loader of ./templates, which read nothing outside the root they are +// given. + +const path = require('node:path'); + +/** + * Whether a resolved path is the directory or under it; a name such as + * `..data` under it is, `../data` is not. + * + * @param file The resolved path of the file. + * @param directory The resolved path of the directory. + * @returns `true` when the file is within the directory. + * @internal + */ +const isWithin = (file, directory) => { + const relative = path.relative(directory, file); + if (path.isAbsolute(relative)) return false; + return relative !== '..' && !relative.startsWith(`..${path.sep}`); +}; + +module.exports = { isWithin }; diff --git a/src/internal/thrown.js b/src/internal/thrown.js new file mode 100644 index 0000000..f732b7d --- /dev/null +++ b/src/internal/thrown.js @@ -0,0 +1,64 @@ +// Not ported: what a decorator makes of an error a transport threw, rather +// than reported in a receipt: a bug in code, rethrown, or a failure of +// delivery, turned into a failed receipt. + +const { + classifyReceiptError, + createFailedReceipt, + createReceiptError, +} = require('../core/receipt.js'); +const { isReceiptError } = require('./backoff.js'); +const { messageOf } = require('./decorator.js'); + +// The built-in errors of a bug in code, such as reading a property of +// undefined, as opposed to a failure of delivery. A network failure is also +// a TypeError under fetch ("fetch failed"), but its causes classify it. +const PROGRAMMER_ERRORS = [TypeError, RangeError, ReferenceError, SyntaxError]; + +/** + * Whether an error is a bug in code rather than a failure of delivery: a + * built-in error of a bug that `classifyReceiptError` does not recognize. + * + * @internal + */ +const isProgrammerError = (error) => { + for (const type of PROGRAMMER_ERRORS) { + if (error instanceof type) return classifyReceiptError(error).category === 'unknown'; + } + return false; +}; + +/** + * The failed receipt of an error a transport threw: the receipt error it is, + * or one classified from it, an `AbortError` a retryable timeout. + * + * @param error What was thrown. + * @param providerId The provider the receipt names when the error names none. + * @param attempts The attempts made, if counted. + * @returns The failed receipt. + * @internal + */ +const receiptOfThrown = (error, providerId, attempts) => { + if (isReceiptError(error)) { + const receiptError = error; + return createFailedReceipt(receiptError, { + provider: receiptError.provider ?? providerId, + attempts, + }); + } + const classification = + error instanceof Error && error.name === 'AbortError' + ? { category: 'timeout', code: 'timeout', retryable: true } + : classifyReceiptError(error); + return createFailedReceipt( + createReceiptError(messageOf(error), { + provider: providerId, + category: classification.category, + code: classification.code, + retryable: classification.retryable, + }), + { provider: providerId, attempts }, + ); +}; + +module.exports = { isProgrammerError, receiptOfThrown }; diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index fa59601..2957fe9 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -1,39 +1,20 @@ // Ported from upyo v0.7.0 (packages/retry/src/retry-transport.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -const { - classifyReceiptError, - createFailedReceipt, - createReceiptError, -} = require('../core/receipt.js'); +const { classifyReceiptError } = require('../core/receipt.js'); const { combineSignals } = require('../core/abort-signal.js'); const { pinMessageId } = require('../core/message-id.js'); const { createLog } = require('../internal/log.js'); const { createRetryConfig } = require('./config.js'); const { abortReason } = require('../internal/sleep.js'); -const { - disposeOf, - messageOf, - refuseUnlessRaw, - verifyWrapped, -} = require('../internal/decorator.js'); +const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); const { iteratorOf } = require('../internal/iterate.js'); +const { computeBackoff, isReceiptError, retryAfterOf } = require('../internal/backoff.js'); +const { isProgrammerError, receiptOfThrown } = require('../internal/thrown.js'); /** * Transport decorator that retries transient delivery failures. */ -// The built-in errors of a bug in code, such as reading a property of -// undefined, as opposed to a failure of delivery. A network failure is also -// a TypeError under fetch ("fetch failed"), but its causes classify it. -const PROGRAMMER_ERRORS = [TypeError, RangeError, ReferenceError, SyntaxError]; - -const isProgrammerError = (error) => { - for (const type of PROGRAMMER_ERRORS) { - if (error instanceof type) return classifyReceiptError(error).category === 'unknown'; - } - return false; -}; - class RetryTransport { id; @@ -292,7 +273,7 @@ class RetryTransport { // again sooner, where it would most likely refuse again: the failure goes // back with its delay, for the caller or a queue to schedule. isRetryAfterBeyondMaxDelay(failure) { - const retryAfterMilliseconds = getRetryAfterMilliseconds(failure); + const retryAfterMilliseconds = retryAfterOf(failure); return ( Number.isFinite(retryAfterMilliseconds) && retryAfterMilliseconds > this.config.backoff.maxDelayMilliseconds @@ -300,7 +281,7 @@ class RetryTransport { } calculateDelay(attempt, failure) { - const retryAfterMilliseconds = getRetryAfterMilliseconds(failure); + const retryAfterMilliseconds = retryAfterOf(failure); const cappedRetryAfter = retryAfterMilliseconds == null || !Number.isFinite(retryAfterMilliseconds) || @@ -309,18 +290,7 @@ class RetryTransport { : Math.min(retryAfterMilliseconds, this.config.backoff.maxDelayMilliseconds); if (cappedRetryAfter != null) return cappedRetryAfter; - - const computedDelay = Math.min( - this.config.backoff.baseDelayMilliseconds * Math.pow(this.config.backoff.factor, attempt - 1), - this.config.backoff.maxDelayMilliseconds, - ); - - if (this.config.jitter === false || this.config.jitter === 'none') { - return computedDelay; - } - - const random = this.config.random; - return Math.floor(random() * computedDelay); + return computeBackoff(attempt, this.config.backoff, this.config.jitter, this.config.random); } waitBeforeRetry(context, signal, onRetry) { @@ -355,35 +325,7 @@ class RetryTransport { } createThrownFailure(error, attempts) { - const receiptError = toReceiptError(error); - if (receiptError != null) { - return createFailedReceipt(receiptError, { - provider: receiptError.provider ?? this.id, - attempts, - }); - } - - const message = messageOf(error); - const classification = - error instanceof Error && error.name === 'AbortError' - ? { - category: 'timeout', - code: 'timeout', - retryable: true, - } - : classifyReceiptError(error); - return createFailedReceipt( - createReceiptError(message, { - provider: this.id, - category: classification.category, - code: classification.code, - retryable: classification.retryable, - }), - { - provider: this.id, - attempts, - }, - ); + return receiptOfThrown(error, this.id, attempts); } } @@ -399,16 +341,6 @@ const createRetryTransport = (baseTransport, config = {}) => { return new RetryTransport(baseTransport, config); }; -const getRetryAfterMilliseconds = (failure) => { - if (failure == null) return undefined; - if (isReceiptError(failure)) return failure.retryAfterMilliseconds; - if (failure.errors == null) return undefined; - for (const error of failure.errors) { - if (error.retryAfterMilliseconds != null) return error.retryAfterMilliseconds; - } - return undefined; -}; - /** * The state of one sendMany() of the retry decorator. Messages are read and * sent up to `maxConcurrent` at a time, and a slot freed by a later message @@ -593,15 +525,4 @@ const toReceiptError = (value) => { return isReceiptError(value) ? value : undefined; }; -const isReceiptError = (value) => { - return ( - typeof value === 'object' && - value != null && - typeof value.message === 'string' && - typeof value.code === 'string' && - typeof value.category === 'string' && - typeof value.retryable === 'boolean' - ); -}; - module.exports = { RetryTransport, createRetryTransport }; From fbffac0fac4f52c651da50ed26d5d02f9a2d7fef Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 20:45:26 +0300 Subject: [PATCH 02/83] feat: a mail queue with a BullMQ driver, templates over nine engines and an audit of states The reliability layer of the roadmap, on the transport of 1.1, all additive. ./queue: createMailQueue({ driver, audit, logger }) enqueues a message as a JSON job, a template with its context or content rendered already, checked when it is enqueued as createMessage checks a message; its id is a UUID of namespace, tenant and idempotencyKey, so the same message enqueued again is one job and another under the key is refused (queue.key_reused), and its Message-ID and Content-IDs are fixed for every attempt. createMailJobHandler, the worker for any driver, renders, resolves attachments by reference, sends through the transport the application composed, and makes of the receipt an outcome: a permanent failure, a missing template or a bug fails at once (a queue that retries everything sent them three times), a temporary one is retried never sooner than Retry-After, an operator's failure after five minutes, and a message of unknown delivery is never sent blindly again. The contracts (QueueDriver, QueueSource, QueueOutcome, MessageState, AuditEvent, AuditSink) are types of the root; createPollingDriver makes a driver of a store that lends its jobs, the outbox to come, and createMemoryQueue is one in memory for tests. ./bullmq: createBullMQDriver over the Queue and Worker the application passes in: jobs under their id, priorities from 1 (0 runs before every prioritized job), retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as UnrecoverableError and releases that spend no attempt, all by the names BullMQ reads, so it is imported nowhere. Checked against BullMQ 6.3.16 and, in the opt-in tests/bullmq suite, on BullMQ and Valkey. ./templates: createTemplateRenderer, a Renderer of templates by name over EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache and React Email passed in: names never paths, variants by locale and part, the subject in front matter, a layout, includes read through the loader, each template compiled once, CSS inlined without fetching a stylesheet, and the text derived. The cases where @nestjs-modules/mailer went wrong have tests: a callback called twice, an async EJS that hung, a broken MJML, global Handlebars helpers and partials read on every send, Pug options in the locals, absolute template paths. ./audit: sinks of the states of a message (memory, logger, combined), stateOf, and auditEventOfDeliveryEvent for the events of ./webhooks. ./render gains RenderOutput.subject and renderFields. The engines, bullmq and ioredis are devDependencies for the tests; src/ requires none of them. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 42 + CLAUDE.md | 64 +- README.md | 59 +- docs/.vitepress/config.mts | 9 + docs/api/exports.md | 10 +- docs/comparison.json | 21 +- docs/guide/audit.md | 83 + docs/guide/comparison.md | 25 +- docs/guide/getting-started.md | 4 +- docs/guide/queue.md | 157 ++ docs/guide/why.md | 10 +- docs/index.md | 17 +- docs/messages/render.md | 20 + docs/messages/templates.md | 250 ++ docs/public/llms.txt | 12 +- docs/transports/bullmq.md | 87 + index.d.ts | 216 ++ package.json | 33 +- pnpm-lock.yaml | 2611 ++++++++++++++++++++- scripts/size-budgets.json | 6 +- src/audit/combine.js | 32 + src/audit/delivery-event.js | 61 + src/audit/index.d.ts | 99 + src/audit/index.js | 18 + src/audit/logger-sink.js | 47 + src/audit/memory-sink.js | 45 + src/audit/states.js | 48 + src/bullmq/backoff.js | 91 + src/bullmq/bullmq-driver.js | 252 ++ src/bullmq/index.d.ts | 204 ++ src/bullmq/index.js | 13 + src/bullmq/processor.js | 109 + src/internal/polling-driver.js | 249 ++ src/queue/errors.js | 21 + src/queue/handler.js | 491 ++++ src/queue/hash.js | 53 + src/queue/index.d.ts | 420 ++++ src/queue/index.js | 24 + src/queue/job.js | 275 +++ src/queue/json.js | 67 + src/queue/mail-queue.js | 147 ++ src/queue/memory-queue.js | 286 +++ src/queue/outcome.js | 114 + src/queue/parse.js | 108 + src/render/index.d.ts | 34 + src/render/index.js | 4 +- src/render/renderer.js | 47 +- src/templates/cache.js | 52 + src/templates/css.js | 38 + src/templates/ejs.js | 48 + src/templates/errors.js | 64 + src/templates/eta.js | 56 + src/templates/front-matter.js | 65 + src/templates/handlebars.js | 117 + src/templates/includes.js | 171 ++ src/templates/index.d.ts | 556 +++++ src/templates/index.js | 34 + src/templates/liquid.js | 110 + src/templates/loader.js | 163 ++ src/templates/mjml.js | 97 + src/templates/mustache.js | 70 + src/templates/nunjucks.js | 103 + src/templates/options.js | 79 + src/templates/pug.js | 67 + src/templates/react-email.js | 61 + src/templates/renderer.js | 445 ++++ tests/bullmq/bullmq-driver.bullmq.test.js | 225 ++ tests/bullmq/docker-compose.yml | 17 + tests/helpers/bullmq/fake-bullmq.js | 87 + tests/helpers/queue/clock.js | 41 + tests/helpers/templates/fake-engine.js | 50 + tests/types/audit.test-d.ts | 31 + tests/types/bullmq.test-d.ts | 43 + tests/types/index.test-d.ts | 29 + tests/types/queue.test-d.ts | 133 ++ tests/types/render.test-d.ts | 8 + tests/types/templates.test-d.ts | 125 + tests/unit/audit.test.js | 233 ++ tests/unit/bullmq.test.js | 403 ++++ tests/unit/graph.test.js | 8 + tests/unit/queue-handler.test.js | 533 +++++ tests/unit/queue-job.test.js | 352 +++ tests/unit/queue-memory.test.js | 378 +++ tests/unit/queue.test.js | 270 +++ tests/unit/render.test.js | 29 + tests/unit/templates-engines.test.js | 632 +++++ tests/unit/templates-loader.test.js | 131 ++ tests/unit/templates-renderer.test.js | 520 ++++ 88 files changed, 13676 insertions(+), 93 deletions(-) create mode 100644 docs/guide/audit.md create mode 100644 docs/guide/queue.md create mode 100644 docs/messages/templates.md create mode 100644 docs/transports/bullmq.md create mode 100644 src/audit/combine.js create mode 100644 src/audit/delivery-event.js create mode 100644 src/audit/index.d.ts create mode 100644 src/audit/index.js create mode 100644 src/audit/logger-sink.js create mode 100644 src/audit/memory-sink.js create mode 100644 src/audit/states.js create mode 100644 src/bullmq/backoff.js create mode 100644 src/bullmq/bullmq-driver.js create mode 100644 src/bullmq/index.d.ts create mode 100644 src/bullmq/index.js create mode 100644 src/bullmq/processor.js create mode 100644 src/internal/polling-driver.js create mode 100644 src/queue/errors.js create mode 100644 src/queue/handler.js create mode 100644 src/queue/hash.js create mode 100644 src/queue/index.d.ts create mode 100644 src/queue/index.js create mode 100644 src/queue/job.js create mode 100644 src/queue/json.js create mode 100644 src/queue/mail-queue.js create mode 100644 src/queue/memory-queue.js create mode 100644 src/queue/outcome.js create mode 100644 src/queue/parse.js create mode 100644 src/templates/cache.js create mode 100644 src/templates/css.js create mode 100644 src/templates/ejs.js create mode 100644 src/templates/errors.js create mode 100644 src/templates/eta.js create mode 100644 src/templates/front-matter.js create mode 100644 src/templates/handlebars.js create mode 100644 src/templates/includes.js create mode 100644 src/templates/index.d.ts create mode 100644 src/templates/index.js create mode 100644 src/templates/liquid.js create mode 100644 src/templates/loader.js create mode 100644 src/templates/mjml.js create mode 100644 src/templates/mustache.js create mode 100644 src/templates/nunjucks.js create mode 100644 src/templates/options.js create mode 100644 src/templates/pug.js create mode 100644 src/templates/react-email.js create mode 100644 src/templates/renderer.js create mode 100644 tests/bullmq/bullmq-driver.bullmq.test.js create mode 100644 tests/bullmq/docker-compose.yml create mode 100644 tests/helpers/bullmq/fake-bullmq.js create mode 100644 tests/helpers/queue/clock.js create mode 100644 tests/helpers/templates/fake-engine.js create mode 100644 tests/types/audit.test-d.ts create mode 100644 tests/types/bullmq.test-d.ts create mode 100644 tests/types/queue.test-d.ts create mode 100644 tests/types/templates.test-d.ts create mode 100644 tests/unit/audit.test.js create mode 100644 tests/unit/bullmq.test.js create mode 100644 tests/unit/queue-handler.test.js create mode 100644 tests/unit/queue-job.test.js create mode 100644 tests/unit/queue-memory.test.js create mode 100644 tests/unit/queue.test.js create mode 100644 tests/unit/templates-engines.test.js create mode 100644 tests/unit/templates-loader.test.js create mode 100644 tests/unit/templates-renderer.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index e3ab669..46efc5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +The reliability layer: a persistent queue whose workers render templates, and the audit of the +states of a message. Everything is additive. + +### Added + +- **`./queue`**: `createMailQueue({ driver, audit, logger })` with `enqueue`, `enqueueMany` and + `createWorker({ transport, renderer, resolveAttachment, checkJob, classify })`. A job is a + template with its context, rendered by the worker, or content rendered already, as JSON + checked when it is enqueued (`prepareMailJob`, `parseMailJob`): its fields as `createMessage` + checks them, its context refused with the path to what JSON would change, attachments as bytes + (bounded) or references the worker resolves. Its id is a UUID of the namespace, tenant and + `idempotencyKey`, so the same message enqueued again is one job and another under the key is + refused (`queue.key_reused`); its Message-ID and Content-IDs are fixed for every attempt. The + worker fails a permanent failure, a missing template or a bug at once, retries a temporary one + never sooner than `Retry-After`, an operator's failure (`auth`, `configuration`) after five + minutes, and never sends again a message of unknown delivery. `scheduledAt` is a delay of the + queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; + `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and + `createMemoryQueue()` is one in memory for tests. +- **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ + passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry + read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases + that spend no attempt. `createBullMQProcessor` and `createBullMQBackoffStrategy` serve a Worker + of your own. `pnpm test:bullmq` runs it on BullMQ and Valkey. +- **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, + defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in + (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, + `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, + `createReactEmailEngine`): names never paths (`createFileLoader` reads nothing outside its + directory), variants by locale and part, the subject in front matter, a layout, includes read + through the loader, each template compiled once, CSS inlined by juice or css-inline without + fetching a stylesheet (`createCssInliner`), and the text derived from the HTML. + `TemplateNotFoundError` and `TemplateRenderError` have a stable `code`. +- **`./audit`**: the states of a message (`queued`, `rendered`, `submitted`, `accepted`, + `delivered`, `bounced`, `complained`, `failed`, `delivery_unknown`) and their sinks: + `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks`, `stateOf`, and + `auditEventOfDeliveryEvent` for the events of `./webhooks`. +- **`./render`**: `RenderOutput.subject` and `renderFields(renderer, input)`, which gives + `{ subject, content }`. +- **The root**: the contracts of the reliability layer as types (`QueueDriver`, `QueueSource`, + `QueueJob`, `QueueOutcome`, `MessageState`, `AuditEvent`, `AuditSink`), with no code. + ## [1.1.0] - 2026-10-18 The features of three libraries of the [comparison](https://mailerat.vercel.app/guide/comparison) diff --git a/CLAUDE.md b/CLAUDE.md index 6c07249..bed0556 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,14 +20,19 @@ Nodemailer, emailjs or Upyo, made of two parts: of delivery events, suppression, idempotency and failover between providers. 1.1 has the model of delivery events with the verified webhooks of fifteen providers (`./webhooks`) and the idempotency decorator whose store interface the outbox is to implement (`./idempotency`). + Unreleased: the queue (`./queue`, a driver-neutral contract, its worker and a memory queue; + `./bullmq`, the BullMQ driver), templates by name over nine engines passed in (`./templates`) + and the audit of the states of a message (`./audit`). It has zero runtime dependencies and runs on Node.js ≥ 22, and on Bun and Deno through their Node compatibility (see "Platform"; no browser entry). **Status: 1.1 is the transport and the first parts of the reliability layer.** The root entry -exports the message model and the transport contract of upyo, with the delivery events of 1.1; -each transport, decorator and helper is a subpath, and their API is under semver from 1.0.0. The -queue, the audit log and suppression have no API yet; their direction is in "Roadmap" below. +exports the message model and the transport contract of upyo, with the delivery events of 1.1 +and, unreleased, the contracts of the reliability layer as types; each transport, decorator and +helper is a subpath, and their API is under semver from 1.0.0. The MongoDB outbox, the +crypto-shredding of the audit and suppression have no API yet; their direction is in "Roadmap" +below. It is maintained by Alexis Technologies and set up like the sibling libraries `@alexify/metaschema` and `@alexify/protoarray` (the scaffold follows protoarray's @@ -45,6 +50,7 @@ pnpm test:bun # tests/unit and tests/runtime on Bun (bun te pnpm test:deno # tests/unit and tests/runtime on Deno (deno test --parallel) pnpm test:mailpit # opt-in: tests/mailpit against a local Mailpit (compose file there) pnpm test:e2e # opt-in: tests/e2e against real providers, from env variables +pnpm test:bullmq # opt-in: tests/bullmq on BullMQ and Valkey (compose file there) node --test tests/unit/mailer.test.js # a single file node --test --test-name-pattern="Bundle" tests/unit/*.test.js # filter by name pnpm run test:coverage # c8 over src/, gated at 98 lines/statements, 90 branches, 100 functions @@ -816,7 +822,9 @@ src/ receipts (provider.js, refusal.js), message fields (headers.js, raw.js, schedule.js, tracking.js, capabilities.js, batch.js, base64.js, uuid.js), the decorators of 1.1 (gate.js, callback.js, options.js, decorator.js), - the base of the transports of 1.1 (provider-transport.js), and abort.js, + the base of the transports of 1.1 (provider-transport.js), what the + decorators and the queue share (backoff.js, thrown.js, fingerprint.js, + path.js), the driver of a QueueSource (polling-driver.js), and abort.js, iterate.js, warn.js, redact.js, redact-addresses.js, sleep.js, log.js tests/ unit/*.test.js node:test suites, one per area (-.test.js for a subpath) @@ -855,15 +863,27 @@ The root entry is the contract every subpath shares, ported from `@upyo/core` (` `retryAfterMilliseconds`; `classifyHttpStatus` and `classifyReceiptError` classify. A transport reports a failure in a receipt rather than by throwing. -The contracts of the reliability layer, still to be designed (Phase 0): - -- **`QueueDriver`**: enqueue, reserve, ack, retry with a delay; drivers for an outbox in MongoDB, - BullMQ and an in-memory one for tests. -- **`AuditSink`**: append-only records of state changes; sinks for MongoDB, stdout/OTel logs and a - custom one. +The contracts of the reliability layer (Phase 0), types of the root with no code, so that +`./queue`, `./bullmq`, `./audit` and a store's driver share them without requiring each other: + +- **`QueueDriver`**: `enqueue`, `enqueueMany`, `consume(handler) → QueueConsumer`; a + `QueueHandler` makes of each run of a job a `QueueOutcome` (`ack`, `retry` after a delay, + `fail`, `release` without spending the attempt), and never throws. BullMQ is a driver + (`./bullmq`); a store that lends its jobs (an outbox) is a **`QueueSource`** (`reserve`, + `extend`, `ack`, `retry`, `fail`, `release`, fenced by the token of the lease), which + `createPollingDriver` (`src/internal/polling-driver.js`) makes a driver of, as it does the + memory queue. The queue retries, not the transport: a worker's chain has `retries: 1` and no + `RetryTransport`. +- **A mail job** (`./queue`): JSON, `v: 1`, a template with its context or content rendered + already, its id a UUID of namespace, tenant and `idempotencyKey`, its Message-ID and + Content-IDs fixed (derived from the id under a key of the caller's), and a SHA-256 of all but + `enqueuedAt` that tells a duplicate from a key used again. +- **`AuditSink`**: `append(event, { transaction })`, append-only, an event written once by its + deterministic `id`; sinks in memory, of a logger, and combined (`./audit`). A MongoDB sink and + the crypto-shredding of recipients are next. - **The states and events of a message:** `queued → rendered → submitted → accepted → delivered | bounced | complained | failed | delivery_unknown`, with correlationId, tenantId, attempt, the - provider's message id and a hash of the payload. + provider's message id and the hashes of the payload and of the message; no recipient. - **`DeliveryEvent`** (root types, 1.1), what `./webhooks` reads from a provider: one per message and recipient, its `id` the same on every redelivery, its `messageId` that of the receipt. `accepted`, `delivered`, `bounced` and `complained` are the states above; `rejected` is @@ -888,6 +908,8 @@ model of states and events. `tenantId` is in the context from the start. providers. - **A persistent queue:** the outbox in MongoDB (transactional outbox, no Redis needed) and a BullMQ driver (Valkey) for throughput and delays; priorities (auth codes before newsletters). + (Unreleased: `./queue` and `./bullmq`, with the memory queue; the outbox is next, a + `QueueSource`.) - **The audit log** of every state, with a keyed HMAC plus an encrypted block for the recipient (crypto-shredding), so the right to erasure does not fight an append-only log. 3–6 records per message; optional hash chain for tamper evidence. @@ -934,6 +956,12 @@ engine of its own; done in 1.1, `./render`); more providers (done in 1.1). more would be fifteen more parts of the API to keep under semver, for less than a kilobyte each. If that cost is ever measured to matter, the entries can come later without a change to the one there is. +- **One `./templates` entry, not one per engine.** The nine adapters weigh under a kilobyte + each, the engines are passed in, and the core (loaders, variants, front matter, cache) is most + of the entry, as the verifiers of `./webhooks` are of theirs. +- **Templates are named, never pathed, and read through the loader.** No engine reads a file + itself: includes, partials and layouts go through the renderer's loader, which holds them to + its root; the one exception, MJML's `mj-include`, is off unless asked for. - **Eighteen providers, then few new ones.** The eight HTTP providers of upyo came with the port; 1.1 adds the ten most used of unemail and sently (Postmark, Brevo, Mailjet, MailerSend, SparkPost, ZeptoMail, SMTP2GO, Azure Communication Services, Scaleway and Cloudflare). Each @@ -954,8 +982,9 @@ accepted but never read (Upyo `validateSsl`); no CRLF sanitization and `os.EOL` inbound mail; the management APIs of the providers (domains, templates, statistics), which the official SDKs cover. -**To check before designing:** whether per-tenant fairness (groups) is a paid BullMQ Pro feature; -whether Fly.io restricts outgoing SMTP (it would make HTTP-first a requirement). +**To check before designing:** whether Fly.io restricts outgoing SMTP (it would make HTTP-first +a requirement). Checked: per-tenant fairness (groups) is a feature of BullMQ Pro only; the type +definitions of BullMQ 6 have no group options. ## Platform @@ -1065,7 +1094,14 @@ change. **This file wins** if they ever disagree. /mailpit sendmail"` for the sendmail case) and `tests/e2e/` (`pnpm test:e2e`, each suite skipped until its provider's variables are set; Stalwart for JMAP in `tests/e2e/jmap/`; the providers of 1.1 in one table, `providers.e2e.test.js`, which also checks that refused credentials are a - non-retryable `auth` failure). The release skill asks for them before a release. + non-retryable `auth` failure) and `tests/bullmq/` (`pnpm test:bullmq`, serial; + `docker compose -f tests/bullmq/docker-compose.yml up -d` and + `BULLMQ_REDIS_URL=redis://localhost:6390`: duplicates, retries after `Retry-After`, priorities, + delays, a release on close and a Worker without the strategy, on BullMQ itself). The release + skill asks for them before a release. +- **The template engines, BullMQ and ioredis are devDependencies** for the tests of `./templates` + and `./bullmq` (the real engines, a fake BullMQ in `tests/helpers/bullmq/`, and tsd against the + real classes); `src/` requires none of them. - **Platform-dependent tests:** the mock SMTP server takes a client that has gone as no failure: a reset (`ECONNRESET`; macOS and Windows send an RST where Linux sends a FIN) and a reply written after the client closed (`EPIPE` on Linux, `ECONNABORTED` on Windows), which the 221 to diff --git a/README.md b/README.md index b3cc0d2..028f5b7 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,10 @@ **Email for Node.js, with a reliability layer.** Its own SMTP client, MIME builder, DKIM signing and eighteen HTTP providers, and on top of them what the other libraries leave to the -application: one model of delivery events, idempotency, and next a persistent queue and outbox, an -audit log and suppression. Zero runtime dependencies. +application: a persistent queue whose workers render templates of the engine you use and know a +permanent failure from a temporary one, an audit of every state of a message, one model of +delivery events and idempotency; next an outbox in MongoDB and suppression. Zero runtime +dependencies. > **Status: 1.1 is the transport layer and the first parts of the reliability layer.** 1.0 ported > the transport from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds what @@ -19,7 +21,9 @@ audit log and suppression. Zero runtime dependencies. > [sently](https://github.com/omqkhafi/sently) did better: sendmail, ten more HTTP providers, > attachments from paths and URLs, one-click List-Unsubscribe, verified webhooks into one model of > delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and -> mirroring. The persistent queue, the audit log and suppression are being designed. +> mirroring. Unreleased: the queue (`./queue`, with a BullMQ driver in `./bullmq`), templates by +> name over nine engines (`./templates`) and the audit of the states of a message (`./audit`). +> The MongoDB outbox and suppression are next. ### 📖 [Read the documentation →](https://mailerat.vercel.app/) @@ -35,9 +39,10 @@ audit log and suppression. Zero runtime dependencies. read for an attachment unless you ask. - **What happens after `send()`.** The webhooks of fifteen providers, each verified before it is read and taken as handled only once your handler commits it, give one model of delivery events - (delivered, bounced, complained, …). Next: a persistent queue and outbox, an append-only audit - log and suppression. None of the - [fourteen libraries compared](https://mailerat.vercel.app/guide/comparison) has the three. + (delivered, bounced, complained, …). A queue on BullMQ or a store of yours sends each message + once per key, retries what may pass and fails what will not, and an audit sink hears every state + from `queued` to `bounced`. Next: an outbox in MongoDB and suppression. None of the + [fourteen libraries compared](https://mailerat.vercel.app/guide/comparison) has them. - **Sent once, or known to be unknown.** Errors are classified as permanent or temporary, and a failure after which the provider may have taken the message is marked `deliveryUnknown`: `./retry` and `./pool` never send it again. `./idempotency` reserves a key before the send, @@ -47,9 +52,10 @@ audit log and suppression. Zero runtime dependencies. golden `.eml` messages and fuzzing of headers, `multipart/related`, dot-stuffing and internationalized addresses are planned. - **Zero dependencies.** Built on `node:*` modules and `fetch`. What a part needs from your - application is passed in: the OpenTelemetry providers, a pino-like logger, the `render` of React - Email, and later the driver of a queue or a database (BullMQ, MongoDB). CommonJS with ESM named - imports and hand-written TypeScript typings, no build step. + application is passed in: the OpenTelemetry providers, a pino-like logger, BullMQ's classes, + your template engine (EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache, React Email), + and later a database driver (MongoDB). CommonJS with ESM named imports and hand-written + TypeScript typings, no build step. ## Installation @@ -92,7 +98,9 @@ if (!receipt.successful) console.error(receipt.errorMessages); | --- | --- | | `@alexify/mailer` | messages, attachments, calendar parts, the `Transport` contract, receipts, delivery events | | `/mime` | MIME composition and DKIM signing | -| `/attachments`, `/render` | attachments from a path, a URL, a data URI or a stream; templates and React Email | +| `/attachments`, `/render` | attachments from a path, a URL, a data URI or a stream; the content of a renderer, React Email | +| `/templates` | templates by name over EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache or React Email: locales, layouts, subjects, CSS inlined | +| `/queue`, `/bullmq`, `/audit` | the mail queue and its worker, the BullMQ driver, the states of a message and their sinks | | `/smtp`, `/sendmail`, `/jmap` | SMTP, sendmail and JMAP transports | | `/ses`, `/resend`, `/sendgrid`, `/mailgun`, `/postmark`, `/brevo`, `/mailjet`, `/mailersend`, `/sparkpost`, `/mailtrap`, `/maileroo`, `/lettermint`, `/zeptomail`, `/smtp2go`, `/azure`, `/scaleway`, `/cloudflare`, `/plunk` | HTTP providers | | `/webhooks` | verified webhooks of fifteen providers, read into delivery events | @@ -104,6 +112,37 @@ UUID) and a pino-like `logger` for what happens inside it, and every HTTP transp such as one bound to an undici `Agent` or a proxy; see [your own fetch and identifiers](https://mailerat.vercel.app/guide/getting-started#your-own-fetch-and-identifiers). +Through a queue, the worker renders the template and sends the message, and the queue retries +what may pass: + +```js +const { Queue, Worker } = require('bullmq'); +const ejs = require('ejs'); +const { createMailQueue } = require('@alexify/mailer/queue'); +const { createBullMQDriver } = require('@alexify/mailer/bullmq'); +const { createEjsEngine, createFileLoader, createTemplateRenderer } = require('@alexify/mailer/templates'); + +const queue = createMailQueue({ + driver: createBullMQDriver({ bullmq: { Queue, Worker }, connection: { host: 'localhost' } }), +}); +await queue.enqueue({ + kind: 'template', + template: 'reset-password', // emails/reset-password.ejs, its subject in its front matter + context: { url }, + from: 'no-reply@example.com', + to: user.email, + idempotencyKey: `reset-password:${token}`, +}); + +queue.createWorker({ + transport, + renderer: createTemplateRenderer({ + engine: createEjsEngine({ ejs }), + loader: createFileLoader({ dir: 'emails' }), + }), +}); +``` + The same names import from ESM (`import { SmtpTransport } from '@alexify/mailer/smtp'`). The [API reference](https://mailerat.vercel.app/api/exports) lists every member, and the [guide](https://mailerat.vercel.app/guide/getting-started) every transport. diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 3fa712f..5907970 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -68,6 +68,7 @@ const guide = [ { text: 'Composing messages', link: '/messages/compose' }, { text: 'Attachments', link: '/messages/attachments' }, { text: 'Rendering templates', link: '/messages/render' }, + { text: 'Templates by name', link: '/messages/templates' }, { text: 'Webhooks and events', link: '/guide/webhooks' }, { text: 'Calendar invitations', link: '/messages/calendar' }, { text: 'MIME composition', link: '/messages/mime' }, @@ -115,6 +116,14 @@ const guide = [ { text: 'Tee', link: '/transports/tee' }, ], }, + { + text: 'Reliability', + items: [ + { text: 'The queue', link: '/guide/queue' }, + { text: 'BullMQ', link: '/transports/bullmq' }, + { text: 'The audit', link: '/guide/audit' }, + ], + }, ]; // https://vitepress.dev/reference/site-config diff --git a/docs/api/exports.md b/docs/api/exports.md index 9890b60..5d13167 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -41,6 +41,10 @@ model of delivery events. - **Delivery events:** the `DeliveryEvent`, `DeliveryEventType` and `DeliveryBounce` types, which `./webhooks` gives. See [webhooks and events](/guide/webhooks). +- **The reliability layer:** the `QueueDriver`, `QueueSource`, `QueueJob`, + `QueueJobRequest`, `QueueOutcome` and `QueueHandler` types every driver + shares, and `MessageState`, `AuditEvent` and `AuditSink`, as types only. See + [the queue](/guide/queue) and [the audit](/guide/audit). ## Subpaths @@ -48,7 +52,11 @@ model of delivery events. | --- | --- | --- | | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | -| `@alexify/mailer/render` | `renderContent`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | +| `@alexify/mailer/render` | `renderContent`, `renderFields`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | +| `@alexify/mailer/templates` | `createTemplateRenderer`, `createFileLoader`, `createMemoryLoader`, `createCssInliner`, `createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, `createReactEmailEngine`, `TemplateNotFoundError`, `TemplateRenderError` | [Templates by name](/messages/templates) | +| `@alexify/mailer/queue` | `createMailQueue`, `createMailJobHandler`, `createMemoryQueue`, `createPollingDriver`, `prepareMailJob`, `parseMailJob`, `MailQueueError`, `QueueLeaseLostError`, `MAIL_JOB_NAME`, `MAIL_JOB_VERSION` | [The queue](/guide/queue) | +| `@alexify/mailer/bullmq` | `createBullMQDriver`, `createBullMQProcessor`, `createBullMQBackoffStrategy`, `MAILER_BACKOFF_TYPE` | [BullMQ](/transports/bullmq) | +| `@alexify/mailer/audit` | `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks`, `stateOf`, `stateOfDeliveryEvent`, `auditEventOfDeliveryEvent`, `isFinalState` | [The audit](/guide/audit) | | `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMailerSendWebhookVerifier`, `createMailtrapWebhookVerifier`, `createBrevoWebhookVerifier`, `createMailjetWebhookVerifier`, `createSparkPostWebhookVerifier`, `createLettermintWebhookVerifier`, `createMailerooWebhookVerifier`, `createSmtp2goWebhookVerifier`, `createAzureWebhookVerifier`, `createScalewayWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | | `@alexify/mailer/sendmail` | `SendmailTransport` | [Sendmail](/transports/sendmail) | diff --git a/docs/comparison.json b/docs/comparison.json index 9fc5a75..2d9c37e 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -339,9 +339,9 @@ "text": "`signal` + timeout over the body" }, "queue": { - "mark": "🔜", - "text": "MongoDB outbox, BullMQ driver", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "✅", + "text": "BullMQ driver, memory queue, any outbox as a `QueueSource`", + "note": "Permanent failures fail at once, `Retry-After` is a minimum, and a delivery of unknown outcome is never sent blindly again. A MongoDB outbox is next on the [roadmap](/guide/why#the-plan)." }, "attachments": { "mark": "✅", @@ -373,8 +373,9 @@ "text": "RFC 8058 one-click, DKIM-signed" }, "templates": { - "mark": "⚠️", - "text": "`Renderer` interface, no engine by design" + "mark": "✅", + "text": "9 engines passed in: EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache, React Email", + "note": "Names, never paths; locales, layouts, front-matter subjects, CSS inlined without remote fetches, text derived." }, "reactEmail": { "mark": "✅", @@ -385,8 +386,8 @@ "text": "native: Resend, Postmark, Mailjet, Mailtrap, JMAP; concurrency elsewhere" }, "scheduled": { - "mark": "⚠️", - "text": "native in 8 providers; queue delay planned", + "mark": "✅", + "text": "queue delay; native in 8 providers", "note": "A transport that cannot schedule refuses the message before sending it, rather than sending it at once." }, "tags": { @@ -476,9 +477,9 @@ "text": "`./mock`" }, "auditLog": { - "mark": "🔜", - "text": "append-only, crypto-shredding", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "⚠️", + "text": "state model + sinks; crypto-shredding next", + "note": "Each state of a message, from the queue to the webhooks, one event per change. The keyed HMAC and encrypted recipients are on the [roadmap](/guide/why#the-plan)." }, "types": { "mark": "✅", diff --git a/docs/guide/audit.md b/docs/guide/audit.md new file mode 100644 index 0000000..4ae18d2 --- /dev/null +++ b/docs/guide/audit.md @@ -0,0 +1,83 @@ +# The audit + +`@alexify/mailer/audit` keeps what happened to each message: one event per +change of its state, from the queue to the provider's webhooks, all with its +`correlationId`. A sink is anything with `append(event)`; the package gives +one in memory, one of your logger, and the mapping of [delivery +events](/guide/webhooks). + +## States + +``` +queued ─▶ rendered ─▶ submitted ─▶ accepted ─▶ delivered │ bounced │ complained + ▲ │ + └──── queued again ◀───┤ (a retry) + ├─▶ failed (no attempt again) + └─▶ delivery_unknown (it may have been taken) +``` + +`stateOf(events)` reads the state of a message from its events whatever their +order: a `delivered` from a webhook settles a `delivery_unknown`, a `bounced` +comes after a `delivered`. `isFinalState` is true for `delivered`, `bounced`, +`complained` and `failed`. + +## An event + +```js +{ + id: '…', // the same for the same change: a sink writes it once + state: 'accepted', + at: '2026-10-18T09:00:00.000Z', + source: 'worker', // 'producer', 'worker' or 'webhook' + correlationId: '…', + jobId: '…', tenantId: 'acme', attempt: 1, + payloadHash: '…', // the job as enqueued + messageHash: '…', // the message as rendered + messageId: '…@example.com', provider: 'ses', providerMessageId: '…', + priority: 'high', template: 'reset-password', tags: ['auth'], + error: { category, code, message, retryable }, // for a failure, redacted + retryAt: '…', // for a retry +} +``` + +An event holds no recipient: the message is found by its `correlationId`. + +## Sinks + +```js +const { + combineAuditSinks, + createLoggerAuditSink, + createMemoryAuditSink, +} = require('@alexify/mailer/audit'); + +const audit = combineAuditSinks(createLoggerAuditSink(logger), myDatabaseSink); +``` + +- `createMemoryAuditSink({ maxEvents })` holds the events, with + `eventsOf(correlationId)` and `stateOf(correlationId)`: for tests. +- `createLoggerAuditSink(logger, { levels })` writes each event as a record + `mail.`: `failed` at error; `delivery_unknown`, `bounced` and + `complained` at warn. +- `combineAuditSinks(...sinks)` tells each in turn, whatever another threw. +- Your own: `{ append(event, { transaction }) }`. An outbox writes the + `queued` event in the transaction of the enqueue; a sink keyed by `id` + writes an event once however often it is told. + +## Webhooks + +```js +const { auditEventOfDeliveryEvent } = require('@alexify/mailer/audit'); + +const { events, commit } = await verifier.verify(request); +for (const event of events) { + const audited = auditEventOfDeliveryEvent(event); + if (audited !== undefined) await audit.append(audited); +} +await commit(); +``` + +`accepted`, `delivered`, `bounced` and `complained` are states; `rejected` is +`failed`; `deferred` keeps a message accepted; an open, a click and an +unsubscribe are engagement, not states. The queue writes `correlationId` into +the metadata of each message, which the providers give back with its events. diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index d28af74..a08dde5 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -70,7 +70,7 @@ it. Rate limiting✅ token bucket per key, FIFO, shared store✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry Circuit breaker✅ one probe in half-open❌❌❌✅ half-open without a single probe⚠️ cooldownMs❌❌—❌❌❌❌❌❌ Timeout / AbortSignal✅ signal + timeout over the body⚠️ timeouts, no AbortSignal⚠️ timeout, no signal✅ signal + timeout⚠️ signal for HTTP, not SMTP⚠️ SMTP connect only⚠️ socket timeouts✅ signal on connect and send—⚠️ signal, no timeout⚠️ global signal only⚠️ timeout, no signal⚠️ 180 s timeout⚠️ hard 10 s✅ abortSignal + requestTimeout -Persistent queue (survives a restart)🔜 MongoDB outbox, BullMQ driver3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌ +Persistent queue (survives a restart)✅ BullMQ driver, memory queue, any outbox as a QueueSource3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌ Messages Attachments✅ path, URL, data URI, stream, bytes, Blob4✅ path, URL, Buffer, stream, data-URI⚠️ path, stream, no URL⚠️ bytes, Blob, File, factory⚠️ bytes, string, URL⚠️ bytes + path⚠️ base64 only⚠️ string, bytes, Blob⚠️ base64 only⚠️ base64 or URL⚠️ base64, helper for Buffer✅ Buffer, Blob, stream, 50 MB✅ base64✅ string / Buffer✅ Uint8Array or Raw MIME Streaming large attachments⚠️ SMTP, sendmail; HTTP providers buffer⚠️ SMTP yes; DKIM and SES buffer✅⚠️ SMTP only❌❌❌❌❌❌❌⚠️ accepts a stream❌❌❌ @@ -79,10 +79,10 @@ it. iCalendar✅ every METHOD✅ icalEvent + METHOD⚠️ manual✅ every METHOD🔜 planned❌❌✅⚠️ as an attachment❌❌⚠️ as an attachment⚠️ as an attachment⚠️ as an attachment⚠️ through Raw MIME Unicode / IDN✅ punycode + SMTPUTF8✅ punycode + SMTPUTF8⚠️ Q-encoding, no punycode⚠️ SMTPUTF8, no automatic punycode⚠️ RFC 2047/2231⚠️ one RFC 2047 word⚠️ RFC 2047⚠️ SMTPUTF8, no punycode⚠️ Subject as one B-word⚠️ UTF-8 JSON⚠️ UTF-8 JSON❌❌⚠️ UTF-8 JSON⚠️ Charset, no IDN List-Unsubscribe✅ RFC 8058 one-click, DKIM-signed✅ list.unsubscribe❌⚠️ manual header✅ RFC 8058 oneClick❌⚠️ headers⚠️ headers❌⚠️ headers, topics⚠️ headers, asm⚠️ headers⚠️ headers⚠️ headers✅ ListManagementOptions -Built-in templates⚠️ Renderer interface, no engine by design❌❌❌⚠️ Renderer interface✅ {{var}} + plugins❌❌❌✅ Templates API✅ dynamic templates✅ Templates API✅ Templates API✅ Templates API✅ Templates API +Built-in templates✅ 9 engines passed in: EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache, React Email5❌❌❌⚠️ Renderer interface✅ {{var}} + plugins❌❌❌✅ Templates API✅ dynamic templates✅ Templates API✅ Templates API✅ Templates API✅ Templates API React Email✅ render passed in❌❌❌✅✅❌❌❌✅❌❌❌❌❌ Batch / bulk✅ native: Resend, Postmark, Mailjet, Mailtrap, JMAP; concurrency elsewhere❌❌✅ native: Resend, Mailtrap, JMAP✅ sendBatch⚠️ sendBulk, SendGrid bug⚠️ queue per session✅—✅ up to 100, no attachments✅ personalizations⚠️ recipient-variables✅ batch + bulk API✅ batchSend, 500✅ SendBulkEmail -Scheduled sending⚠️ native in 8 providers; queue delay planned5❌❌❌⚠️ 8 drivers❌❌❌—✅✅✅ up to 3 days❌⚠️ campaigns only❌ +Scheduled sending✅ queue delay; native in 8 providers6❌❌❌⚠️ 8 drivers❌❌❌—✅✅✅ up to 3 days❌⚠️ campaigns only❌ Tags / metadata✅ metadata in 17 providers, tags in 13⚠️ headers; tags on SES only⚠️ headers⚠️ tags in 7 providers✅⚠️ headers only⚠️ headers⚠️ headers⚠️ headers✅✅✅✅✅✅ Open/click tracking✅ per message in 10 providers❌❌⚠️ config only✅ 10 drivers❌❌❌—⚠️ per domain✅✅✅⚠️ per domain✅ per-message override Idempotency key✅ decorator with reserved keys + 5 providers❌❌⚠️ Resend, Lettermint, Maileroo✅ middleware + providers⚠️ decorator; batch bypasses it❌❌—✅❌❌❌❌❌ @@ -92,13 +92,13 @@ it. ARC❌❌❌❌❌❌❌❌❌—————— DSN✅✅❌✅❌❌✅✅——————— Proxy✅ SMTP: HTTP CONNECT or any connect; HTTP: fetch⚠️ HTTP CONNECT, SOCKS through a module❌❌⚠️ custom fetch❌❌❌—⚠️ baseUrl only⚠️ axios proxy✅⚠️ custom fetch + undici⚠️ environment only⚠️ through requestHandler -verify() / connection check✅ SMTP, sendmail, JMAP, 13 HTTP providers6✅⚠️ VRFY, not a check⚠️ SMTP/JMAP only⚠️ isAvailable(), nominal✅❌⚠️ probe() without auth———⚠️ domain DNS only⚠️ getServer()⚠️ getAllAccounts()⚠️ GetAccount +verify() / connection check✅ SMTP, sendmail, JMAP, 13 HTTP providers7✅⚠️ VRFY, not a check⚠️ SMTP/JMAP only⚠️ isAvailable(), nominal✅❌⚠️ probe() without auth———⚠️ domain DNS only⚠️ getServer()⚠️ getAllAccounts()⚠️ GetAccount SMTP AUTH mechanismsPLAIN, LOGIN, XOAUTH2, OAUTHBEARERPLAIN, LOGIN, CRAM-MD5, XOAUTH2CRAM-MD5, LOGIN, PLAIN, XOAUTH2PLAIN, LOGIN, XOAUTH2, OAUTHBEARERPLAIN, LOGIN, CRAM-MD5, XOAUTH2PLAIN, LOGIN, CRAM-MD5, XOAUTH2PLAIN, LOGIN, CRAM-MD5PLAIN, LOGIN, CRAM-MD5, XOAUTH2——————— SMTP TLS policies✅ requireTls, SNI, CA, cert, minVersion✅ REQUIRETLS, SNI, no DANE⚠️ STARTTLS accepts any certificate✅ requireTls, CA, cert, minVersion✅ requireTLS, off by default✅ requireTLS by default with auth⚠️ falls back to plaintext + AUTH✅ tlsPolicy ×4, REQUIRETLS——————— Platform features (events, domains, provider API) -Webhooks: parsing and signature verification✅ 15 providers, verified before parsing7❌❌❌🔜 planned✅ 10 parsers, HMAC in 5❌❌—✅ standardwebhooks✅ ECDSA⚠️ CRUD, no verification⚠️ CRUD + types, no verification✅ HMAC verify❌ SNS signature not in the SDK +Webhooks: parsing and signature verification✅ 15 providers, verified before parsing8❌❌❌🔜 planned✅ 10 parsers, HMAC in 5❌❌—✅ standardwebhooks✅ ECDSA⚠️ CRUD, no verification⚠️ CRUD + types, no verification✅ HMAC verify❌ SNS signature not in the SDK Inbound mail❌ out of scope❌❌❌🔜 planned❌❌❌—✅⚠️ parser with a bug⚠️ routes✅✅❌ legacy client-ses only -Suppression / bounce API🔜 per tenant, before the queue3❌❌❌🔜 planned❌❌❌—✅⚠️ generic client✅✅✅✅ +Suppression / bounce API🔜 per tenant, before the queue9❌❌❌🔜 planned❌❌❌—✅⚠️ generic client✅✅✅✅ Domain management (API)❌ out of scope❌❌❌❌❌❌❌—✅⚠️ generic client✅✅✅✅ identities Address validation⚠️ RFC 5322/6531 syntax⚠️ parser only⚠️ parser only⚠️ RFC 5322/6531 syntax⚠️ regex⚠️ regex❌❌ syntax assert❌❌❌✅ /v4/address/validate❌❌✅ GetEmailAddressInsights Sending log on the provider side—————————✅ logs API⚠️ generic /v3/messages✅ logs API⚠️ events, opens, clicks⚠️ emailLogs⚠️ event destinations + archiving @@ -108,7 +108,7 @@ it. OpenTelemetry✅ decorator: spans + metrics❌❌✅ decorator: spans + metrics⚠️ withLogger⚠️ hooks + consoleObserver❌❌❌❌❌❌❌❌⚠️ middlewareStack Hooks / middleware✅ ./hooks, decorators, a pino-like logger✅ use(compile|stream)⚠️ logger function⚠️ decorators, LogTape✅ Middleware pipeline✅ hooks + plugins⚠️ logger✅ onEvent⚠️ dump()⚠️ console.error⚠️ setClient❌❌❌✅ logger + middlewareStack Mock / test mode✅ ./mock✅ jsonTransport, streamTransport, Ethereal⚠️ Message#read()✅ @upyo/mock✅ mock + Mailpit⚠️ preview, Mailpit❌❌—❌✅ sandboxMode✅ testmode⚠️ Sandbox server✅ Sandbox API⚠️ TestRenderEmailTemplate -Local audit log / event journal🔜 append-only, crypto-shredding3❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌ +Local audit log / event journal⚠️ state model + sinks; crypto-shredding next10❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌ Quality, license, maturity TypeScript types✅ hand-written, tsd✅ built in✅✅✅✅✅✅✅✅✅ hand-written .d.ts✅✅✅✅ generated Runtime dependencies0000–1 (@noble/hashes)0000422+ (axios)3 (axios)02 (axios, qs)9 @@ -127,11 +127,14 @@ it. 1. Through npm:@alexify/mailer. Deno’s fetch sends a request again when a reused connection closes before its answer, so the docs give it a fetch of its own (known issues). 2. A failure after which the provider may have taken the message is marked deliveryUnknown: ./retry does not retry it and ./pool does not fail over with it. -3. On the roadmap of the reliability layer: designed, with no code yet. +3. Permanent failures fail at once, Retry-After is a minimum, and a delivery of unknown outcome is never sent blindly again. A MongoDB outbox is next on the roadmap. 4. Nothing is read implicitly: fromPath and fromUrl are called by the application, fromUrl checks every redirect against allowUrl (no private address by default) and bounds time and bytes. -5. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. -6. The other five have no request that checks a sending key without sending a message. -7. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time, and an id kept as taken only once the handler commits its events. +5. Names, never paths; locales, layouts, front-matter subjects, CSS inlined without remote fetches, text derived. +6. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. +7. The other five have no request that checks a sending key without sending a message. +8. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time, and an id kept as taken only once the handler commits its events. +9. On the roadmap of the reliability layer: designed, with no code yet. +10. Each state of a message, from the queue to the webhooks, one event per change. The keyed HMAC and encrypted recipients are on the roadmap. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 355ebfe..9ace7dd 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -4,7 +4,8 @@ Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds sendmail, ten more HTTP providers, verified webhooks into delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and -mirroring. The queue and the audit log are still being designed. +mirroring. Unreleased: [the queue](/guide/queue), [templates by name](/messages/templates) +and [the audit](/guide/audit). ::: ## Installation @@ -215,6 +216,7 @@ their own unless the part is given a store they share: | --- | --- | --- | --- | | [`./ratelimit`](/transports/ratelimit) | the token buckets | at most `maxKeys` buckets | a `store` of `take()`, such as a script on Redis, so that N processes send no more than the one limit | | [`./idempotency`](/transports/idempotency) | the keys reserved and the receipts | `createMemoryIdempotencyStore()` | a store of `reserve`, `complete` and `release`, the outbox to come among them | +| [`./queue`](/guide/queue) | the jobs | `createMemoryQueue()` | a driver they share: [BullMQ](/transports/bullmq), or a store's outbox | | [`./webhooks`](/guide/webhooks#time-and-replays) | the ids of the requests taken | `createMemoryReplayStore()` | a store of `reserve`, `commit` and `release`; required, `null` to keep none | | [`./breaker`](/transports/breaker) | the state of the circuit | one per transport | none: each process opens its own circuit on the failures it sees | | [`./pool`](/transports/pool) | the turn of round-robin | one per transport | none | diff --git a/docs/guide/queue.md b/docs/guide/queue.md new file mode 100644 index 0000000..fd12c22 --- /dev/null +++ b/docs/guide/queue.md @@ -0,0 +1,157 @@ +# The queue + +`@alexify/mailer/queue` sends mail through a queue: a producer enqueues a +message as a job, and a worker renders it, sends it through the transport you +composed, and decides from the receipt what the queue does next. The queue is a +driver of your choice: [BullMQ](/transports/bullmq) on Redis or Valkey, the +memory queue for tests, or a store's outbox (a `QueueSource`). + +``` +producer queue.enqueue(job) ──▶ driver (BullMQ, outbox, memory) +worker driver ──▶ render ──▶ attachments ──▶ createMessage ──▶ transport.send ──▶ receipt + │ + ack · retry after a delay · fail for good · release ◀─────────┘ +``` + +## Enqueue and work + +```js +const { Queue, Worker } = require('bullmq'); +const { createMailQueue } = require('@alexify/mailer/queue'); +const { createBullMQDriver } = require('@alexify/mailer/bullmq'); +const { createLoggerAuditSink } = require('@alexify/mailer/audit'); + +const driver = createBullMQDriver({ bullmq: { Queue, Worker }, connection: { host: 'valkey' } }); +const queue = createMailQueue({ driver, audit: createLoggerAuditSink(logger), logger }); + +// The producer: an HTTP handler, a cron. +await queue.enqueue({ + kind: 'template', + template: 'reset-password', + context: { name: user.name, url }, + locale: user.locale, + from: 'Hirevel ', + to: user.email, + idempotencyKey: `reset-password:${token}`, + priority: 'high', + expiresAt: new Date(Date.now() + 15 * 60_000), +}); + +// The worker, in a process of its own or the same one. +const worker = queue.createWorker({ + transport: new IdempotencyTransport(new SesTransport({ ...config, retries: 1 }), { store }), + renderer: templates, + concurrency: 5, +}); +process.on('SIGTERM', () => worker.close()); +``` + +A job is `{ kind: 'template', template, context, locale }`, which the worker +renders with its `renderer` (such as one of [`./templates`](/messages/templates)), +or `{ kind: 'content', subject, content }`, rendered already, such as with React +Email in the producer. The other fields are those of +[`createMessage`](/messages/compose), plus: + +| Field | What it does | +| --- | --- | +| `idempotencyKey` | the key the job is enqueued once under (one is made when absent) | +| `scheduledAt` | a delay of the queue: not sent before then | +| `expiresAt` | not sent after then: a code of sign-in sent late is of no use | +| `tenantId` | scopes the key of the job | +| `correlationId` | what the events of the message share, from the queue to the webhooks | +| `maxAttempts` | the attempts in all (5 by default) | + +## What a job holds + +A job is JSON, checked when it is enqueued, so a bad address fails the caller, +not the worker: + +- the fields are checked as `createMessage` checks them; +- `context` must be JSON as it is: a `Date`, a `Map`, a class, `NaN` or a cycle is + refused with the path to it (`context.user.createdAt is a Date; pass an ISO + string`), so a template does not meet a string where it expected a date; +- an attachment goes as its bytes (base64, 256 KiB in all by default) or as + `{ ref }`, which the worker's `resolveAttachment(ref, { job, attachment, + signal })` reads, with `fromPath` or `fromUrl` of + [`./attachments`](/messages/attachments). A job names no file or URL itself: + whoever can write to the store could; +- the Message-ID and the Content-IDs are fixed, so every attempt sends the + same message; +- a job is 256 KiB at most. + +## Once per key + +The id of a job is a UUID of its namespace, tenant and `idempotencyKey`: the +same message enqueued again is the same job (`duplicate: true`), and another +message under the key is refused with `MailQueueError` `queue.key_reused`. +The worker gives the message that id as its `idempotencyKey`, so +[`IdempotencyTransport`](/transports/idempotency), with a store the workers +share, and the providers that take a key (Resend, Lettermint, Brevo, Azure) +dedupe it again. A worker that died mid-send leaves the key reserved, and the +attempt again is of unknown delivery (`delivery_unknown`), not a second send. + +The rendering must then be the same for the same context: no `Date.now()` in a +template, or an attempt again is refused as another message. + +## What the queue does with an attempt + +| The attempt | The queue | The state | +| --- | --- | --- | +| accepted | done | `accepted` | +| a failure of unknown delivery (a timeout after the request, a key in flight, the worker stopped mid-send) | failed for good, never sent blindly again | `delivery_unknown` | +| a retryable failure, attempts left | retried after the backoff, never sooner than the provider's `Retry-After` | `queued` | +| a retryable failure, the last attempt | failed (`queue.attempts_exhausted`) | `failed` | +| a refused message (`validation`, `rejected`, `unsupported`) | failed at once | `failed` | +| a failure of the operator (`auth`, `configuration`) | retried after `operatorDelayMilliseconds` (5 minutes) | `queued` | +| a template missing or broken, no subject, no renderer | failed at once (`render.*`) | `failed` | +| an outage of the template loader or of `resolveAttachment` | retried | `queued` | +| a bug: a `TypeError` thrown by the transport | failed at once (`queue.thrown`), logged with its stack | `failed` | +| a job past `expiresAt`, or whose retry would come after it | failed (`queue.expired`) | `failed` | +| a job written by a later version of the worker | retried, for a worker of that version | `queued` | +| the worker stops before the send | given back, its attempt not spent | — | + +The backoff is exponential with full jitter: 30 seconds, doubled per attempt, +an hour at most (`backoff`). A `Retry-After` is a minimum, with a little jitter +over it, so that a rate limit does not bring every job back at once. +`rateLimitScope: 'queue'` makes a rate limit pause the whole queue where the +driver can. `classify({ receipt, job })` decides in the place of these rules +(`'retry'`, `'fail'`, `'delivery_unknown'`, or `undefined` for the rules), and +`checkJob(data, job)` refuses a job for good by throwing. + +**The queue retries; the transport should not.** Each attempt of a +`RetryTransport` inside the worker holds the job and multiplies its attempts: +give the providers `retries: 1` (they retry only what was provably not sent) +and leave `RetryTransport` out. + +## The audit and the log + +The worker tells the [audit sink](/guide/audit) each state: `queued` at the +enqueue (in the transaction of an outbox), `rendered`, `submitted` just before +the send, then `accepted`, `failed` or `delivery_unknown`, and `queued` again +for a retry. A sink that fails is a warning (`MAILER_AUDIT_FAILED`); +`auditStrict: true` makes it retry the job before the send, so that no message +goes unrecorded. + +`logger`, pino-like, hears what the queue does, never an address: the +enqueues and the starts at `debug`, the sends at `info`, the retries, +redeliveries and releases at `warn`, the failures at `error` with their code, +category and reason. + +## Several processes + +A job is shared by every worker of its driver; the key store of +`IdempotencyTransport` must be shared too (Redis, the outbox), or a crash +mid-send can be sent twice. The breaker and the rate limit of the chain are +per process unless given a shared store (see +[getting started](/guide/getting-started#several-processes)). + +## The memory queue and the polling driver + +`createMemoryQueue()` is a queue in memory for tests and a single process, +with `list()`, `get(id)`, `drain()` and `wake()`, and `now` and `sleep` for a +clock of the tests. It is the polling driver over a source in memory: +`createPollingDriver(source)` makes a driver of any `QueueSource`, a store that +lends its jobs (`reserve`, `extend`, `ack`, `retry`, `fail`, `release`), such +as an outbox in your database. It keeps the lease of a job while it runs, gives +a job whose lease ran out to another worker as `redelivered`, and on `close()` +waits for the jobs in flight, then aborts them. diff --git a/docs/guide/why.md b/docs/guide/why.md index cb68c36..1e8769f 100644 --- a/docs/guide/why.md +++ b/docs/guide/why.md @@ -3,8 +3,9 @@ ::: tip Status Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0, and 1.1 adds what nodemailer, unemail and sently did better, delivery events and the decorators of a -closed loop. The queue and the audit log are being designed: the direction below is settled, their -API is not. +closed loop. Unreleased on `main`: the [queue](/guide/queue) with its BullMQ driver, +[templates by name](/messages/templates) and the [audit](/guide/audit) of the states of a message; +the MongoDB outbox and suppression are next. ::: ## Two parts @@ -19,8 +20,9 @@ of two parts: signing, Amazon SES v2 (signed with SigV4, without the AWS SDK) and seventeen more HTTP providers on `fetch`, JMAP, and decorators for failover, retries, telemetry and logs. 2. **A reliability layer on top of it,** in part here. One model of delivery events read from - verified webhooks, idempotency keys, a circuit breaker and rate limits came in 1.1; a persistent - queue and outbox, an append-only audit log and suppression are being designed. + verified webhooks, idempotency keys, a circuit breaker and rate limits came in 1.1; the queue, + templates and the audit of states are unreleased on `main`; an outbox in MongoDB, the + crypto-shredding of the audit and suppression are next. ## The gap it fills diff --git a/docs/index.md b/docs/index.md index 9e23f7e..ce36f14 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,7 +7,7 @@ titleTemplate: Email for Node.js, with a reliability layer hero: name: mailer text: Email for Node.js, with a reliability layer - tagline: SMTP, sendmail, MIME and DKIM of its own, eighteen HTTP providers and JMAP, with failover, retries, rate limits and idempotency around them, and verified webhooks into one model of delivery events. Zero dependencies. Next on top, a persistent queue and an audit log. + tagline: SMTP, sendmail, MIME and DKIM of its own, eighteen HTTP providers and JMAP, with failover, retries, rate limits and idempotency around them, and verified webhooks into one model of delivery events. Zero dependencies. On top, a persistent queue with templates of your engine and an audit of every state. image: src: /logo-mark.svg alt: mailer @@ -45,14 +45,14 @@ features: linkText: Pool, retry, breaker - icon: 🪶 title: Zero dependencies - details: "Built on node:* and fetch. The OpenTelemetry providers, the logger, React Email's render and, later, the queue driver come from your application. Node.js 22+, tested on Bun and Deno, CommonJS with ESM imports, hand-written types." + details: "Built on node:* and fetch. The OpenTelemetry providers, the logger, React Email's render, your template engine and BullMQ come from your application. Node.js 22+, tested on Bun and Deno, CommonJS with ESM imports, hand-written types." link: /api/exports linkText: API - icon: 🧾 - title: Delivery events, then the reliability layer - details: "The webhooks of fifteen providers, each verified before it is read, give one model of delivery events. In design: a persistent queue and outbox, an append-only audit log and suppression. None of the fourteen libraries compared has the three." - link: /guide/webhooks - linkText: Webhooks and events + title: The reliability layer + details: "A queue on BullMQ or a store of yours, whose workers render your templates, fail a permanent failure at once and retry a temporary one after Retry-After; the webhooks of fifteen providers in one model of delivery events; an audit of every state. None of the fourteen libraries compared has them." + link: /guide/queue + linkText: The queue --- ::: tip Status @@ -60,8 +60,9 @@ The transport layer is here, ported from [upyo](https://github.com/dahlia/upyo) what [nodemailer](https://github.com/nodemailer/nodemailer), [unemail](https://github.com/productdevbook/unemail) and [sently](https://github.com/omqkhafi/sently) did better ported in 1.1: one package, with a subpath -per transport and decorator. Delivery events are here too; the queue and the audit log are being -designed. See [Why @alexify/mailer?](/guide/why) for the plan. +per transport and decorator. Delivery events are here too, and, unreleased, the +[queue](/guide/queue), [templates](/messages/templates) and the [audit](/guide/audit). See +[Why @alexify/mailer?](/guide/why) for the plan. ::: ## A quick look diff --git a/docs/messages/render.md b/docs/messages/render.md index 03af0d0..db17c26 100644 --- a/docs/messages/render.md +++ b/docs/messages/render.md @@ -49,6 +49,26 @@ const content = await renderContent(markdown, '# Hello\n\nYour order has shipped When a renderer gives HTML without text, `renderContent` derives the text with `htmlToText`; `{ autoText: false }` leaves it without one. +## The subject + +A renderer may give the subject of the message with its content, as the +templates of [`./templates`](/messages/templates) do from their front matter. +`renderContent` leaves it out; `renderFields` gives `{ subject, content }`, the +content rendered as `renderContent` renders it: + +```js +const { renderFields } = require('@alexify/mailer/render'); + +const { subject, content } = await renderFields(templates, { + template: 'welcome', + context: { name: user.name }, +}); +const message = createMessage({ from: 'shop@example.com', to: user.email, subject, content }); +``` + +`subject` is undefined when the renderer gives none; a subject that is not a +string fails the render. + ## htmlToText `htmlToText(html)` gives the plain text an HTML body reads as, for the text diff --git a/docs/messages/templates.md b/docs/messages/templates.md new file mode 100644 index 0000000..ac224b3 --- /dev/null +++ b/docs/messages/templates.md @@ -0,0 +1,250 @@ +# Templates by name + +`@alexify/mailer/templates` renders a template by its name, with the engine you +already use: EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache or +React Email. The package has no engine of its own and depends on none of them: +you pass the engine in, and the adapter reads its templates through a loader, +caches what it compiles, applies a layout, inlines the CSS, gives the subject +from the template, and derives the text from the HTML. + +A template renderer is a [`Renderer`](/messages/render) of `./render`, so +`renderContent` and `renderFields` take it, and so does the worker of the +[queue](/guide/queue). + +## Quick start + +```js +const path = require('node:path'); +const ejs = require('ejs'); +const { createMessage } = require('@alexify/mailer'); +const { renderFields } = require('@alexify/mailer/render'); +const { + createEjsEngine, + createFileLoader, + createTemplateRenderer, +} = require('@alexify/mailer/templates'); + +const templates = createTemplateRenderer({ + engine: createEjsEngine({ ejs }), + loader: createFileLoader({ dir: path.join(__dirname, 'emails'), extensions: ['.ejs'] }), + layout: 'layouts/letter', + globals: { appUrl: 'https://app.example.com' }, +}); + +const { subject, content } = await renderFields(templates, { + template: 'reset-password', + context: { name: user.name, url }, + locale: user.locale, +}); +const message = createMessage({ from: 'no-reply@example.com', to: user.email, subject, content }); +``` + +`emails/reset-password.ejs`: + +```ejs +--- +subject: Reset your password, <%= name %> +--- +

Hi <%= name %>,

+

Choose a new password

+``` + +`emails/layouts/letter.ejs`, with the header and footer every letter shares: + +```ejs + + + + <%- body %> +
…
+ + +``` + +`render()` gives `{ html, text, subject, template, locale }`: the text is +derived from the HTML unless a text template is there, and `template` names the +file the message came from. + +## Names, variants and locales + +A template is asked for by a **name**, never a path: `welcome`, +`auth/reset-password`. A name has no `..`, no leading `/` and no `\`; the file +loader also reads nothing a symbolic link leads out of its directory to. + +Each part of a message may have a file of its own, next to the HTML: + +| File | Part | +| --- | --- | +| `welcome.hbs` | the HTML | +| `welcome.text.hbs` | the text, in place of the one derived from the HTML | +| `welcome.subject.hbs` | the subject, when the front matter gives none | +| `welcome.uk.hbs`, `welcome.uk.text.hbs`, … | the same in Ukrainian | + +A locale is looked up with its subtags dropped one by one, then +`defaultLocale` the same way, then the template without a locale: `uk-UA` reads +`welcome.uk-UA.hbs`, `welcome.uk.hbs`, then `welcome.hbs`. The text and subject +are taken at the locale the HTML was found at, so that a Ukrainian body never +goes with an English subject. `variantName(template, { locale, part })` names +the variants another way. + +A name with the extension of an engine (`welcome.hbs`) is that engine's alone; +without one, the engines are tried in their order. + +## Front matter + +A template may start with its subject and layout: + +```hbs +--- +subject: "Welcome, {{name}}" +layout: layouts/marketing +--- +

…

+``` + +The subject is rendered with the template's engine and context, its entities +decoded and its whitespace collapsed. `layout: none` renders the template +without the renderer's `layout`. Any other key is an error, so a typo does not +pass unnoticed. `frontMatter: false` reads none. + +## Layouts, partials and includes + +`layout` names the layout of every HTML template; it is rendered with the +context and `body`, the HTML of the template, raw. A layout has no layout of +its own. Engines that have layouts of their own (Pug's `extends`, Nunjucks's +and Liquid's blocks, Handlebars partial blocks, Eta's `layout()`) keep them. + +Every include, partial and layout an engine reads goes through the loader of +the renderer: + +- `/x` is read from the root of the loader; +- `./x` and `../x` against the template it is written in (EJS: the template + rendered, as EJS tells no more); +- any other name from the root, then from each directory of `partials`; +- the extensions of the engine are added when the name has none. + +A name that leads out of the root is not found. + +## Engines + +::: v-pre + +Each engine takes the module of its package; what the adapter sets itself (the +file name includes are read by, the cache, a file system) is refused as an +option rather than accepted and ignored. + +| Engine | Made with | Extensions | Escapes by default | +| --- | --- | --- | --- | +| EJS | `createEjsEngine({ ejs: require('ejs'), options: { async } })` | `.ejs` | `<%= %>`; `<%- %>` is raw | +| Handlebars | `createHandlebarsEngine({ handlebars, helpers, configure })` | `.hbs`, `.handlebars` | `{{ }}`; `{{{ }}}` is raw | +| Pug | `createPugEngine({ pug, options: { doctype } })` | `.pug` | `#{}`, `=`; `!{}` is raw | +| MJML | `createMjmlEngine({ mjml2html: require('mjml'), engine })` | `.mjml`, or `.mjml` before the engine's | — | +| Liquid | `createLiquidEngine({ Liquid: require('liquidjs').Liquid, filters })` | `.liquid` | yes (`outputEscape: 'escape'`); `\| raw` | +| Nunjucks | `createNunjucksEngine({ nunjucks, filters, configure })` | `.njk`, `.nunjucks` | yes (`autoescape`) | +| Eta | `createEtaEngine({ Eta: require('eta').Eta, async })` | `.eta` | `<%= %>`; `<%~ %>` is raw | +| Mustache | `createMustacheEngine({ mustache, tags })` | `.mustache` | `{{ }}`; `{{{ }}}` is raw | +| React Email | `createReactEmailEngine({ render, createElement, components })` | — | JSX | + +Several engines go into one renderer with `engines: [...]`, chosen by +extension, the longest first. + +Some engines behave a little differently here than they do on their own: + +- **Handlebars** gets an environment per renderer (`handlebars.create()`), so + its helpers and partials stay out of the global one. The partials a template + names are read when it is compiled. A dynamic partial (`{{> (name)}}`) has to + be registered in `configure`. +- **Liquid** escapes output by default, where liquidjs escapes nothing unless + told. It reads own properties only. +- **Mustache** fails on a partial that is not there, where it renders nothing. + Its writer, and the cache of what it parsed, are per renderer and bounded. +- **MJML** renders after the engine inside it (`.mjml.hbs`), once the layout is + applied, so the layout of an MJML template is MJML too. A static `.mjml` is + rendered once. What MJML finds wrong under `validationLevel: 'soft'` goes to + `onErrors`, or to a warning (`MAILER_TEMPLATE_MJML`); `'strict'` fails the + render. `mj-include` reads files itself, unchecked, so it is off unless + `includes: true`. Use the engine's partials instead. +- **React Email**: its templates are its components, by name (`welcome.uk` for + a locale), each with its subject: + + ```js + const engine = createReactEmailEngine({ + render: require('@react-email/render').render, + createElement: require('react').createElement, + components: { + welcome: { component: Welcome, subject: ({ name }) => `Welcome, ${name}` }, + }, + }); + const templates = createTemplateRenderer({ engine }); + ``` + +Templates are code: EJS, Eta, Pug and Nunjucks run JavaScript or close to it. +For templates your tenants write, use Liquid (with `parseLimit`, `renderLimit` +and `memoryLimit`), Handlebars or Mustache. The context is data. + +::: + +## CSS + +```js +const templates = createTemplateRenderer({ + engine, + loader, + inlineCss: createCssInliner({ inline: require('juice').default }), + // or: createCssInliner({ inline: require('@css-inline/css-inline').inline }) +}); +``` + +The CSS is inlined after MJML and before the text is derived. The inliner is +given `{ loadRemoteStylesheets: false, keepLinkTags: true }` under your +options, so css-inline fetches no stylesheet on a send and keeps the `` +of a web font for the mail clients that read it; juice fetches nothing itself. +css-inline drops `

x

'; + // Before: css-inline fetched every remote stylesheet on each send and + // dropped the of the font. + for (const inliner of [createCssInliner({ inline: juice }), createCssInliner({ inline })]) { + const inlined = await inliner(html); + assert.match(inlined, /

x<\/p>/); + assert.match(inlined, /fonts\.googleapis\.com/); + } +}); diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js new file mode 100644 index 0000000..2eec3e6 --- /dev/null +++ b/tests/unit/templates-loader.test.js @@ -0,0 +1,131 @@ +// Not ported: tests of the loaders of ./templates, on a directory of their +// own under the system's temporary directory. +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { + createFileLoader, + createMemoryLoader, + createTemplateRenderer, + createEjsEngine, +} = require('../../src/templates/index.js'); +const ejs = require('ejs'); + +const makeTree = (t) => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-templates-')); + t.after(() => fs.rmSync(base, { recursive: true, force: true })); + const root = path.join(base, 'emails'); + fs.mkdirSync(path.join(root, 'auth'), { recursive: true }); + fs.mkdirSync(path.join(root, 'folder.ejs')); + fs.writeFileSync(path.join(root, 'welcome.ejs'), '

Hi <%= name %>

'); + fs.writeFileSync(path.join(root, 'auth', 'reset.ejs'), '

reset

'); + fs.writeFileSync(path.join(root, 'notes.txt'), 'not a template'); + fs.writeFileSync(path.join(root, 'big.ejs'), 'x'.repeat(2048)); + fs.writeFileSync(path.join(base, 'secret.ejs'), 'secret'); + return { base, root }; +}; + +test('Templates: the file loader reads a name under its directory, its BOM dropped', async (t) => { + const { root } = makeTree(t); + const loader = createFileLoader({ dir: root, extensions: ['.ejs'], maxBytes: 1024 }); + const welcome = await loader.load('welcome.ejs'); + assert.strictEqual(welcome.name, 'welcome.ejs'); + assert.strictEqual(welcome.source, '

Hi <%= name %>

'); + assert.strictEqual(welcome.path, fs.realpathSync(path.join(root, 'welcome.ejs'))); + assert.strictEqual((await loader.load('auth/reset.ejs')).source, '

reset

'); + assert.strictEqual(loader.loadSync('auth/reset.ejs').source, '

reset

'); + // Missing, a directory, another extension and a name that is not one: none. + for (const name of [ + 'missing.ejs', + 'folder.ejs', + 'notes.txt', + '../secret.ejs', + 'auth/../welcome.ejs', + '/etc/passwd', + ]) { + assert.strictEqual(await loader.load(name), undefined, name); + assert.strictEqual(loader.loadSync(name), undefined, name); + } + assert.strictEqual(await loader.load('welcome.ejs/x.ejs'), undefined); + assert.strictEqual(loader.loadSync('welcome.ejs/x.ejs'), undefined); + // Over maxBytes. + await assert.rejects(loader.load('big.ejs'), /more than 1024 bytes/); + assert.throws(() => loader.loadSync('big.ejs'), /more than 1024 bytes/); + // Any extension by default. + assert.strictEqual( + (await createFileLoader({ dir: root }).load('notes.txt')).source, + 'not a template', + ); +}); + +test('Templates: the file loader follows no symbolic link out of its directory', async (t) => { + if (process.platform === 'win32') { + t.skip('symbolic links need privileges on Windows'); + return; + } + const { base, root } = makeTree(t); + fs.symlinkSync(path.join(base, 'secret.ejs'), path.join(root, 'link.ejs')); + fs.symlinkSync(path.join(root, 'welcome.ejs'), path.join(root, 'inside.ejs')); + const loader = createFileLoader({ dir: root }); + await assert.rejects(loader.load('link.ejs'), /leads out of/); + assert.throws(() => loader.loadSync('link.ejs'), /leads out of/); + assert.strictEqual((await loader.load('inside.ejs')).source, '

Hi <%= name %>

'); + // A directory reached through a link is its real path. + const linked = path.join(base, 'linked'); + fs.symlinkSync(root, linked); + assert.strictEqual( + (await createFileLoader({ dir: linked }).load('welcome.ejs')).name, + 'welcome.ejs', + ); +}); + +test('Templates: the file loader surfaces what is not a missing file', async (t) => { + const { root } = makeTree(t); + const missingRoot = createFileLoader({ dir: path.join(root, 'nope') }); + await assert.rejects(missingRoot.load('welcome.ejs'), { code: 'ENOENT' }); + assert.throws(() => missingRoot.loadSync('welcome.ejs'), { code: 'ENOENT' }); + // A loader made before its directory exists reads it once it does. + const later = path.join(root, 'later'); + const early = createFileLoader({ dir: later }); + fs.mkdirSync(later); + fs.writeFileSync(path.join(later, 'a.ejs'), 'a'); + assert.strictEqual((await early.load('a.ejs')).source, 'a'); + if (process.platform !== 'win32') { + const loop = path.join(root, 'loop.ejs'); + fs.symlinkSync(loop, loop); + const loader = createFileLoader({ dir: root }); + await assert.rejects(loader.load('loop.ejs'), { code: 'ELOOP' }); + assert.throws(() => loader.loadSync('loop.ejs'), { code: 'ELOOP' }); + } +}); + +test('Templates: the loaders check their options', () => { + assert.throws(() => createFileLoader(), /needs dir/); + assert.throws(() => createFileLoader({ dir: '' }), /needs dir/); + assert.throws(() => createFileLoader({ dir: 'x', extensions: '.ejs' }), /must be an array/); + assert.throws(() => createFileLoader({ dir: 'x', extensions: ['ejs'] }), /with their dot/); + assert.throws(() => createFileLoader({ dir: 'x', maxBytes: 0 }), /maxBytes/); + assert.throws(() => createMemoryLoader(), /needs the templates/); + assert.throws(() => createMemoryLoader([]), /needs the templates/); + assert.throws(() => createMemoryLoader({ '../x': 'a' }), /template name/); + const memory = createMemoryLoader({ a: '

', 'b/c': { component: true } }); + assert.deepStrictEqual(memory.load('a'), { name: 'a', source: '

' }); + assert.deepStrictEqual(memory.loadSync('b/c'), { name: 'b/c', source: { component: true } }); + assert.strictEqual(memory.load('z'), undefined); +}); + +test('Templates: a renderer over the file loader renders from disk', async (t) => { + const { root } = makeTree(t); + const renderer = createTemplateRenderer({ + engine: createEjsEngine({ ejs }), + loader: createFileLoader({ dir: root, extensions: ['.ejs'] }), + }); + const output = await renderer.render({ template: 'welcome', context: { name: 'Ada' } }); + assert.deepStrictEqual(output, { + template: 'welcome.ejs', + html: '

Hi Ada

', + text: 'Hi Ada', + }); +}); diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js new file mode 100644 index 0000000..6b2375b --- /dev/null +++ b/tests/unit/templates-renderer.test.js @@ -0,0 +1,520 @@ +// Not ported: tests of the template renderer of ./templates, against the fake +// engine of tests/helpers/templates. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { renderContent, renderFields } = require('../../src/render/index.js'); +const { + createCssInliner, + createMemoryLoader, + createTemplateRenderer, + TemplateNotFoundError, + TemplateRenderError, +} = require('../../src/templates/index.js'); +const { fakeEngine } = require('../helpers/templates/fake-engine.js'); + +const rendererOf = (templates, options = {}) => { + const engine = options.engine ?? fakeEngine(); + const renderer = createTemplateRenderer({ + engine, + loader: createMemoryLoader(templates), + ...options, + }); + return { renderer, engine }; +}; + +test('Templates: a template renders its HTML, its text derived', async () => { + const { renderer } = rendererOf({ 'welcome.tpl': '

Hi {{name}}

' }); + assert.deepStrictEqual(await renderer.render({ template: 'welcome', context: { name: 'A&B' } }), { + template: 'welcome.tpl', + html: '

Hi A&B

', + text: 'Hi A&B', + }); + // An explicit extension names the engine, and the result is the same. + assert.strictEqual( + (await renderer.render({ template: 'welcome.tpl', context: { name: 'x' } })).html, + '

Hi x

', + ); + const { renderer: noText } = rendererOf({ 'a.tpl': '

x

' }, { autoText: false }); + assert.deepStrictEqual(await noText.render({ template: 'a' }), { + template: 'a.tpl', + html: '

x

', + }); +}); + +test('Templates: the subject comes from the front matter, the text from its variant', async () => { + const { renderer } = rendererOf({ + 'reset.tpl': '---\n# a comment\nsubject: "Reset, {{name}} & co"\n---\n

{{url}}

', + 'reset.text.tpl': 'Open {{url}} & go', + }); + const output = await renderer.render({ + template: 'reset', + context: { name: 'Ada', url: 'https://example.com/r?a=1&b=2' }, + }); + assert.strictEqual(output.subject, 'Reset, Ada & co'); + assert.strictEqual(output.text, 'Open https://example.com/r?a=1&b=2 & go'); + assert.strictEqual(output.html, '

https://example.com/r?a=1&b=2

'); + // Its subject variant, its whitespace collapsed. + const { renderer: variant } = rendererOf({ + 'a.tpl': '

x

', + 'a.subject.tpl': ' Hello\n {{name}}\n', + }); + assert.strictEqual( + (await variant.render({ template: 'a', context: { name: 'B' } })).subject, + 'Hello B', + ); + // renderFields and renderContent take the renderer. + assert.deepStrictEqual( + await renderFields(renderer, { template: 'reset', context: { url: 'u' } }), + { + subject: 'Reset, & co', + content: { html: '

u

', text: 'Open u & go' }, + }, + ); + assert.deepStrictEqual(await renderContent(variant, { template: 'a' }), { + html: '

x

', + text: 'x', + }); +}); + +test('Templates: a template of text alone', async () => { + const { renderer } = rendererOf({ + 'note.text.tpl': '---\nsubject: Note\n---\nHi {{name}} & all', + }); + assert.deepStrictEqual(await renderer.render({ template: 'note', context: { name: '' } }), { + template: 'note.text.tpl', + text: 'Hi & all', + subject: 'Note', + }); +}); + +test('Templates: locales fall back by subtag, then the default, then none', async () => { + const { renderer } = rendererOf( + { + 'welcome.tpl': '---\nsubject: Welcome\n---\n

Hi

', + 'welcome.uk.tpl': '---\nsubject: Вітаємо\n---\n

Привіт

', + 'welcome.de.tpl': '

Hallo

', + }, + { defaultLocale: 'de-AT' }, + ); + const uk = await renderer.render({ template: 'welcome', locale: 'uk-UA' }); + assert.deepStrictEqual([uk.template, uk.locale, uk.subject], ['welcome.uk.tpl', 'uk', 'Вітаємо']); + const fr = await renderer.render({ template: 'welcome', locale: 'fr' }); + assert.deepStrictEqual([fr.template, fr.locale], ['welcome.de.tpl', 'de']); + // The subject is that of the locale the body was found at: none here, not + // the English one. + assert.strictEqual(fr.subject, undefined); + const { renderer: plain } = rendererOf({ 'welcome.tpl': '

Hi

' }); + const none = await plain.render({ template: 'welcome', locale: 'EN-us' }); + assert.deepStrictEqual([none.template, none.locale], ['welcome.tpl', undefined]); + await assert.rejects(plain.render({ template: 'welcome', locale: 'not a tag' }), RangeError); +}); + +test('Templates: a missing template names what was tried, and is not retryable', async () => { + const { renderer } = rendererOf({}, { defaultLocale: 'en' }); + const error = await renderer.render({ template: 'nope', locale: 'uk' }).catch((e) => e); + assert.ok(error instanceof TemplateNotFoundError); + assert.strictEqual(error.code, 'MAILER_TEMPLATE_NOT_FOUND'); + assert.strictEqual(error.retryable, false); + assert.deepStrictEqual(error.tried, [ + 'nope.uk.tpl', + 'nope.en.tpl', + 'nope.tpl', + 'nope.uk.text.tpl', + 'nope.en.text.tpl', + 'nope.text.tpl', + ]); + const many = new TemplateNotFoundError('x', ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10']); + assert.match(many.message, /8, and 2 more|8 and 2 more/); +}); + +test('Templates: a name is never a path', async () => { + const { renderer } = rendererOf({}); + for (const template of ['../secret', '/etc/passwd', 'a//b', 'a/./b', 'a\\b', '', '.hidden', 42]) { + await assert.rejects(renderer.render({ template }), TypeError, String(template)); + } + await assert.rejects(renderer.render(null), TypeError); + await assert.rejects(renderer.render({ template: 'a', context: [] }), TypeError); + await assert.rejects(renderer.render({ template: 'a', locale: 1 }), TypeError); +}); + +test('Templates: includes are read through the loader, from the root or the partials', async () => { + const { renderer } = rendererOf( + { + 'emails/welcome.tpl': '{{> header}}

{{name}}

{{> ./sign}}{{> /legal}}', + 'partials/header.tpl': '
', + 'emails/sign.tpl': 'sign', + 'legal.tpl': 'legal', + }, + { partials: ['partials'] }, + ); + assert.strictEqual( + (await renderer.render({ template: 'emails/welcome', context: { name: 'x' } })).html, + '

x

signlegal', + ); + const { renderer: out } = rendererOf({ 'a.tpl': '{{> ../../etc/passwd}}' }); + const error = await out.render({ template: 'a' }).catch((e) => e); + assert.ok(error instanceof TemplateNotFoundError); + assert.deepStrictEqual(error.tried, []); + assert.strictEqual(error.from, 'a.tpl'); +}); + +test('Templates: a layout wraps the HTML, opted out or named by front matter', async () => { + const templates = { + 'layouts/main.tpl': '{{title}}{{{body}}}', + 'layouts/alt.tpl': '{{{body}}}', + 'a.tpl': '

{{name}}

', + 'b.tpl': '---\nlayout: none\n---\n

b

', + 'c.tpl': '---\nlayout: layouts/alt\n---\n

c

', + 'd.tpl': '---\nlayout: layouts/missing\n---\n

d

', + 'layouts/nested.tpl': '---\nlayout: layouts/main\n---\n{{{body}}}', + 'e.tpl': '---\nlayout: layouts/nested\n---\n

e

', + 'f.text.tpl': 'text alone', + }; + const { renderer } = rendererOf(templates, { + layout: 'layouts/main', + globals: { title: 'T' }, + }); + assert.strictEqual( + (await renderer.render({ template: 'a', context: { name: 'x' } })).html, + 'T

x

', + ); + assert.strictEqual((await renderer.render({ template: 'b' })).html, '

b

'); + assert.strictEqual((await renderer.render({ template: 'c' })).html, '

c

'); + const missing = await renderer.render({ template: 'd' }).catch((e) => e); + assert.ok(missing instanceof TemplateNotFoundError); + assert.strictEqual(missing.from, 'd.tpl'); + const nested = await renderer.render({ template: 'e' }).catch((e) => e); + assert.ok(nested instanceof TemplateRenderError); + assert.strictEqual(nested.phase, 'layout'); + // A template of text alone takes no layout. + assert.deepStrictEqual(await renderer.render({ template: 'f' }), { + template: 'f.text.tpl', + text: 'text alone', + }); + // A layout of text alone has no HTML to wrap with. + const { renderer: textLayout } = rendererOf( + { 'a.tpl': '

a

', 'l.text.tpl': 'x' }, + { layout: 'l' }, + ); + const noHtml = await textLayout.render({ template: 'a' }).catch((e) => e); + assert.strictEqual(noHtml.phase, 'layout'); +}); + +test('Templates: globals sit under the context, per input when a function', async () => { + const { renderer } = rendererOf( + { 'a.tpl': '{{who}} {{site}} {{locale}}' }, + { globals: async (input) => ({ who: 'all', site: 'S', locale: input.locale ?? '-' }) }, + ); + assert.strictEqual( + (await renderer.render({ template: 'a', context: { who: 'me' }, locale: 'uk' })).html, + 'me S uk', + ); + const { renderer: bad } = rendererOf({ 'a.tpl': 'x' }, { globals: () => 42 }); + await assert.rejects(bad.render({ template: 'a' }), /globals must give an object/); + // A failure of globals, such as an i18n backend that is down, stays as it is. + const down = new Error('i18n down'); + const { renderer: failing } = rendererOf( + { 'a.tpl': 'x' }, + { globals: () => Promise.reject(down) }, + ); + assert.strictEqual(await failing.render({ template: 'a' }).catch((e) => e), down); + const { renderer: none } = rendererOf({ 'a.tpl': 'x{{y}}' }, { globals: () => undefined }); + assert.strictEqual((await none.render({ template: 'a', context: { y: 1 } })).html, 'x1'); +}); + +test('Templates: each template is compiled once, concurrent renders included', async () => { + const engine = fakeEngine({ async: true }); + const { renderer } = rendererOf( + { 'a.tpl': '---\nsubject: S {{n}}\n---\n

{{n}}

{{> p}}', 'p.tpl': '' }, + { engine }, + ); + const outputs = await Promise.all([ + renderer.render({ template: 'a', context: { n: 1 } }), + renderer.render({ template: 'a', context: { n: 2 } }), + renderer.render({ template: 'a', context: { n: 3 } }), + ]); + assert.deepStrictEqual( + outputs.map((output) => output.subject), + ['S 1', 'S 2', 'S 3'], + ); + assert.strictEqual(engine.compiled.length, 2, 'the body and the subject, once'); + renderer.clear(); + await renderer.render({ template: 'a', context: { n: 4 } }); + assert.strictEqual(engine.compiled.length, 4); + // Without a cache, each render compiles anew. + const fresh = fakeEngine(); + const { renderer: uncached } = rendererOf({ 'a.tpl': '

' }, { engine: fresh, cache: false }); + await uncached.render({ template: 'a' }); + await uncached.render({ template: 'a' }); + assert.strictEqual(fresh.compiled.length, 2); + // A cache of one entry drops the oldest. + const small = fakeEngine(); + const { renderer: bounded } = rendererOf( + { 'a.tpl': 'a', 'b.tpl': 'b' }, + { engine: small, cache: { maxEntries: 1 } }, + ); + await bounded.render({ template: 'a' }); + await bounded.render({ template: 'b' }); + await bounded.render({ template: 'a' }); + assert.strictEqual(small.compiled.length, 3); +}); + +test('Templates: a template that is missing at first is found once it is there', async () => { + const sources = Object.create(null); + const loader = { + load: (name) => (name in sources ? { name, source: sources[name] } : undefined), + }; + const renderer = createTemplateRenderer({ engine: fakeEngine(), loader }); + await assert.rejects(renderer.render({ template: 'late' }), TemplateNotFoundError); + sources['late.tpl'] = '

late

'; + renderer.clear(); + assert.strictEqual((await renderer.render({ template: 'late' })).html, '

late

'); +}); + +test('Templates: failures are classified by phase, the engine error their cause', async () => { + const { renderer } = rendererOf({ + 'syntax.tpl': '

{{!syntax

', + 'boom.tpl': '

{{x}}

', + 'front.tpl': '---\nsubjct: typo\n---\n

', + 'open.tpl': '---\nsubject: x\n', + 'line.tpl': '---\nnot a pair\n---\n', + 'twice.tpl': '---\nsubject: a\nsubject: b\n---\n', + }); + const syntax = await renderer.render({ template: 'syntax' }).catch((e) => e); + assert.ok(syntax instanceof TemplateRenderError); + assert.deepStrictEqual( + [syntax.phase, syntax.engine, syntax.code], + ['compile', 'fake', 'MAILER_TEMPLATE_RENDER'], + ); + assert.ok(syntax.cause instanceof SyntaxError); + assert.strictEqual(syntax.retryable, false); + const boom = await renderer + .render({ template: 'boom', context: { explode: true } }) + .catch((e) => e); + assert.strictEqual(boom.phase, 'render'); + assert.match(boom.message, /A helper exploded/); + for (const [template, pattern] of [ + ['front', /no key "subjct"/], + ['open', /no closing ---/], + ['line', /not "key: value"/], + ['twice', /subject twice/], + ]) { + const error = await renderer.render({ template }).catch((e) => e); + assert.strictEqual(error.phase, 'front-matter', template); + assert.match(error.message, pattern); + } +}); + +test('Templates: a loader that fails fails with its own error, which may pass on a retry', async () => { + const outage = Object.assign(new Error('EMFILE: too many open files'), { code: 'EMFILE' }); + const loader = { + load: (name) => { + if (name === 'a.tpl') return { name, source: '{{> b}}' }; + throw outage; + }, + }; + const renderer = createTemplateRenderer({ engine: fakeEngine(), loader }); + assert.strictEqual(await renderer.render({ template: 'a' }).catch((e) => e), outage); + const failing = createTemplateRenderer({ + engine: fakeEngine(), + loader: { load: () => Promise.reject(outage) }, + }); + assert.strictEqual(await failing.render({ template: 'x' }).catch((e) => e), outage); +}); + +test('Templates: finish makes the HTML of the outermost template, then the CSS is inlined', async () => { + const order = []; + const engine = fakeEngine({ + finish: (html, context) => { + order.push(`finish ${context.name}`); + return html.replace('', ''); + }, + }); + const inline = (html, options) => { + order.push(`inline ${JSON.stringify(options)}`); + return html.replace('

', '

'); + }; + const { renderer } = rendererOf( + { 'a.tpl': '

a

', 'layout.tpl': '{{{body}}}' }, + { engine, layout: 'layout', inlineCss: createCssInliner({ inline }) }, + ); + const output = await renderer.render({ template: 'a' }); + assert.strictEqual(output.html, '

a

'); + assert.deepStrictEqual(order, [ + 'finish layout.tpl', + 'inline {"loadRemoteStylesheets":false,"keepLinkTags":true}', + ]); + // A failure of the inliner rejects once, as a failure of its phase. + const { renderer: failing } = rendererOf( + { 'a.tpl': '

' }, + { + inlineCss: createCssInliner({ + inline: () => { + throw new Error('bad css'); + }, + }), + }, + ); + const error = await failing.render({ template: 'a' }).catch((e) => e); + assert.strictEqual(error.phase, 'inline-css'); + const { renderer: notString } = rendererOf( + { 'a.tpl': '

' }, + { inlineCss: createCssInliner({ inline: () => 1 }) }, + ); + assert.strictEqual( + (await notString.render({ template: 'a' }).catch((e) => e)).phase, + 'inline-css', + ); + const { renderer: overridden } = rendererOf( + { 'a.tpl': '

' }, + { + inlineCss: createCssInliner({ + inline: (html, o) => `${html}${o.keepLinkTags}`, + options: { keepLinkTags: false }, + }), + }, + ); + assert.strictEqual((await overridden.render({ template: 'a' })).html, '

false'); +}); + +test('Templates: an engine that renders its parts gives HTML, text and subject at once', async () => { + const parts = { + name: 'parts', + extensions: [], + compile: (source) => (scope) => ({ + html: `${scope.n}`, + text: `t${scope.n}`, + subject: source, + }), + }; + const renderer = createTemplateRenderer({ + engine: parts, + loader: createMemoryLoader({ welcome: 'Subject here' }), + layout: 'ignored', + }); + assert.deepStrictEqual(await renderer.render({ template: 'welcome', context: { n: 1 } }), { + template: 'welcome', + html: '1', + text: 't1', + subject: 'Subject here', + }); + const empty = createTemplateRenderer({ + engine: { ...parts, compile: () => () => ({}) }, + loader: createMemoryLoader({ x: '' }), + }); + assert.match( + (await empty.render({ template: 'x' }).catch((e) => e)).message, + /neither html nor text/, + ); + const wrong = createTemplateRenderer({ + engine: { ...parts, compile: () => () => ({ html: 1 }) }, + loader: createMemoryLoader({ x: '' }), + }); + assert.match( + (await wrong.render({ template: 'x' }).catch((e) => e)).message, + /html as no string/, + ); + const nothing = createTemplateRenderer({ + engine: { ...parts, compile: () => () => 42 }, + loader: createMemoryLoader({ x: '' }), + }); + assert.match((await nothing.render({ template: 'x' }).catch((e) => e)).message, /gave no string/); + const textNotString = createTemplateRenderer({ + engine: { + name: 'x', + extensions: ['.x'], + compile: (source, context) => () => (context.part === 'text' ? 1 : '

'), + }, + loader: createMemoryLoader({ 'a.x': '', 'a.text.x': '' }), + }); + const notText = await textNotString.render({ template: 'a' }).catch((e) => e); + assert.match(notText.message, /gave no string/); + assert.strictEqual(notText.template, 'a.text.x'); + const notFunction = createTemplateRenderer({ + engine: { ...parts, compile: () => 'nope' }, + loader: createMemoryLoader({ x: '' }), + }); + assert.match( + (await notFunction.render({ template: 'x' }).catch((e) => e)).message, + /gave no function/, + ); +}); + +test('Templates: engines are chosen by extension, the longest first', async () => { + const hbs = fakeEngine({ name: 'hbs', extensions: ['.hbs'] }); + const mjml = fakeEngine({ name: 'mjml', extensions: ['.mjml.hbs'] }); + const renderer = createTemplateRenderer({ + engines: [hbs, mjml], + loader: createMemoryLoader({ 'a.mjml.hbs': 'mjml', 'b.hbs': 'hbs', 'a.hbs': 'shadow' }), + }); + assert.strictEqual((await renderer.render({ template: 'a.mjml.hbs' })).html, 'mjml'); + assert.strictEqual((await renderer.render({ template: 'b' })).html, 'hbs'); + // Without an extension, the engines are tried in their order. + assert.strictEqual((await renderer.render({ template: 'a' })).html, 'shadow'); + const text = await renderer.render({ template: 'b', context: {} }); + assert.strictEqual(text.text, 'hbs'); +}); + +test('Templates: the renderer checks its options', () => { + const engine = fakeEngine(); + const loader = createMemoryLoader({}); + const bad = [ + [{}, /needs an engine/], + [{ engine, engines: [engine], loader }, /not both/], + [{ engines: [], loader }, /needs an engine/], + [{ engine: {}, loader }, /must be a template engine/], + [{ engine: { compile() {}, extensions: [] }, loader }, /must have a name/], + [{ engine: { name: 'x', compile() {} }, loader }, /must list its extensions/], + [{ engines: [engine, fakeEngine()], loader }, /Two engines take .tpl/], + [ + { engines: [engine, { name: 'b', extensions: [], compile() {} }], loader }, + /must be the only one/, + ], + [{ engine }, /needs a loader/], + [{ engine, loader, partials: 'p' }, /partials must be an array/], + [{ engine, loader, partials: ['../p'] }, /directory of partials/], + [{ engine, loader, layout: '/abs' }, /layout must be a template name/], + [{ engine, loader, globals: 1 }, /globals must be an object or a function/], + [{ engine, loader, frontMatter: 'yes' }, /frontMatter must be a boolean/], + [{ engine, loader, autoText: 1 }, /autoText must be a boolean/], + [{ engine, loader, cache: 'yes' }, /cache must be a boolean/], + [{ engine, loader, cache: { maxEntries: 0 } }, /cache.maxEntries/], + [{ engine, loader, variantName: 1 }, /variantName must be a function/], + [{ engine, loader, inlineCss: 1 }, /inlineCss must be a function/], + [{ engine, loader, defaultLocale: 1 }, /defaultLocale must be a string/], + ]; + for (const [options, pattern] of bad) { + assert.throws(() => createTemplateRenderer(options), pattern, String(pattern)); + } + assert.throws(() => createTemplateRenderer(), /needs an engine/); + assert.throws(() => createCssInliner({}), /needs inline/); + assert.throws(() => createCssInliner({ inline() {}, options: 1 }), /must be an object/); +}); + +test('Templates: variantName names the variants, frontMatter: false reads none', async () => { + const { renderer } = rendererOf( + { 'welcome-uk-html.tpl': '

uk

', 'welcome-uk-subject.tpl': 'S' }, + { variantName: (template, { locale, part }) => `${template}-${locale ?? 'x'}-${part}` }, + ); + const output = await renderer.render({ template: 'welcome', locale: 'uk' }); + assert.deepStrictEqual([output.html, output.subject], ['

uk

', 'S']); + const { renderer: broken } = rendererOf({}, { variantName: () => 1 }); + await assert.rejects(broken.render({ template: 'a' }), /variantName must give a name/); + const { renderer: raw } = rendererOf( + { 'a.tpl': '---\nsubject: x\n---\n

' }, + { frontMatter: false }, + ); + const plain = await raw.render({ template: 'a' }); + assert.strictEqual(plain.subject, undefined); + assert.match(plain.html, /^---/); + // A subject that is only whitespace is none. + const { renderer: blank } = rendererOf({ 'a.tpl': '

', 'a.subject.tpl': ' ' }); + assert.strictEqual((await blank.render({ template: 'a' })).subject, undefined); + // A BOM before the front matter, and CRLF lines. + const { renderer: bom } = rendererOf({ 'a.tpl': '---\r\nsubject: "B"\r\n---\r\n

' }); + assert.strictEqual((await bom.render({ template: 'a' })).subject, 'B'); + // A first line that only starts with --- is no front matter. + const { renderer: rule } = rendererOf({ 'a.tpl': '----\n

', 'b.tpl': '---' }); + assert.strictEqual((await rule.render({ template: 'a' })).html, '----\n

'); + assert.strictEqual((await rule.render({ template: 'b' })).html, '---'); +}); From 29f3df73b0ad8d09bcb7714257c73daa32ed5a6c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 22:10:05 +0300 Subject: [PATCH 03/83] feat(mongodb): a transactional outbox, with its audit sink and idempotency store ./mongodb keeps the mail queue in a collection of the MongoDB driver the application passes in, so a queue needs no Redis, and a job is written in the transaction of its cause: createMongoOutbox({ collection }) is a QueueSource, read by the polling driver of ./queue. - A job enqueued with { transaction: session } is written in it by an upsert, so a duplicate under its key does not abort the transaction, as an insert failing on the key would. - A job is lent by one atomic findOneAndUpdate (priority, then time) under a token every later write is fenced by; a worker whose lease ran out writes nothing, and its job goes to another as redelivered, its attempt run again. - A rate limit of the whole queue pauses every worker through a document of its own; changeStream: true wakes the workers on a replica set. - ensureIndexes() makes the indexes of the ready jobs and the leases, and a TTL of finished jobs, which is the window of duplicates. createMongoAuditSink writes each event once by its id, in the transaction it is given, and orders the events of one time and attempt by their state (rankOf in ./audit). createMongoIdempotencyStore makes each step of IdempotencyTransport atomic across processes: a free key is taken by an upsert only a free key meets, so of concurrent reserves one holds it. The driver is a devDependency only: a collection in memory for the unit suite, tsd against mongodb 7, and pnpm test:mongodb on a replica set of one (transactions committed and aborted, workers of several processes, leases, concurrent reserves, the change stream). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + CLAUDE.md | 20 +- README.md | 25 +- docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/comparison.json | 8 +- docs/guide/audit.md | 2 + docs/guide/comparison.md | 8 +- docs/guide/getting-started.md | 4 +- docs/guide/outbox.md | 122 +++++++ docs/guide/queue.md | 7 +- docs/public/llms.txt | 5 +- package.json | 7 + pnpm-lock.yaml | 111 +++++++ scripts/size-budgets.json | 3 +- src/audit/states.js | 10 +- src/mongodb/audit-sink.js | 65 ++++ src/mongodb/collection.js | 64 ++++ src/mongodb/idempotency-store.js | 101 ++++++ src/mongodb/index.d.ts | 146 ++++++++ src/mongodb/index.js | 9 + src/mongodb/outbox.js | 277 ++++++++++++++++ tests/helpers/mongodb/fake-collection.js | 228 +++++++++++++ tests/mongodb/docker-compose.yml | 24 ++ tests/mongodb/outbox.mongodb.test.js | 196 +++++++++++ tests/types/mongodb.test-d.ts | 59 ++++ tests/unit/graph.test.js | 2 + tests/unit/mongodb.test.js | 405 +++++++++++++++++++++++ 28 files changed, 1884 insertions(+), 36 deletions(-) create mode 100644 docs/guide/outbox.md create mode 100644 src/mongodb/audit-sink.js create mode 100644 src/mongodb/collection.js create mode 100644 src/mongodb/idempotency-store.js create mode 100644 src/mongodb/index.d.ts create mode 100644 src/mongodb/index.js create mode 100644 src/mongodb/outbox.js create mode 100644 tests/helpers/mongodb/fake-collection.js create mode 100644 tests/mongodb/docker-compose.yml create mode 100644 tests/mongodb/outbox.mongodb.test.js create mode 100644 tests/types/mongodb.test-d.ts create mode 100644 tests/unit/mongodb.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 46efc5a..921ed83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,16 @@ states of a message. Everything is additive. read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases that spend no attempt. `createBullMQProcessor` and `createBullMQBackoffStrategy` serve a Worker of your own. `pnpm test:bullmq` runs it on BullMQ and Valkey. +- **`./mongodb`**: the transactional outbox, `createMongoOutbox({ collection })`, a + `QueueSource` in a collection of the MongoDB driver passed in: a job enqueued with + `{ transaction: session }` is written in that transaction (by an upsert, so a duplicate does + not abort it), lent by one atomic `findOneAndUpdate` under a token every later write is + fenced by, given to another worker as `redelivered` when its lease runs out, with a pause of + the whole queue and an optional change stream to wake the workers; `ensureIndexes()` makes + its indexes and the TTL of finished jobs. `createMongoAuditSink` (each event once by its id, + in the transaction) and `createMongoIdempotencyStore` (each step atomic, for + `IdempotencyTransport` across processes) go with it. `pnpm test:mongodb` runs them on a + replica set. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/CLAUDE.md b/CLAUDE.md index bed0556..a6b3065 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,6 +51,7 @@ pnpm test:deno # tests/unit and tests/runtime on Deno (deno pnpm test:mailpit # opt-in: tests/mailpit against a local Mailpit (compose file there) pnpm test:e2e # opt-in: tests/e2e against real providers, from env variables pnpm test:bullmq # opt-in: tests/bullmq on BullMQ and Valkey (compose file there) +pnpm test:mongodb # opt-in: tests/mongodb on a MongoDB replica set (compose file there) node --test tests/unit/mailer.test.js # a single file node --test --test-name-pattern="Bundle" tests/unit/*.test.js # filter by name pnpm run test:coverage # c8 over src/, gated at 98 lines/statements, 90 branches, 100 functions @@ -879,8 +880,8 @@ The contracts of the reliability layer (Phase 0), types of the root with no code Content-IDs fixed (derived from the id under a key of the caller's), and a SHA-256 of all but `enqueuedAt` that tells a duplicate from a key used again. - **`AuditSink`**: `append(event, { transaction })`, append-only, an event written once by its - deterministic `id`; sinks in memory, of a logger, and combined (`./audit`). A MongoDB sink and - the crypto-shredding of recipients are next. + deterministic `id`; sinks in memory, of a logger, and combined (`./audit`), and in MongoDB + (`./mongodb`). The crypto-shredding of recipients is next. - **The states and events of a message:** `queued → rendered → submitted → accepted → delivered | bounced | complained | failed | delivery_unknown`, with correlationId, tenantId, attempt, the provider's message id and the hashes of the payload and of the message; no recipient. @@ -908,8 +909,8 @@ model of states and events. `tenantId` is in the context from the start. providers. - **A persistent queue:** the outbox in MongoDB (transactional outbox, no Redis needed) and a BullMQ driver (Valkey) for throughput and delays; priorities (auth codes before newsletters). - (Unreleased: `./queue` and `./bullmq`, with the memory queue; the outbox is next, a - `QueueSource`.) + (Unreleased: `./queue`, `./bullmq` and `./mongodb`, the outbox as a `QueueSource` with its + audit sink and idempotency store, and the memory queue.) - **The audit log** of every state, with a keyed HMAC plus an encrypted block for the recipient (crypto-shredding), so the right to erasure does not fight an append-only log. 3–6 records per message; optional hash chain for tamper evidence. @@ -1099,9 +1100,14 @@ change. **This file wins** if they ever disagree. `BULLMQ_REDIS_URL=redis://localhost:6390`: duplicates, retries after `Retry-After`, priorities, delays, a release on close and a Worker without the strategy, on BullMQ itself). The release skill asks for them before a release. -- **The template engines, BullMQ and ioredis are devDependencies** for the tests of `./templates` - and `./bullmq` (the real engines, a fake BullMQ in `tests/helpers/bullmq/`, and tsd against the - real classes); `src/` requires none of them. +- **The template engines, BullMQ, ioredis and mongodb are devDependencies** for the tests of + `./templates`, `./bullmq` and `./mongodb` (the real engines, a fake BullMQ in + `tests/helpers/bullmq/`, a collection in memory in `tests/helpers/mongodb/`, and tsd against + the real classes); `src/` requires none of them. `tests/mongodb/` (`pnpm test:mongodb`, + serial; `docker compose -f tests/mongodb/docker-compose.yml up -d --wait` and + `MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0'`) runs the outbox, the audit sink and + the store on a replica set: transactions committed and aborted, workers of several processes, + leases, concurrent reserves of a key and the change stream. - **Platform-dependent tests:** the mock SMTP server takes a client that has gone as no failure: a reset (`ECONNRESET`; macOS and Windows send an RST where Linux sends a FIN) and a reply written after the client closed (`EPIPE` on Linux, `ECONNABORTED` on Windows), which the 221 to diff --git a/README.md b/README.md index 028f5b7..fd641da 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ **Email for Node.js, with a reliability layer.** Its own SMTP client, MIME builder, DKIM signing and eighteen HTTP providers, and on top of them what the other libraries leave to the -application: a persistent queue whose workers render templates of the engine you use and know a -permanent failure from a temporary one, an audit of every state of a message, one model of -delivery events and idempotency; next an outbox in MongoDB and suppression. Zero runtime -dependencies. +application: a persistent queue, on BullMQ or a transactional outbox in MongoDB, whose workers +render templates of the engine you use and know a permanent failure from a temporary one, an +audit of every state of a message, one model of delivery events and idempotency; next +suppression. Zero runtime dependencies. > **Status: 1.1 is the transport layer and the first parts of the reliability layer.** 1.0 ported > the transport from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds what @@ -21,9 +21,9 @@ dependencies. > [sently](https://github.com/omqkhafi/sently) did better: sendmail, ten more HTTP providers, > attachments from paths and URLs, one-click List-Unsubscribe, verified webhooks into one model of > delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and -> mirroring. Unreleased: the queue (`./queue`, with a BullMQ driver in `./bullmq`), templates by -> name over nine engines (`./templates`) and the audit of the states of a message (`./audit`). -> The MongoDB outbox and suppression are next. +> mirroring. Unreleased: the queue (`./queue`, with a BullMQ driver in `./bullmq` and a +> transactional outbox in `./mongodb`), templates by name over nine engines (`./templates`) and +> the audit of the states of a message (`./audit`). Suppression is next. ### 📖 [Read the documentation →](https://mailerat.vercel.app/) @@ -41,7 +41,8 @@ dependencies. read and taken as handled only once your handler commits it, give one model of delivery events (delivered, bounced, complained, …). A queue on BullMQ or a store of yours sends each message once per key, retries what may pass and fails what will not, and an audit sink hears every state - from `queued` to `bounced`. Next: an outbox in MongoDB and suppression. None of the + from `queued` to `bounced`; in MongoDB, the job is written in the transaction of its cause. + Next: suppression. None of the [fourteen libraries compared](https://mailerat.vercel.app/guide/comparison) has them. - **Sent once, or known to be unknown.** Errors are classified as permanent or temporary, and a failure after which the provider may have taken the message is marked `deliveryUnknown`: @@ -53,9 +54,9 @@ dependencies. internationalized addresses are planned. - **Zero dependencies.** Built on `node:*` modules and `fetch`. What a part needs from your application is passed in: the OpenTelemetry providers, a pino-like logger, BullMQ's classes, - your template engine (EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache, React Email), - and later a database driver (MongoDB). CommonJS with ESM named imports and hand-written - TypeScript typings, no build step. + your template engine (EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache, React Email) + and the collections of MongoDB. CommonJS with ESM named imports and hand-written TypeScript + typings, no build step. ## Installation @@ -100,7 +101,7 @@ if (!receipt.successful) console.error(receipt.errorMessages); | `/mime` | MIME composition and DKIM signing | | `/attachments`, `/render` | attachments from a path, a URL, a data URI or a stream; the content of a renderer, React Email | | `/templates` | templates by name over EJS, Handlebars, Pug, MJML, Liquid, Nunjucks, Eta, Mustache or React Email: locales, layouts, subjects, CSS inlined | -| `/queue`, `/bullmq`, `/audit` | the mail queue and its worker, the BullMQ driver, the states of a message and their sinks | +| `/queue`, `/bullmq`, `/mongodb`, `/audit` | the mail queue and its worker, the BullMQ driver, the MongoDB outbox (and its audit and idempotency stores), the states of a message and their sinks | | `/smtp`, `/sendmail`, `/jmap` | SMTP, sendmail and JMAP transports | | `/ses`, `/resend`, `/sendgrid`, `/mailgun`, `/postmark`, `/brevo`, `/mailjet`, `/mailersend`, `/sparkpost`, `/mailtrap`, `/maileroo`, `/lettermint`, `/zeptomail`, `/smtp2go`, `/azure`, `/scaleway`, `/cloudflare`, `/plunk` | HTTP providers | | `/webhooks` | verified webhooks of fifteen providers, read into delivery events | diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 5907970..d41304f 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -121,6 +121,7 @@ const guide = [ items: [ { text: 'The queue', link: '/guide/queue' }, { text: 'BullMQ', link: '/transports/bullmq' }, + { text: 'MongoDB outbox', link: '/guide/outbox' }, { text: 'The audit', link: '/guide/audit' }, ], }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 5d13167..aa5410c 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -56,6 +56,7 @@ model of delivery events. | `@alexify/mailer/templates` | `createTemplateRenderer`, `createFileLoader`, `createMemoryLoader`, `createCssInliner`, `createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, `createReactEmailEngine`, `TemplateNotFoundError`, `TemplateRenderError` | [Templates by name](/messages/templates) | | `@alexify/mailer/queue` | `createMailQueue`, `createMailJobHandler`, `createMemoryQueue`, `createPollingDriver`, `prepareMailJob`, `parseMailJob`, `MailQueueError`, `QueueLeaseLostError`, `MAIL_JOB_NAME`, `MAIL_JOB_VERSION` | [The queue](/guide/queue) | | `@alexify/mailer/bullmq` | `createBullMQDriver`, `createBullMQProcessor`, `createBullMQBackoffStrategy`, `MAILER_BACKOFF_TYPE` | [BullMQ](/transports/bullmq) | +| `@alexify/mailer/mongodb` | `createMongoOutbox`, `createMongoAuditSink`, `createMongoIdempotencyStore` | [MongoDB outbox](/guide/outbox) | | `@alexify/mailer/audit` | `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks`, `stateOf`, `stateOfDeliveryEvent`, `auditEventOfDeliveryEvent`, `isFinalState` | [The audit](/guide/audit) | | `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMailerSendWebhookVerifier`, `createMailtrapWebhookVerifier`, `createBrevoWebhookVerifier`, `createMailjetWebhookVerifier`, `createSparkPostWebhookVerifier`, `createLettermintWebhookVerifier`, `createMailerooWebhookVerifier`, `createSmtp2goWebhookVerifier`, `createAzureWebhookVerifier`, `createScalewayWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | diff --git a/docs/comparison.json b/docs/comparison.json index 2d9c37e..1ef3b4f 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -340,8 +340,8 @@ }, "queue": { "mark": "✅", - "text": "BullMQ driver, memory queue, any outbox as a `QueueSource`", - "note": "Permanent failures fail at once, `Retry-After` is a minimum, and a delivery of unknown outcome is never sent blindly again. A MongoDB outbox is next on the [roadmap](/guide/why#the-plan)." + "text": "MongoDB outbox (transactional), BullMQ driver, memory queue", + "note": "Permanent failures fail at once, `Retry-After` is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause." }, "attachments": { "mark": "✅", @@ -478,8 +478,8 @@ }, "auditLog": { "mark": "⚠️", - "text": "state model + sinks; crypto-shredding next", - "note": "Each state of a message, from the queue to the webhooks, one event per change. The keyed HMAC and encrypted recipients are on the [roadmap](/guide/why#the-plan)." + "text": "state model + sinks, MongoDB among them; crypto-shredding next", + "note": "Each state of a message, from the queue to the webhooks, one event per change, written once by its id. The keyed HMAC and encrypted recipients are on the [roadmap](/guide/why#the-plan)." }, "types": { "mark": "✅", diff --git a/docs/guide/audit.md b/docs/guide/audit.md index 4ae18d2..4b73c54 100644 --- a/docs/guide/audit.md +++ b/docs/guide/audit.md @@ -60,6 +60,8 @@ const audit = combineAuditSinks(createLoggerAuditSink(logger), myDatabaseSink); `mail.`: `failed` at error; `delivery_unknown`, `bounced` and `complained` at warn. - `combineAuditSinks(...sinks)` tells each in turn, whatever another threw. +- `createMongoAuditSink({ collection })` of [`./mongodb`](/guide/outbox), each + event once by its id, in the transaction of the enqueue. - Your own: `{ append(event, { transaction }) }`. An outbox writes the `queued` event in the transaction of the enqueue; a sink keyed by `id` writes an event once however often it is told. diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index a08dde5..9846564 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -70,7 +70,7 @@ it.

- + @@ -108,7 +108,7 @@ it. - + @@ -127,14 +127,14 @@ it. 1. Through npm:@alexify/mailer. Deno’s fetch sends a request again when a reused connection closes before its answer, so the docs give it a fetch of its own (known issues). 2. A failure after which the provider may have taken the message is marked deliveryUnknown: ./retry does not retry it and ./pool does not fail over with it. -3. Permanent failures fail at once, Retry-After is a minimum, and a delivery of unknown outcome is never sent blindly again. A MongoDB outbox is next on the roadmap. +3. Permanent failures fail at once, Retry-After is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause. 4. Nothing is read implicitly: fromPath and fromUrl are called by the application, fromUrl checks every redirect against allowUrl (no private address by default) and bounds time and bytes. 5. Names, never paths; locales, layouts, front-matter subjects, CSS inlined without remote fetches, text derived. 6. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. 7. The other five have no request that checks a sending key without sending a message. 8. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time, and an id kept as taken only once the handler commits its events. 9. On the roadmap of the reliability layer: designed, with no code yet. -10. Each state of a message, from the queue to the webhooks, one event per change. The keyed HMAC and encrypted recipients are on the roadmap. +10. Each state of a message, from the queue to the webhooks, one event per change, written once by its id. The keyed HMAC and encrypted recipients are on the roadmap. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9ace7dd..9c62fa9 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -215,8 +215,8 @@ their own unless the part is given a store they share: | Part | What it keeps | In one process | Shared by several | | --- | --- | --- | --- | | [`./ratelimit`](/transports/ratelimit) | the token buckets | at most `maxKeys` buckets | a `store` of `take()`, such as a script on Redis, so that N processes send no more than the one limit | -| [`./idempotency`](/transports/idempotency) | the keys reserved and the receipts | `createMemoryIdempotencyStore()` | a store of `reserve`, `complete` and `release`, the outbox to come among them | -| [`./queue`](/guide/queue) | the jobs | `createMemoryQueue()` | a driver they share: [BullMQ](/transports/bullmq), or a store's outbox | +| [`./idempotency`](/transports/idempotency) | the keys reserved and the receipts | `createMemoryIdempotencyStore()` | a store of `reserve`, `complete` and `release`, such as [`createMongoIdempotencyStore()`](/guide/outbox) | +| [`./queue`](/guide/queue) | the jobs | `createMemoryQueue()` | a driver they share: [BullMQ](/transports/bullmq), or the [MongoDB outbox](/guide/outbox) | | [`./webhooks`](/guide/webhooks#time-and-replays) | the ids of the requests taken | `createMemoryReplayStore()` | a store of `reserve`, `commit` and `release`; required, `null` to keep none | | [`./breaker`](/transports/breaker) | the state of the circuit | one per transport | none: each process opens its own circuit on the failures it sees | | [`./pool`](/transports/pool) | the turn of round-robin | one per transport | none | diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md new file mode 100644 index 0000000..852db97 --- /dev/null +++ b/docs/guide/outbox.md @@ -0,0 +1,122 @@ +# MongoDB outbox + +`@alexify/mailer/mongodb` keeps the [mail queue](/guide/queue) in a collection +of MongoDB: the transactional outbox. A job is written in the transaction of +your own writes, so a message is queued exactly when what it tells of is +committed, with no Redis. The same subpath keeps the [audit](/guide/audit) and +the keys of [`IdempotencyTransport`](/transports/idempotency) in MongoDB. The +package depends on no MongoDB driver: you pass in its collections. + +```js +const { MongoClient } = require('mongodb'); +const { createMailQueue } = require('@alexify/mailer/queue'); +const { IdempotencyTransport } = require('@alexify/mailer/idempotency'); +const { + createMongoAuditSink, + createMongoIdempotencyStore, + createMongoOutbox, +} = require('@alexify/mailer/mongodb'); + +const client = new MongoClient(process.env.MONGODB_URL); +const db = client.db('app'); + +const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), changeStream: true }); +const audit = createMongoAuditSink({ collection: db.collection('mail_audit') }); +const store = createMongoIdempotencyStore({ collection: db.collection('mail_idempotency') }); +await Promise.all([outbox.ensureIndexes(), audit.ensureIndexes(), store.ensureIndexes()]); + +const queue = createMailQueue({ driver: outbox, audit }); + +// The producer: the job and its audit event go in the transaction. +const session = client.startSession(); +await session.withTransaction(async () => { + await users.updateOne({ _id: user.id }, { $set: { resetToken } }, { session }); + await queue.enqueue( + { + kind: 'template', + template: 'reset-password', + context: { url }, + from: 'no-reply@example.com', + to: user.email, + idempotencyKey: `reset-password:${resetToken}`, + }, + { transaction: session }, + ); +}); + +// A worker, in as many processes as you like. +queue.createWorker({ + transport: new IdempotencyTransport(new SesTransport({ ...config, retries: 1 }), { store }), + renderer: templates, + concurrency: 5, +}); +``` + +If the transaction aborts, neither the job nor its `queued` event is written. +A job enqueued again under its key in the transaction is a duplicate, and does +not abort it: the outbox writes with an upsert, never an insert that would +fail on the key. + +## How workers share it + +The outbox is a `QueueSource`, read by the polling driver of `./queue`: + +- a job is lent by one atomic `findOneAndUpdate`, the next ready one by + priority, then time, under a token of its own, for the lease + (`leaseMilliseconds`, a minute), which the worker extends while it runs; +- every later write of that run (done, retried, failed, given back) is fenced + by the token: a worker whose lease ran out writes nothing; +- a job whose lease ran out (its worker died) is ready again, its attempt run + again, `redelivered`; +- a rate limit of the whole queue (`rateLimitScope: 'queue'`) pauses every + worker, through a document of its own. + +Workers poll (`pollIntervalMilliseconds`, a second). With `changeStream: true`, +on a replica set, a change of the collection wakes them at once, and the poll +is only the fallback. + +## Indexes and retention + +`ensureIndexes()` makes: + +| Collection | Index | +| --- | --- | +| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; a TTL on `finishedAt` | +| audit | `{ correlationId, at, attempt, rank }`; `{ jobId }` | +| idempotency | a TTL on `expiresAt` | + +A finished job is kept `retainFinishedSeconds` (a week), and while it is kept, +an enqueue under its key is a duplicate: the retention is the window of +deduplication. A key of the store goes when its time passes. + +## A job as stored + +```js +{ + _id: '1b9d…', // the id of the job: a UUID of namespace, tenant and key + name: 'mail', data: { … }, hash: '…', + priority: 'high', rank: 0, maxAttempts: 5, + state: 'ready', // 'active', 'completed', 'failed' + attempts: 0, availableAt: ISODate(…), enqueuedAt: ISODate(…), + token: null, leaseUntil: null, redelivered: false, + failure: { code, reason }, // of the last failed attempt + result: { provider, providerMessageId }, + finishedAt: ISODate(…), +} +``` + +`outbox.get(id)` reads one. The data of a job is stored as it is, a token of a +reset link among it: restrict who reads the collection. + +## The audit and the keys + +`createMongoAuditSink` writes each event once by its id (an upsert), in the +transaction it is given, with `eventsOf(correlationId)` in their order and +`stateOf(correlationId)`. `createMongoIdempotencyStore` makes each step of +`IdempotencyTransport` atomic, so that of two workers sending one key, one +sends and the other is told it is in flight; a worker that died mid-send +leaves the key reserved, and the attempt again is of unknown delivery, not a +second send. + +`pnpm test:mongodb` runs all three on MongoDB (`tests/mongodb/docker-compose.yml`, +a replica set of one). diff --git a/docs/guide/queue.md b/docs/guide/queue.md index fd12c22..4d6212d 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -4,7 +4,8 @@ message as a job, and a worker renders it, sends it through the transport you composed, and decides from the receipt what the queue does next. The queue is a driver of your choice: [BullMQ](/transports/bullmq) on Redis or Valkey, the -memory queue for tests, or a store's outbox (a `QueueSource`). +[MongoDB outbox](/guide/outbox), the memory queue for tests, or an outbox of +your own store (a `QueueSource`). ``` producer queue.enqueue(job) ──▶ driver (BullMQ, outbox, memory) @@ -140,8 +141,8 @@ category and reason. ## Several processes A job is shared by every worker of its driver; the key store of -`IdempotencyTransport` must be shared too (Redis, the outbox), or a crash -mid-send can be sent twice. The breaker and the rate limit of the chain are +`IdempotencyTransport` must be shared too (`createMongoIdempotencyStore`, or +one on Redis), or a crash mid-send can be sent twice. The breaker and the rate limit of the chain are per process unless given a shared store (see [getting started](/guide/getting-started#several-processes)). diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 7028e6d..207d89b 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -10,7 +10,8 @@ Key facts: - Status: 1.0 is the transport layer, ported from upyo 0.7.0 (MIT); 1.1 ports what nodemailer, unemail and sently did better and adds delivery events with verified webhooks; the queue - (`/queue`, `/bullmq`), templates (`/templates`) and the audit (`/audit`) are unreleased + (`/queue`, `/bullmq`, `/mongodb`), templates (`/templates`) and the audit (`/audit`) are + unreleased - Package (npm): `@alexify/mailer`, installed with `npm install @alexify/mailer` - Node.js >= 22, tested on Bun and Deno too; CommonJS with ESM named-import interop; hand-written d.ts per entry @@ -24,7 +25,7 @@ Key facts: `/mailjet`, `/mailersend`, `/sparkpost`, `/mailtrap`, `/maileroo`, `/lettermint`, `/zeptomail`, `/smtp2go`, `/azure`, `/scaleway`, `/cloudflare`, `/plunk`, `/mock`; decorators `/pool`, `/retry`, `/breaker`, `/ratelimit`, `/idempotency`, `/hooks`, `/tee`, `/otel`, `/logger`; - the reliability layer `/queue`, `/bullmq`, `/templates`, `/audit` + the reliability layer `/queue`, `/bullmq`, `/mongodb`, `/templates`, `/audit` - Guide: https://mailerat.vercel.app/guide/getting-started - API: https://mailerat.vercel.app/api/exports - Comparison with 14 alternatives (Nodemailer, emailjs, Upyo, unemail, sently, provider SDKs): diff --git a/package.json b/package.json index 1ad909f..1491f10 100644 --- a/package.json +++ b/package.json @@ -180,6 +180,10 @@ "./bullmq": { "types": "./src/bullmq/index.d.ts", "default": "./src/bullmq/index.js" + }, + "./mongodb": { + "types": "./src/mongodb/index.d.ts", + "default": "./src/mongodb/index.js" } }, "directories": { @@ -205,6 +209,7 @@ "test:mailpit": "node --test --test-concurrency=1 tests/mailpit/*.test.js", "test:e2e": "node --test --test-concurrency=1 tests/e2e/*.test.js", "test:bullmq": "node --test --test-concurrency=1 tests/bullmq/*.test.js", + "test:mongodb": "node --test --test-concurrency=1 tests/mongodb/*.test.js", "test:coverage": "c8 --all --include 'src/**' --check-coverage --lines 98 --statements 98 --branches 90 --functions 100 --reporter text --reporter lcov node --test tests/unit/*.test.js", "test:types": "tsd", "check:dts": "node scripts/check-dts.js", @@ -238,6 +243,7 @@ "transactional-email", "email-queue", "bullmq", + "mongodb", "email-templates", "outbox", "audit-log", @@ -275,6 +281,7 @@ "juice": "^12.2.0", "liquidjs": "^10.30.0", "mjml": "^5.4.1", + "mongodb": "^7.7.0", "mustache": "^4.2.0", "nunjucks": "^3.2.4", "oxfmt": "^0.72.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f87716f..4343b11 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: mjml: specifier: ^5.4.1 version: 5.4.1(svgo@4.1.0)(typescript@7.0.2) + mongodb: + specifier: ^7.7.0 + version: 7.7.0 mustache: specifier: ^4.2.0 version: 4.2.0 @@ -645,6 +648,9 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@mongodb-js/saslprep@1.6.1': + resolution: {integrity: sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg==} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} cpu: [arm64] @@ -1207,6 +1213,12 @@ packages: '@types/web-bluetooth@0.0.21': resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@13.0.0': + resolution: {integrity: sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==} + '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -1575,6 +1587,10 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + bson@7.3.3: + resolution: {integrity: sha512-oz3LwE3qGEhTCSyhzQRLY3Bj1NLOsjFMf6tcmY5J8M+3R7EVcmoBfSHWvjUK3yXCLDUsGpIuB3aQL4nDZ1EV/g==} + engines: {node: '>=20.19.0'} + bullmq@6.3.16: resolution: {integrity: sha512-savxnrdqckobnm31XWwCKTAswkxrNIqLD/PsdwgYaDpst0UdULu/QjOtPdn86KNrBq8WoHKhJvbgGK13dgPJ8w==} engines: {node: '>=14.17.0'} @@ -2349,6 +2365,9 @@ packages: mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + mensch@0.3.4: resolution: {integrity: sha512-IAeFvcOnV9V0Yk+bFhYR07O3yNina9ANIN5MoXBKYJ/RLYPurd2d0yw14MDhpr9/momp0WofT1bPUh3hkzdi/g==} @@ -2504,6 +2523,37 @@ packages: resolution: {integrity: sha512-yreEmxjcU1jf5aXihZ3oNwRzuTxxoOiY9QoKBgbCwjfrQTDddQJQVrYSDJk/yBTwt4mhTcyjK3zQgybyf7mF2Q==} hasBin: true + mongodb-connection-string-url@7.0.2: + resolution: {integrity: sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==} + engines: {node: '>=20.19.0'} + + mongodb@7.7.0: + resolution: {integrity: sha512-XG1xXEqyouAV8GCb3PuRaujofeXKpAJAPvXYWZgHl6Dz4QHSJ1nRLl0+GpqaLTwKvC0RFE9truOur34RNFY1nw==} + engines: {node: '>=20.19.0'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.806.0 + '@mongodb-js/zstd': ^7.0.0 + gcp-metadata: ^7.0.1 + kerberos: ^7.0.0 + mongodb-client-encryption: ^7.2.0 + snappy: ^7.3.2 + socks: ^2.8.6 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -2950,6 +3000,10 @@ packages: pug@3.0.4: resolution: {integrity: sha512-kFfq5mMzrS7+wrl5pLJzZEzemx34OQ0w4SARfhy/3yxTlhbstsudDwJzhf1hP02yHzbjoVMSXUj/Sz6RNfMyXg==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -3097,6 +3151,9 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + spdx-correct@3.2.0: resolution: {integrity: sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==} @@ -3196,6 +3253,10 @@ packages: token-stream@1.0.0: resolution: {integrity: sha512-VSsyNPPW74RpHwR8Fc21uubwHY7wMDeJLys2IX5zJNih+OnAnaifKHo+1LHT7DAdloQ7apeaaWg8l7qnf/TnEg==} + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -3345,6 +3406,10 @@ packages: resolution: {integrity: sha512-Ezr98sqXW/+OCGoUEXuOKVR+oVFlSdn1tIySEEJdiSAw4IjrW8hQkwARSSBJTSB5Us5dnytDgL0ZDliAYBhaNA==} engines: {node: '>=10.0.0'} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + whatwg-encoding@3.1.1: resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} engines: {node: '>=18'} @@ -3354,6 +3419,10 @@ packages: resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} engines: {node: '>=18'} + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -3799,6 +3868,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@mongodb-js/saslprep@1.6.1': + dependencies: + sparse-bitfield: 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': optional: true @@ -4155,6 +4228,12 @@ snapshots: '@types/web-bluetooth@0.0.21': {} + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@13.0.0': + dependencies: + '@types/webidl-conversions': 7.0.3 + '@typescript/typescript-aix-ppc64@7.0.2': optional: true @@ -4412,6 +4491,8 @@ snapshots: node-releases: 2.0.59 update-browserslist-db: 1.3.4(browserslist@4.29.3) + bson@7.3.3: {} + bullmq@6.3.16(ioredis@6.0.0): dependencies: cron-parser: 5.10.2 @@ -5273,6 +5354,8 @@ snapshots: mdn-data@2.27.1: {} + memory-pager@1.5.0: {} + mensch@0.3.4: {} meow@9.0.0: @@ -5795,6 +5878,17 @@ snapshots: - typescript - uncss + mongodb-connection-string-url@7.0.2: + dependencies: + '@types/whatwg-url': 13.0.0 + whatwg-url: 14.2.0 + + mongodb@7.7.0: + dependencies: + '@mongodb-js/saslprep': 1.6.1 + bson: 7.3.3 + mongodb-connection-string-url: 7.0.2 + ms@2.1.3: {} msgpackr-extract@3.0.4: @@ -6281,6 +6375,8 @@ snapshots: pug-runtime: 3.0.1 pug-strip-comments: 2.0.0 + punycode@2.3.1: {} + queue-microtask@1.2.3: {} quick-format-unescaped@4.0.4: {} @@ -6437,6 +6533,10 @@ snapshots: space-separated-tokens@2.0.2: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + spdx-correct@3.2.0: dependencies: spdx-expression-parse: 3.0.1 @@ -6542,6 +6642,10 @@ snapshots: token-stream@1.0.0: {} + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + trim-lines@3.0.1: {} trim-newlines@3.0.1: {} @@ -6728,12 +6832,19 @@ snapshots: mime: 2.6.0 valid-data-url: 3.0.1 + webidl-conversions@7.0.0: {} + whatwg-encoding@3.1.1: dependencies: iconv-lite: 0.6.3 whatwg-mimetype@4.0.0: {} + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index fca1575..e6e5d89 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -38,5 +38,6 @@ "./tee": 3.5, "./audit": 2, "./queue": 14.5, - "./bullmq": 3.5 + "./bullmq": 3.5, + "./mongodb": 5.5 } diff --git a/src/audit/states.js b/src/audit/states.js index 1b1b41a..f036388 100644 --- a/src/audit/states.js +++ b/src/audit/states.js @@ -16,6 +16,14 @@ const RANKS = new Map([ const FINAL = new Set(['delivered', 'bounced', 'complained', 'failed']); +/** + * The place of a state in the order of states: of two events at one time and + * attempt, the one further along came later. + * + * @internal + */ +const rankOf = (state) => RANKS.get(state) ?? -1; + /** * Whether no later state but a complaint follows: `delivered`, `bounced`, * `complained` and `failed`. @@ -45,4 +53,4 @@ const stateOf = (events) => { return state; }; -module.exports = { isFinalState, stateOf }; +module.exports = { isFinalState, rankOf, stateOf }; diff --git a/src/mongodb/audit-sink.js b/src/mongodb/audit-sink.js new file mode 100644 index 0000000..09c6a17 --- /dev/null +++ b/src/mongodb/audit-sink.js @@ -0,0 +1,65 @@ +// Not ported: an audit sink in a collection of MongoDB: append-only, each +// event written once by its id however often it is told, and in the +// transaction of the enqueue when there is one. + +const { rankOf, stateOf } = require('../audit/states.js'); +const { checkCollection, sessionOf } = require('./collection.js'); + +// An event as a document: its id the key, its time a Date, and the rank of +// its state, which orders the events of one time and attempt. +const documentOfEvent = (event) => { + const { id, at, ...rest } = event; + return { _id: id, ...rest, at: new Date(at), rank: rankOf(event.state) }; +}; + +const eventOfDocument = (document) => { + const { _id, at, rank, ...rest } = document; + return { id: _id, ...rest, at: at instanceof Date ? at.toISOString() : at }; +}; + +/** + * Makes an audit sink in a collection. + * + * @param options `collection`, `db.collection('mail_audit')`. + * @returns The sink, with `eventsOf(correlationId)`, + * `stateOf(correlationId)` and `ensureIndexes()`. + * @throws {TypeError} If it is given no collection. + */ +const createMongoAuditSink = (options = {}) => { + const collection = checkCollection(options.collection, 'createMongoAuditSink', [ + 'updateOne', + 'find', + 'createIndex', + ]); + const eventsOf = async (correlationId) => { + const documents = await collection + // oxlint-disable-next-line no-restricted-properties -- a MongoDB collection, not an array + .find({ correlationId }) + .sort({ at: 1, attempt: 1, rank: 1 }) + .toArray(); + const events = []; + for (const document of documents) events.push(eventOfDocument(document)); + return events; + }; + return { + async append(event, context) { + if (typeof event?.id !== 'string') throw new TypeError('An audit event has an id.'); + await collection.updateOne( + { _id: event.id }, + { $setOnInsert: documentOfEvent(event) }, + { ...sessionOf(context?.transaction), upsert: true }, + ); + }, + eventsOf, + stateOf: async (correlationId) => stateOf(await eventsOf(correlationId)), + async ensureIndexes() { + await collection.createIndex( + { correlationId: 1, at: 1, attempt: 1, rank: 1 }, + { name: 'mailer_correlation' }, + ); + await collection.createIndex({ jobId: 1 }, { name: 'mailer_job', sparse: true }); + }, + }; +}; + +module.exports = { createMongoAuditSink }; diff --git a/src/mongodb/collection.js b/src/mongodb/collection.js new file mode 100644 index 0000000..2ec3841 --- /dev/null +++ b/src/mongodb/collection.js @@ -0,0 +1,64 @@ +// Not ported: what the stores of ./mongodb share in reading a collection of +// the MongoDB driver the application passes in, typed by the members they +// call, so the package depends on no version of it. + +const METHODS = [ + 'updateOne', + 'updateMany', + 'findOne', + 'findOneAndUpdate', + 'deleteOne', + 'find', + 'createIndex', +]; + +/** + * Checks a collection of the MongoDB driver: `db.collection(name)`. + * + * @param collection The collection. + * @param what What needs it, for the message. + * @param methods The methods it calls. + * @returns The collection. + * @throws {TypeError} If it lacks one. + * @internal + */ +const checkCollection = (collection, what, methods = METHODS) => { + for (const method of methods) { + if (typeof collection?.[method] !== 'function') { + throw new TypeError(`${what} needs a collection of the MongoDB driver, with ${method}().`); + } + } + return collection; +}; + +/** + * Whether an error is MongoDB's duplicate key (E11000). + * + * @internal + */ +const isDuplicateKey = (error) => error?.code === 11000 || error?.code === 11001; + +/** + * The document `findOneAndUpdate` gave: the driver from 5.7 gives it as it + * is with `includeResultMetadata: false`, an older one in `{ value }`. + * + * @internal + */ +const documentOf = (result) => { + if (result !== null && typeof result === 'object' && 'ok' in result && 'value' in result) { + return result.value; + } + return result ?? null; +}; + +/** + * The options of a call in a transaction, when there is one. + * + * @internal + */ +const sessionOf = (transaction, options = {}) => + transaction === undefined || transaction === null + ? options + : { ...options, session: transaction }; + +module.exports = { checkCollection, documentOf, isDuplicateKey, sessionOf }; diff --git a/src/mongodb/idempotency-store.js b/src/mongodb/idempotency-store.js new file mode 100644 index 0000000..b7cc93f --- /dev/null +++ b/src/mongodb/idempotency-store.js @@ -0,0 +1,101 @@ +// Not ported: a store of idempotency keys in a collection of MongoDB, which +// the workers of several processes share: each method is one atomic step. +// A free key (absent, or whose time has passed) is taken by an upsert whose +// filter only a free key meets, so of two reserves one inserts and the other +// meets the duplicate key and reads the record. + +const { randomUUID } = require('node:crypto'); +const { checkOptionalFunction } = require('../internal/options.js'); +const { checkCollection, isDuplicateKey } = require('./collection.js'); + +// Two reserves of a key that is dropped between them read it again. +const MAX_TRIES = 3; + +const recordOf = (document) => { + const record = { state: document.state, fingerprint: document.fingerprint }; + if (document.receipt !== undefined && document.receipt !== null) { + record.receipt = document.receipt; + } + return record; +}; + +/** + * Makes a store of idempotency keys in a collection, for `./idempotency`. + * + * @param options `collection`, `db.collection('mail_idempotency')`; `now`. + * @returns The store: `reserve`, `complete` and `release`, and + * `ensureIndexes()`, which makes the index that drops a key once its time + * has passed. + * @throws {TypeError} If an option is invalid. + */ +const createMongoIdempotencyStore = (options = {}) => { + const collection = checkCollection(options.collection, 'createMongoIdempotencyStore', [ + 'updateOne', + 'findOne', + 'deleteOne', + 'createIndex', + ]); + const now = checkOptionalFunction(options.now, 'now') ?? Date.now; + return { + async reserve(key, fingerprint, leaseMs) { + for (let tries = 0; tries < MAX_TRIES; tries++) { + const at = new Date(now()); + const token = randomUUID(); + try { + await collection.updateOne( + { _id: key, expiresAt: { $lte: at } }, + { + $set: { + state: 'pending', + fingerprint, + token, + receipt: null, + expiresAt: new Date(at.getTime() + leaseMs), + }, + }, + { upsert: true }, + ); + return { token }; + } catch (error) { + if (!isDuplicateKey(error)) throw error; + } + // Held within its time: what holds it. + const held = await collection.findOne({ _id: key }); + if (held !== null && held !== undefined) return { record: recordOf(held) }; + } + throw new Error(`The idempotency key could not be reserved in ${MAX_TRIES} tries.`); + }, + async complete(key, token, record, ttlMs) { + const at = new Date(now()); + try { + await collection.updateOne( + { _id: key, $or: [{ token }, { expiresAt: { $lte: at } }] }, + { + $set: { + state: 'done', + fingerprint: record.fingerprint, + receipt: record.receipt, + token, + expiresAt: new Date(at.getTime() + ttlMs), + }, + }, + { upsert: true }, + ); + } catch (error) { + // Another reservation holds it within its time: it is theirs. + if (!isDuplicateKey(error)) throw error; + } + }, + async release(key, token) { + await collection.deleteOne({ _id: key, token, state: 'pending' }); + }, + async ensureIndexes() { + await collection.createIndex( + { expiresAt: 1 }, + { name: 'mailer_expires', expireAfterSeconds: 0 }, + ); + }, + }; +}; + +module.exports = { createMongoIdempotencyStore }; diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts new file mode 100644 index 0000000..aaa3dbd --- /dev/null +++ b/src/mongodb/index.d.ts @@ -0,0 +1,146 @@ +// Not ported: the types of ./mongodb. The MongoDB driver is not a dependency +// of any kind: you pass in its collections, which these name by the members +// the stores call, so that this file resolves without it. Methods use +// shorthand, compared bivariantly, so that the driver's generic and +// overloaded signatures satisfy them without being imported. + +import type { + AuditEvent, + AuditSink, + Logger, + MessageState, + QueueDriver, + QueueSource, +} from '../../index.js'; +import type { IdempotencyStore } from '../idempotency/index.js'; +import type { PollingDriverOptions } from '../queue/index.js'; + +/** What a write of a collection gives. */ +export interface MongoUpdateResultLike { + readonly matchedCount: number; + readonly upsertedCount?: number; +} + +/** A cursor of `find()`. */ +export interface MongoCursorLike { + sort(sort: any): MongoCursorLike; + toArray(): Promise; +} + +/** A change stream of `watch()`. */ +export interface MongoChangeStreamLike { + on(event: string, listener: (...args: any[]) => void): unknown; + close(): Promise; +} + +/** The members of a `Collection` of the MongoDB driver the stores call. */ +export interface MongoCollectionLike { + updateOne(filter: any, update: any, options?: any): Promise; + updateMany(filter: any, update: any, options?: any): Promise; + findOne(filter: any, options?: any): Promise; + findOneAndUpdate(filter: any, update: any, options: any): Promise; + deleteOne(filter: any, options?: any): Promise; + find(filter: any, options?: any): MongoCursorLike; + createIndex(spec: any, options?: any): Promise; + /** For `changeStream: true`, on a replica set. */ + watch?(pipeline?: any[], options?: any): MongoChangeStreamLike; +} + +/** The options of {@link createMongoOutbox}. */ +export interface MongoOutboxOptions extends PollingDriverOptions { + /** `db.collection('mail_outbox')`. */ + readonly collection: MongoCollectionLike; + /** Wakes the workers on a change of the collection, on a replica set. @default false */ + readonly changeStream?: boolean; + /** The time a finished job is kept, and so the window of its duplicates. @default 604800 */ + readonly retainFinishedSeconds?: number; + readonly logger?: Logger; +} + +/** A job of the outbox, as it holds it. */ +export interface MongoOutboxJob { + readonly id: string; + readonly name: string; + readonly data: any; + readonly state: 'ready' | 'active' | 'completed' | 'failed'; + readonly priority: 'high' | 'normal' | 'low'; + readonly attempt: number; + readonly maxAttempts: number; + readonly enqueuedAt?: number; + readonly redelivered: boolean; + readonly failure?: { readonly code: string; readonly reason: string }; + readonly result?: any; +} + +/** The mail queue in a collection: a driver, its source, and what it holds. */ +export interface MongoOutbox extends QueueDriver { + readonly source: QueueSource; + /** Makes the indexes of the outbox: the ready jobs, the leases, and the expiry of finished ones. */ + ensureIndexes(): Promise; + get(id: string): Promise; +} + +/** + * Makes a mail queue in a collection of MongoDB, the transactional outbox: + * a job enqueued with `{ transaction: session }` is written in that + * transaction, its `queued` audit event too. + * + * @example + * ```js + * const outbox = createMongoOutbox({ collection: db.collection('mail_outbox') }); + * await outbox.ensureIndexes(); + * const queue = createMailQueue({ driver: outbox, audit }); + * await session.withTransaction(async () => { + * await users.updateOne({ _id }, { $set: { resetToken } }, { session }); + * await queue.enqueue(job, { transaction: session }); + * }); + * ``` + * @throws {TypeError} If an option is invalid. + */ +export declare function createMongoOutbox(options: MongoOutboxOptions): MongoOutbox; + +/** The options of {@link createMongoAuditSink}. */ +export interface MongoAuditSinkOptions { + /** `db.collection('mail_audit')`. */ + readonly collection: MongoCollectionLike; +} + +/** An audit sink in a collection. */ +export interface MongoAuditSink extends AuditSink { + append(event: AuditEvent, context?: { readonly transaction?: unknown }): Promise; + eventsOf(correlationId: string): Promise; + stateOf(correlationId: string): Promise; + /** Makes the indexes of the events: by correlation id and time, by job. */ + ensureIndexes(): Promise; +} + +/** + * Makes an audit sink in a collection: append-only, each event written once + * by its id, in the transaction it is given. + * + * @throws {TypeError} If it is given no collection. + */ +export declare function createMongoAuditSink(options: MongoAuditSinkOptions): MongoAuditSink; + +/** The options of {@link createMongoIdempotencyStore}. */ +export interface MongoIdempotencyStoreOptions { + /** `db.collection('mail_idempotency')`. */ + readonly collection: MongoCollectionLike; + readonly now?: () => number; +} + +/** A store of idempotency keys in a collection. */ +export interface MongoIdempotencyStore extends IdempotencyStore { + /** Makes the index that drops a key once its time has passed. */ + ensureIndexes(): Promise; +} + +/** + * Makes a store of idempotency keys in a collection, which the workers of + * several processes share, for `IdempotencyTransport`. + * + * @throws {TypeError} If an option is invalid. + */ +export declare function createMongoIdempotencyStore( + options: MongoIdempotencyStoreOptions, +): MongoIdempotencyStore; diff --git a/src/mongodb/index.js b/src/mongodb/index.js new file mode 100644 index 0000000..47ed5a9 --- /dev/null +++ b/src/mongodb/index.js @@ -0,0 +1,9 @@ +// Not ported: the entry of ./mongodb, the stores of the reliability layer in +// collections of the MongoDB driver the application passes in: the outbox +// of the mail queue, its audit and its idempotency keys. + +const { createMongoAuditSink } = require('./audit-sink.js'); +const { createMongoIdempotencyStore } = require('./idempotency-store.js'); +const { createMongoOutbox } = require('./outbox.js'); + +module.exports = { createMongoAuditSink, createMongoIdempotencyStore, createMongoOutbox }; diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js new file mode 100644 index 0000000..572aefc --- /dev/null +++ b/src/mongodb/outbox.js @@ -0,0 +1,277 @@ +// Not ported: a mail queue in a collection of MongoDB, the transactional +// outbox: a job is written in the transaction of the application's own +// writes, so that a message is queued exactly when what it tells of is +// committed, with no Redis. It is a QueueSource, which the polling driver +// reads: a job is lent by one atomic findOneAndUpdate, under a token every +// later write is fenced by, and a job whose lease ran out goes to another +// worker. + +const { randomUUID } = require('node:crypto'); +const { createPollingDriver } = require('../internal/polling-driver.js'); +const { createLog } = require('../internal/log.js'); +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { checkCollection, documentOf, sessionOf } = require('./collection.js'); + +const RANK = new Map([ + ['high', 0], + ['normal', 1], + ['low', 2], +]); +// The document of a pause of the whole queue: no job id has a colon. +const PAUSE_ID = 'mailer:pause'; +const DEFAULT_RETAIN_SECONDS = 7 * 24 * 60 * 60; + +const isRequest = (request) => + typeof request?.id === 'string' && + request.id !== '' && + !request.id.includes(':') && + typeof request.name === 'string' && + typeof request.data === 'object' && + request.data !== null && + RANK.has(request.priority) && + Number.isInteger(request.maxAttempts) && + request.maxAttempts >= 1; + +const documentOfRequest = (request, now) => ({ + _id: request.id, + name: request.name, + data: request.data, + hash: request.hash, + priority: request.priority, + rank: RANK.get(request.priority), + maxAttempts: request.maxAttempts, + enqueuedAt: new Date(request.enqueuedAt ?? now), + availableAt: new Date(request.availableAt ?? request.enqueuedAt ?? now), + state: 'ready', + attempts: 0, + redelivered: false, + token: null, + leaseUntil: null, + updatedAt: new Date(now), +}); + +const jobOf = (document) => { + const { _id: id } = document; + return { + id, + name: document.name, + data: document.data, + attempt: document.attempts, + maxAttempts: document.maxAttempts, + enqueuedAt: document.enqueuedAt?.getTime?.(), + redelivered: document.redelivered === true, + }; +}; + +// The filter of a write under a lease: the job is still lent to it. +const leased = (lease) => ({ _id: lease.job.id, token: lease.token, state: 'active' }); + +/** + * The source of an outbox in a collection. + * + * @internal + */ +class MongoOutboxSource { + id = 'mongodb'; + #collection; + #config; + + constructor(collection, config) { + this.#collection = collection; + this.#config = config; + if (config.changeStream) this.subscribe = (listener) => subscribe(collection, listener, config); + } + + async #settle(lease, set, inc) { + const update = { + $set: { ...set, token: null, leaseUntil: null, updatedAt: new Date(this.#config.now()) }, + }; + if (inc !== undefined) update.$inc = inc; + await this.#collection.updateOne(leased(lease), update); + } + + async enqueue(request, options = {}) { + if (!isRequest(request)) throw new TypeError('The MongoDB outbox takes a job request.'); + // An upsert, not an insert: a duplicate key would abort the transaction + // of the application. + const session = sessionOf(options.transaction); + const result = await this.#collection.updateOne( + { _id: request.id }, + { $setOnInsert: documentOfRequest(request, this.#config.now()) }, + { ...session, upsert: true }, + ); + if ((result?.upsertedCount ?? 0) > 0) return { id: request.id, duplicate: false }; + const held = await this.#collection.findOne( + { _id: request.id }, + { ...session, projection: { hash: 1 } }, + ); + const answer = { id: request.id, duplicate: true }; + if (typeof held?.hash === 'string') answer.existingHash = held.hash; + return answer; + } + + async #paused(now) { + const pause = await this.#collection.findOne( + { _id: PAUSE_ID }, + { projection: { pausedUntil: 1 } }, + ); + const until = pause?.pausedUntil; + return until instanceof Date && until > now ? until : undefined; + } + + async reserve(leaseMilliseconds) { + const now = new Date(this.#config.now()); + const paused = await this.#paused(now); + if (paused !== undefined) return { nextAvailableAt: paused.getTime() }; + // A job whose lease ran out is ready again, its attempt to run again. + await this.#collection.updateMany( + { state: 'active', leaseUntil: { $lte: now } }, + { + $set: { state: 'ready', token: null, leaseUntil: null, redelivered: true }, + $inc: { attempts: -1 }, + }, + ); + const token = randomUUID(); + const taken = documentOf( + await this.#collection.findOneAndUpdate( + { state: 'ready', availableAt: { $lte: now } }, + { + $set: { + state: 'active', + token, + leaseUntil: new Date(now.getTime() + leaseMilliseconds), + redelivered: false, + updatedAt: now, + }, + $inc: { attempts: 1 }, + }, + // As it was: whether it comes back, and its attempt before this one. + { + sort: { rank: 1, availableAt: 1, enqueuedAt: 1 }, + returnDocument: 'before', + includeResultMetadata: false, + }, + ), + ); + if (taken !== null) { + return { lease: { job: { ...jobOf(taken), attempt: taken.attempts + 1 }, token } }; + } + const next = await this.#collection.findOne( + { state: 'ready' }, + { sort: { availableAt: 1 }, projection: { availableAt: 1 } }, + ); + return next?.availableAt instanceof Date ? { nextAvailableAt: next.availableAt.getTime() } : {}; + } + + async extend(lease, leaseMilliseconds) { + const result = await this.#collection.updateOne(leased(lease), { + $set: { leaseUntil: new Date(this.#config.now() + leaseMilliseconds) }, + }); + return (result?.matchedCount ?? 0) > 0; + } + + ack(lease, result) { + const finishedAt = new Date(this.#config.now()); + return this.#settle(lease, { state: 'completed', result: result ?? null, finishedAt }); + } + + retry(lease, availableAt, failure) { + return this.#settle(lease, { state: 'ready', availableAt: new Date(availableAt), failure }); + } + + fail(lease, failure) { + const finishedAt = new Date(this.#config.now()); + return this.#settle(lease, { state: 'failed', failure, finishedAt }); + } + + release(lease, availableAt) { + const at = new Date(availableAt ?? this.#config.now()); + return this.#settle(lease, { state: 'ready', availableAt: at }, { attempts: -1 }); + } + + async pause(until) { + await this.#collection.updateOne( + { _id: PAUSE_ID }, + { $max: { pausedUntil: new Date(until) } }, + { upsert: true }, + ); + } +} + +// Wakes the readers on a change of the collection, where MongoDB has change +// streams (a replica set); the polling goes on without them. +const subscribe = (collection, listener, config) => { + const stream = collection.watch( + [{ $match: { operationType: { $in: ['insert', 'update', 'replace'] } } }], + { fullDocument: 'default' }, + ); + stream.on('change', () => listener()); + stream.on('error', (error) => { + config.log.write('warn', () => ({ error: error?.message }), 'queue.change_stream_failed'); + }); + return () => { + stream.close().catch(() => {}); + }; +}; + +/** + * Makes a mail queue in a collection of MongoDB. + * + * @param options `collection`, `db.collection('mail_outbox')`; + * `changeStream` (false), to wake the workers on a change (a replica set); + * `retainFinishedSeconds` (7 days), the time a finished job is kept, and so + * the window of its duplicates; `now`, `logger`, and the options of the + * polling driver. + * @returns The driver, with `source`, `ensureIndexes()` and `get(id)`. + * @throws {TypeError} If an option is invalid. + */ +const createMongoOutbox = (options = {}) => { + const collection = checkCollection(options.collection, 'createMongoOutbox'); + const changeStream = options.changeStream ?? false; + if (typeof changeStream !== 'boolean') throw new TypeError('changeStream must be a boolean.'); + if (changeStream && typeof collection.watch !== 'function') { + throw new TypeError('changeStream needs a collection with watch().'); + } + const config = { + now: checkOptionalFunction(options.now, 'now') ?? Date.now, + changeStream, + retainFinishedSeconds: checkPositiveInteger( + options.retainFinishedSeconds ?? DEFAULT_RETAIN_SECONDS, + 'retainFinishedSeconds', + ), + log: createLog(options.logger, { component: 'queue', queue: 'mongodb' }), + }; + const source = new MongoOutboxSource(collection, config); + const driver = createPollingDriver(source, { ...options, now: config.now }); + return { + ...driver, + source, + async ensureIndexes() { + await collection.createIndex( + { state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, + { name: 'mailer_ready' }, + ); + await collection.createIndex({ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }); + // A finished job goes after its time, and with it the memory of its key. + await collection.createIndex( + { finishedAt: 1 }, + { name: 'mailer_finished', expireAfterSeconds: config.retainFinishedSeconds }, + ); + }, + async get(id) { + const document = await collection.findOne({ _id: id }); + // The pause of the queue is no job. + if (document === null || document === undefined || id === PAUSE_ID) return undefined; + const job = jobOf(document); + return { + ...job, + state: document.state, + priority: document.priority, + failure: document.failure, + result: document.result, + }; + }, + }; +}; + +module.exports = { PAUSE_ID, createMongoOutbox }; diff --git a/tests/helpers/mongodb/fake-collection.js b/tests/helpers/mongodb/fake-collection.js new file mode 100644 index 0000000..ade4bd4 --- /dev/null +++ b/tests/helpers/mongodb/fake-collection.js @@ -0,0 +1,228 @@ +// Not ported: a collection of MongoDB in memory for the tests of ./mongodb, +// with what the stores call of it: filters of equality, `$lte` and `$or`; +// `$set`, `$setOnInsert`, `$inc` and `$max`; an upsert that meets an id held +// fails as MongoDB does (E11000); sorts; and a change stream. + +const comparable = (value) => (value instanceof Date ? value.getTime() : value); + +const equal = (a, b) => { + if (a === null || a === undefined) return b === null || b === undefined; + return comparable(a) === comparable(b); +}; + +const matches = (document, filter) => { + for (const key of Object.keys(filter)) { + const condition = filter[key]; + if (key === '$or') { + let any = false; + for (const branch of condition) { + any ||= matches(document, branch); + } + if (!any) return false; + continue; + } + const value = document[key]; + if (condition !== null && typeof condition === 'object' && !(condition instanceof Date)) { + if ('$lte' in condition) { + if ( + value === null || + value === undefined || + !(comparable(value) <= comparable(condition.$lte)) + ) { + return false; + } + continue; + } + if ('$in' in condition) { + if (!condition.$in.some((candidate) => equal(value, candidate))) return false; + continue; + } + } + if (!equal(value, condition)) return false; + } + return true; +}; + +const clone = (value) => structuredClone(value); + +// MongoDB's key, by name, so the lint of dangling underscores reads it once. +const ID = '_id'; + +const apply = (document, update, inserting) => { + for (const [field, value] of Object.entries(update.$set ?? {})) { + document[field] = clone(value); + } + if (inserting) { + for (const [field, value] of Object.entries(update.$setOnInsert ?? {})) { + document[field] = clone(value); + } + } + for (const [field, value] of Object.entries(update.$inc ?? {})) { + document[field] = (document[field] ?? 0) + value; + } + for (const [field, value] of Object.entries(update.$max ?? {})) { + if (document[field] === undefined || comparable(value) > comparable(document[field])) { + document[field] = clone(value); + } + } +}; + +const sortBy = (documents, sort = {}) => + documents.toSorted((a, b) => { + for (const [field, direction] of Object.entries(sort)) { + const left = comparable(a[field]); + const right = comparable(b[field]); + if (left === right) continue; + if (left === undefined || left === null) return -direction; + if (right === undefined || right === null) return direction; + return left < right ? -direction : direction; + } + return 0; + }); + +const project = (document, projection) => { + if (document === undefined) return null; + if (projection === undefined) return clone(document); + const projected = { [ID]: document[ID] }; + for (const field of Object.keys(projection)) { + if (field in document) projected[field] = clone(document[field]); + } + return projected; +}; + +const duplicateKey = (id) => + Object.assign(new Error(`E11000 duplicate key error collection: _id ${JSON.stringify(id)}`), { + code: 11000, + }); + +const fakeCollection = (options = {}) => { + const documents = new Map(); + const calls = []; + const indexes = []; + const streams = []; + const notify = (operationType) => { + for (const stream of streams) { + stream.listeners.get('change')?.({ operationType }); + } + }; + const insertFrom = (filter, update) => { + const document = {}; + for (const [key, value] of Object.entries(filter)) { + if ( + !key.startsWith('$') && + (value === null || typeof value !== 'object' || value instanceof Date) + ) { + document[key] = clone(value); + } + } + apply(document, update, true); + if (documents.has(document[ID])) throw duplicateKey(document[ID]); + documents.set(document[ID], document); + return document; + }; + const collection = { + documents, + calls, + indexes, + streams, + async updateOne(filter, update, opts = {}) { + calls.push(['updateOne', filter, update, opts]); + if (options.failOn?.updateOne) throw options.failOn.updateOne; + for (const document of documents.values()) { + if (matches(document, filter)) { + apply(document, update, false); + notify('update'); + return { matchedCount: 1, modifiedCount: 1, upsertedCount: 0 }; + } + } + if (!opts.upsert) return { matchedCount: 0, modifiedCount: 0, upsertedCount: 0 }; + insertFrom(filter, update); + notify('insert'); + return { matchedCount: 0, modifiedCount: 0, upsertedCount: 1 }; + }, + async updateMany(filter, update, opts = {}) { + calls.push(['updateMany', filter, update, opts]); + let count = 0; + for (const document of documents.values()) { + if (matches(document, filter)) { + apply(document, update, false); + count++; + } + } + return { matchedCount: count, modifiedCount: count }; + }, + async findOne(filter, opts = {}) { + calls.push(['findOne', filter, opts]); + const found = sortBy( + [...documents.values()].filter((document) => matches(document, filter)), + opts.sort, + ); + return project(found[0], opts.projection); + }, + async findOneAndUpdate(filter, update, opts = {}) { + calls.push(['findOneAndUpdate', filter, update, opts]); + const found = sortBy( + [...documents.values()].filter((document) => matches(document, filter)), + opts.sort, + )[0]; + if (found === undefined) return options.modifyResult ? { ok: 1, value: null } : null; + const before = clone(found); + apply(found, update, false); + const value = opts.returnDocument === 'after' ? clone(found) : before; + return options.modifyResult ? { ok: 1, value } : value; + }, + async deleteOne(filter, opts = {}) { + calls.push(['deleteOne', filter, opts]); + for (const [id, document] of documents) { + if (matches(document, filter)) { + documents.delete(id); + return { deletedCount: 1 }; + } + } + return { deletedCount: 0 }; + }, + find(filter, opts = {}) { + calls.push(['find', filter, opts]); + let sort; + const cursor = { + sort(spec) { + sort = spec; + return cursor; + }, + async toArray() { + return sortBy( + [...documents.values()].filter((document) => matches(document, filter)), + sort, + ).map(clone); + }, + }; + return cursor; + }, + async createIndex(spec, opts = {}) { + indexes.push([spec, opts]); + return opts.name ?? 'index'; + }, + watch(pipeline, opts) { + const stream = { + pipeline, + opts, + listeners: new Map(), + closed: false, + on(event, listener) { + stream.listeners.set(event, listener); + return stream; + }, + async close() { + stream.closed = true; + streams.splice(streams.indexOf(stream), 1); + if (options.failOn?.close) throw options.failOn.close; + }, + }; + streams.push(stream); + return stream; + }, + }; + return collection; +}; + +module.exports = { fakeCollection }; diff --git a/tests/mongodb/docker-compose.yml b/tests/mongodb/docker-compose.yml new file mode 100644 index 0000000..d816452 --- /dev/null +++ b/tests/mongodb/docker-compose.yml @@ -0,0 +1,24 @@ +# Not ported: a local MongoDB for `pnpm test:mongodb`, a replica set of one, +# as transactions and change streams need one: +# docker compose -f tests/mongodb/docker-compose.yml up -d --wait +# MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0' pnpm test:mongodb +services: + mongo: + image: mongo:7 + container_name: mailer-mongo + command: ['--replSet', 'rs0', '--bind_ip_all', '--port', '27027'] + ports: + - '27027:27027' + healthcheck: + # Initiates the replica set on its first run, under the name the host reaches it by. + test: + - CMD + - mongosh + - --port + - '27027' + - --quiet + - --eval + - "try { rs.status().ok } catch (e) { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host: 'localhost:27027' }] }).ok }" + interval: 3s + timeout: 10s + retries: 30 diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js new file mode 100644 index 0000000..5028608 --- /dev/null +++ b/tests/mongodb/outbox.mongodb.test.js @@ -0,0 +1,196 @@ +// Not ported: the stores of ./mongodb and the mail queue of ./queue on +// MongoDB itself (tests/mongodb/docker-compose.yml), skipped unless +// MONGODB_URL is set. Each test has a database of its own, dropped after. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const { MongoClient } = require('mongodb'); +const { IdempotencyTransport } = require('../../src/idempotency/index.js'); +const { createMailQueue } = require('../../src/queue/index.js'); +const { + createMongoAuditSink, + createMongoIdempotencyStore, + createMongoOutbox, +} = require('../../src/mongodb/index.js'); +const { fakeTransport } = require('../helpers/decorators/fake.js'); + +const url = process.env.MONGODB_URL; +const skip = url === undefined || url === ''; + +const until = async (check, timeout = 10_000) => { + const start = Date.now(); + for (;;) { + const value = await check(); + if (value) return value; + if (Date.now() - start > timeout) throw new Error('timed out'); + await new Promise((resolve) => setTimeout(resolve, 25)); + } +}; + +const job = (extra = {}) => ({ + kind: 'content', + from: 'app@example.com', + to: 'ada@example.com', + subject: 'Hello', + content: { text: 'Hi' }, + idempotencyKey: randomUUID(), + ...extra, +}); + +const setup = async (t, options = {}) => { + const client = new MongoClient(url); + await client.connect(); + const db = client.db(`mailer_test_${randomUUID().replaceAll('-', '')}`); + const outbox = createMongoOutbox({ + collection: db.collection('mail_outbox'), + pollIntervalMilliseconds: 20, + ...options, + }); + const audit = createMongoAuditSink({ collection: db.collection('mail_audit') }); + const store = createMongoIdempotencyStore({ collection: db.collection('mail_idempotency') }); + await outbox.ensureIndexes(); + await audit.ensureIndexes(); + await store.ensureIndexes(); + const queue = createMailQueue({ driver: outbox, audit }); + const workers = []; + t.after(async () => { + for (const worker of workers) await worker.close({ timeoutMilliseconds: 1000 }); + await db.dropDatabase(); + await client.close(); + }); + const worker = (workerOptions) => { + const consumer = queue.createWorker(workerOptions); + workers.push(consumer); + return consumer; + }; + return { client, db, outbox, audit, store, queue, worker }; +}; + +test( + 'MongoDB (replica set): a job is written in the transaction of its cause, or not at all', + { skip }, + async (t) => { + const { client, db, outbox, audit, queue, worker } = await setup(t); + const users = db.collection('users'); + // Aborted: neither the user's write nor the job, nor its audit event. + const aborted = client.startSession(); + let abortedId; + await aborted + .withTransaction(async () => { + await users.insertOne({ _id: 'u1', resetToken: 't1' }, { session: aborted }); + abortedId = (await queue.enqueue(job(), { transaction: aborted })).id; + throw new Error('roll back'); + }) + .catch((error) => assert.equal(error.message, 'roll back')); + await aborted.endSession(); + assert.equal(await outbox.get(abortedId), undefined); + assert.equal(await users.findOne({ _id: 'u1' }), null); + // Committed, with the same message enqueued twice in it: one job, and + // the duplicate does not abort the transaction. + const session = client.startSession(); + const input = job(); + let enqueued; + await session.withTransaction(async () => { + await users.insertOne({ _id: 'u2', resetToken: 't2' }, { session }); + enqueued = await queue.enqueue(input, { transaction: session }); + const again = await queue.enqueue(input, { transaction: session }); + assert.equal(again.duplicate, true); + }); + await session.endSession(); + assert.ok(await users.findOne({ _id: 'u2' })); + const transport = fakeTransport(); + worker({ transport }); + await until(async () => (await outbox.get(enqueued.id))?.state === 'completed'); + assert.equal(transport.calls.length, 1); + assert.equal(await audit.stateOf(enqueued.correlationId), 'accepted'); + assert.deepEqual( + (await audit.eventsOf(enqueued.correlationId)).map((event) => event.state), + ['queued', 'rendered', 'submitted', 'accepted'], + ); + await assert.rejects(queue.enqueue({ ...input, to: 'other@example.com' }), { + code: 'queue.key_reused', + }); + }, +); + +test( + 'MongoDB (replica set): workers of several processes run each job once', + { skip }, + async (t) => { + const { outbox, queue, worker } = await setup(t); + const transport = fakeTransport(); + const ids = []; + for (let index = 0; index < 30; index++) { + ids.push((await queue.enqueue(job({ subject: `m${index}` }))).id); + } + worker({ transport, concurrency: 3 }); + worker({ transport, concurrency: 3 }); + await until(async () => transport.calls.length >= 30); + await new Promise((resolve) => setTimeout(resolve, 200)); + const subjects = transport.calls.map((call) => call.message.subject); + assert.equal(subjects.length, 30); + assert.equal(new Set(subjects).size, 30); + for (const id of ids) assert.equal((await outbox.get(id)).state, 'completed'); + }, +); + +test( + 'MongoDB (replica set): a job whose lease ran out goes to another worker, redelivered', + { skip }, + async (t) => { + const { outbox, queue } = await setup(t, { leaseMilliseconds: 100 }); + const { id } = await queue.enqueue(job()); + const { lease } = await outbox.source.reserve(100); + assert.equal(lease.job.id, id); + await new Promise((resolve) => setTimeout(resolve, 150)); + const again = await outbox.source.reserve(1000); + assert.deepEqual( + [again.lease.job.id, again.lease.job.attempt, again.lease.job.redelivered], + [id, 1, true], + ); + assert.equal(await outbox.source.extend(lease, 1000), false); + await outbox.source.ack(lease); + assert.equal((await outbox.get(id)).state, 'active'); + await outbox.source.ack(again.lease, { ok: 1 }); + assert.equal((await outbox.get(id)).state, 'completed'); + }, +); + +test( + 'MongoDB (replica set): of concurrent reserves of a key, one holds it', + { skip }, + async (t) => { + const { store } = await setup(t); + const reservations = await Promise.all( + Array.from({ length: 10 }, () => store.reserve('k', 'f', 10_000)), + ); + const tokens = reservations.filter((reservation) => reservation.token !== undefined); + assert.equal(tokens.length, 1); + assert.equal( + reservations.filter((reservation) => reservation.record?.state === 'pending').length, + 9, + ); + // Behind it, a message is sent once however many workers send it. + const inner = fakeTransport(); + const transport = new IdempotencyTransport(inner, { store }); + const { createMessage } = require('../../index.js'); + const message = createMessage({ ...job(), idempotencyKey: 'once' }); + await Promise.all([transport.send(message), transport.send(message), transport.send(message)]); + assert.equal(inner.calls.length, 1); + }, +); + +test('MongoDB (replica set): a change stream wakes the workers at once', { skip }, async (t) => { + const { outbox, queue, worker } = await setup(t, { + changeStream: true, + pollIntervalMilliseconds: 10_000, + }); + const transport = fakeTransport(); + worker({ transport }); + // The first reserve finds nothing; the next poll is ten seconds away. + await new Promise((resolve) => setTimeout(resolve, 300)); + const started = Date.now(); + const { id } = await queue.enqueue(job()); + await until(async () => (await outbox.get(id))?.state === 'completed', 5000); + assert.ok(Date.now() - started < 3000, 'woken by the change stream'); +}); diff --git a/tests/types/mongodb.test-d.ts b/tests/types/mongodb.test-d.ts new file mode 100644 index 0000000..513c366 --- /dev/null +++ b/tests/types/mongodb.test-d.ts @@ -0,0 +1,59 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { MongoClient, type Collection } from 'mongodb'; +import type { MessageState, QueueDriver } from '../../index.js'; +import { IdempotencyTransport, type IdempotencyStore } from '../../src/idempotency/index.js'; +import { createMailQueue } from '../../src/queue/index.js'; +import { + createMongoAuditSink, + createMongoIdempotencyStore, + createMongoOutbox, + type MongoAuditSink, + type MongoCollectionLike, + type MongoOutbox, + type MongoOutboxJob, +} from '../../src/mongodb/index.js'; +import type { Transport } from '../../index.js'; + +// The collections of the real driver fit the structural type. +const client = new MongoClient('mongodb://localhost:27017'); +const db = client.db('app'); +expectAssignable(db.collection('mail_outbox')); +declare const typed: Collection<{ _id: string; state: string }>; +expectAssignable(typed); + +const outbox = createMongoOutbox({ + collection: db.collection('mail_outbox'), + changeStream: true, + retainFinishedSeconds: 3600, + leaseMilliseconds: 30_000, + pollIntervalMilliseconds: 500, +}); +expectType(outbox); +expectAssignable(outbox); +expectType>(outbox.ensureIndexes()); +expectType>(outbox.get('id')); +expectError(createMongoOutbox({})); + +const audit = createMongoAuditSink({ collection: db.collection('mail_audit') }); +expectType(audit); +expectType>(audit.stateOf('c')); +const store = createMongoIdempotencyStore({ + collection: db.collection('mail_idempotency'), + now: Date.now, +}); +expectAssignable(store); + +declare const transport: Transport; +const queue = createMailQueue({ driver: outbox, audit }); +const session = client.startSession(); +await queue.enqueue( + { + kind: 'content', + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 't' }, + }, + { transaction: session }, +); +queue.createWorker({ transport: new IdempotencyTransport(transport, { store }) }); diff --git a/tests/unit/graph.test.js b/tests/unit/graph.test.js index ef84178..dc9cb2a 100644 --- a/tests/unit/graph.test.js +++ b/tests/unit/graph.test.js @@ -103,6 +103,8 @@ const SUBPATH_EDGES = [ // The worker of ./queue renders the subject and content of a template job // as ./render does, the text derived from the HTML. 'queue/handler.js -> render/renderer.js', + // The audit sink of ./mongodb reads the state of a message as ./audit does. + 'mongodb/audit-sink.js -> audit/states.js', ]; const subpathOf = (name) => { diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js new file mode 100644 index 0000000..c50abb4 --- /dev/null +++ b/tests/unit/mongodb.test.js @@ -0,0 +1,405 @@ +// Not ported: tests of ./mongodb against a collection in memory +// (tests/helpers/mongodb); tests/mongodb runs them on MongoDB itself. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMemoryAuditSink } = require('../../src/audit/index.js'); +const { IdempotencyTransport } = require('../../src/idempotency/index.js'); +const { createMailQueue } = require('../../src/queue/index.js'); +const { + createMongoAuditSink, + createMongoIdempotencyStore, + createMongoOutbox, +} = require('../../src/mongodb/index.js'); +const { fakeCollection } = require('../helpers/mongodb/fake-collection.js'); +const { fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const requestOf = (id, extra = {}) => ({ + id, + name: 'mail', + data: { n: id }, + hash: `h-${id}`, + priority: 'normal', + maxAttempts: 3, + enqueuedAt: 1000, + unique: true, + ...extra, +}); + +const outboxOf = (options = {}) => { + let time = options.start ?? 10_000; + const collection = options.collection ?? fakeCollection(); + const outbox = createMongoOutbox({ collection, now: () => time, ...options.extra }); + return { + collection, + outbox, + source: outbox.source, + now: () => time, + advance: (ms) => { + time += ms; + }, + }; +}; + +test('MongoDB: a job is written once under its id, in the transaction it is given', async () => { + const { collection, source } = outboxOf(); + const session = { id: 'session-1' }; + assert.deepStrictEqual(await source.enqueue(requestOf('a'), { transaction: session }), { + id: 'a', + duplicate: false, + }); + // An upsert, not an insert, so that a duplicate does not abort the transaction. + const [, filter, update, options] = collection.calls[0]; + assert.deepStrictEqual([filter, options], [{ _id: 'a' }, { session, upsert: true }]); + assert.strictEqual(update.$setOnInsert.state, 'ready'); + assert.strictEqual(update.$setOnInsert.rank, 1); + assert.deepStrictEqual( + await source.enqueue(requestOf('a', { hash: 'other' }), { transaction: session }), + { + id: 'a', + duplicate: true, + existingHash: 'h-a', + }, + ); + assert.strictEqual(collection.calls[2][2].session, session); + const stored = collection.documents.get('a'); + assert.ok(stored.availableAt instanceof Date && stored.enqueuedAt instanceof Date); + // A held job without a hash, and the requests it refuses. + collection.documents.get('a').hash = undefined; + assert.deepStrictEqual(await source.enqueue(requestOf('a')), { id: 'a', duplicate: true }); + for (const bad of [ + {}, + requestOf('x:y'), + requestOf('p', { priority: 'urgent' }), + requestOf('m', { maxAttempts: 0 }), + ]) { + await assert.rejects(source.enqueue(bad), /takes a job request/); + } +}); + +test('MongoDB: jobs are lent by priority, then time, their attempts counted', async () => { + const { source, advance } = outboxOf(); + await source.enqueue(requestOf('low', { priority: 'low', enqueuedAt: 1 })); + await source.enqueue(requestOf('n2', { enqueuedAt: 3 })); + await source.enqueue(requestOf('n1', { enqueuedAt: 2 })); + await source.enqueue(requestOf('high', { priority: 'high', enqueuedAt: 4 })); + await source.enqueue(requestOf('later', { availableAt: 60_000 })); + const order = []; + for (;;) { + const { lease, nextAvailableAt } = await source.reserve(1000); + if (lease === undefined) { + assert.strictEqual(nextAvailableAt, 60_000); + break; + } + order.push([lease.job.id, lease.job.attempt, lease.job.redelivered]); + await source.ack(lease, { ok: true }); + } + assert.deepStrictEqual(order, [ + ['high', 1, false], + ['n1', 1, false], + ['n2', 1, false], + ['low', 1, false], + ]); + advance(60_000); + const { lease } = await source.reserve(1000); + assert.strictEqual(lease.job.id, 'later'); + assert.deepStrictEqual(Object.keys(lease.job), [ + 'id', + 'name', + 'data', + 'attempt', + 'maxAttempts', + 'enqueuedAt', + 'redelivered', + ]); + await source.ack(lease); + assert.deepStrictEqual(await source.reserve(1000), {}); +}); + +test('MongoDB: a lease that runs out gives the job to another, the late settle ignored', async () => { + const { source, outbox, advance, now } = outboxOf(); + await source.enqueue(requestOf('a')); + const first = (await source.reserve(1000)).lease; + assert.strictEqual(await source.extend(first, 1000), true); + advance(1001); + const second = (await source.reserve(1000)).lease; + assert.deepStrictEqual([second.job.attempt, second.job.redelivered], [1, true]); + // The first holder's writes do nothing now. + assert.strictEqual(await source.extend(first, 1000), false); + await source.ack(first, { from: 'first' }); + await source.fail(first, { code: 'x', reason: 'late' }); + assert.strictEqual((await outbox.get('a')).state, 'active'); + await source.retry(second, now() + 10_000, { code: 'http.503', reason: 'busy' }); + const retried = await outbox.get('a'); + assert.deepStrictEqual( + [retried.state, retried.failure.code, retried.attempt], + ['ready', 'http.503', 1], + ); + advance(10_000); + const third = (await source.reserve(1000)).lease; + assert.deepStrictEqual([third.job.attempt, third.job.redelivered], [2, false]); + // A release gives it back without spending the attempt. + await source.release(third); + const fourth = (await source.reserve(1000)).lease; + assert.strictEqual(fourth.job.attempt, 2); + const later = now() + 5000; + await source.release(fourth, later); + assert.deepStrictEqual(await source.reserve(1000), { nextAvailableAt: later }); + advance(5000); + const fifth = (await source.reserve(1000)).lease; + await source.fail(fifth, { code: 'x.rejected', reason: 'no such user' }); + const failed = await outbox.get('a'); + assert.deepStrictEqual([failed.state, failed.failure.code], ['failed', 'x.rejected']); + assert.strictEqual(await outbox.get('missing'), undefined); +}); + +test('MongoDB: a pause of the queue holds every worker until it ends', async () => { + const { source, outbox, advance } = outboxOf(); + await source.enqueue(requestOf('a')); + await source.pause(15_000); + await source.pause(12_000); + assert.deepStrictEqual(await source.reserve(1000), { nextAvailableAt: 15_000 }); + assert.strictEqual(await outbox.get('mailer:pause'), undefined); + advance(5000); + assert.strictEqual((await source.reserve(1000)).lease.job.id, 'a'); +}); + +test('MongoDB: an older driver’s findOneAndUpdate, which gives { value }, is read too', async () => { + const { source } = outboxOf({ collection: fakeCollection({ modifyResult: true }) }); + await source.enqueue(requestOf('a')); + assert.strictEqual((await source.reserve(1000)).lease.job.id, 'a'); + assert.deepStrictEqual(await source.reserve(1000), {}); +}); + +test('MongoDB: the outbox makes its indexes, and wakes its workers on a change stream', async () => { + const records = []; + const { collection, outbox } = outboxOf({ + extra: { + changeStream: true, + retainFinishedSeconds: 3600, + logger: { warn: (record, message) => records.push([message, record.error]) }, + }, + }); + await outbox.ensureIndexes(); + assert.deepStrictEqual(collection.indexes, [ + [{ state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, { name: 'mailer_ready' }], + [{ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }], + [{ finishedAt: 1 }, { name: 'mailer_finished', expireAfterSeconds: 3600 }], + ]); + let woken = 0; + const unsubscribe = outbox.source.subscribe(() => woken++); + const [stream] = collection.streams; + assert.deepStrictEqual(stream.pipeline, [ + { $match: { operationType: { $in: ['insert', 'update', 'replace'] } } }, + ]); + await outbox.source.enqueue(requestOf('a')); + assert.strictEqual(woken, 1); + stream.listeners.get('error')(new Error('not a replica set')); + assert.deepStrictEqual(records, [['queue.change_stream_failed', 'not a replica set']]); + unsubscribe(); + assert.strictEqual(collection.streams.length, 0); + // A close that fails goes unheard. + const failing = outboxOf({ + collection: fakeCollection({ failOn: { close: new Error('closed') } }), + extra: { changeStream: true }, + }); + failing.outbox.source.subscribe(() => {})(); + await new Promise(setImmediate); +}); + +test('MongoDB: the outbox checks its options', () => { + assert.throws( + () => createMongoOutbox(), + /needs a collection of the MongoDB driver, with updateOne/, + ); + assert.throws(() => createMongoOutbox({ collection: { updateOne() {} } }), /with updateMany/); + const collection = fakeCollection(); + assert.throws( + () => createMongoOutbox({ collection, changeStream: 'yes' }), + /changeStream must be a boolean/, + ); + assert.throws( + () => + createMongoOutbox({ collection: { ...collection, watch: undefined }, changeStream: true }), + /changeStream needs a collection with watch/, + ); + assert.throws( + () => createMongoOutbox({ collection, retainFinishedSeconds: 0 }), + /retainFinishedSeconds/, + ); + assert.throws(() => createMongoOutbox({ collection, now: 1 }), /now/); + // Without a change stream, there is nothing to subscribe to. + assert.strictEqual(createMongoOutbox({ collection }).source.subscribe, undefined); +}); + +test('MongoDB: the mail queue runs on the outbox, its audit in MongoDB', async () => { + const collection = fakeCollection(); + const auditCollection = fakeCollection(); + const outbox = createMongoOutbox({ collection, pollIntervalMilliseconds: 5 }); + const audit = createMongoAuditSink({ collection: auditCollection }); + const queue = createMailQueue({ driver: outbox, audit }); + const session = { id: 's' }; + const { id, correlationId } = await queue.enqueue( + { + kind: 'content', + from: 'app@example.com', + to: 'ada@example.com', + subject: 'Hi', + content: { text: 'Hello' }, + idempotencyKey: 'welcome:1', + }, + { transaction: session }, + ); + // The queued event goes in the transaction of the enqueue. + assert.strictEqual(auditCollection.calls[0][3].session, session); + const transport = fakeTransport(() => sent('p-1')); + const worker = queue.createWorker({ transport }); + for (let index = 0; index < 100 && (await outbox.get(id)).state !== 'completed'; index++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await worker.close(); + assert.strictEqual((await outbox.get(id)).state, 'completed'); + assert.deepStrictEqual((await outbox.get(id)).result, { + provider: 'fake', + providerMessageId: 'p-1', + }); + assert.strictEqual(await audit.stateOf(correlationId), 'accepted'); + assert.deepStrictEqual( + (await audit.eventsOf(correlationId)).map((event) => event.state), + ['queued', 'rendered', 'submitted', 'accepted'], + ); +}); + +test('MongoDB: the audit sink writes an event once by its id', async () => { + const collection = fakeCollection(); + const sink = createMongoAuditSink({ collection }); + const event = { + id: 'e1', + state: 'queued', + at: '2026-10-18T09:00:00.000Z', + source: 'producer', + correlationId: 'c1', + attempt: 0, + }; + // Two events of one time and attempt read in the order of their states. + await sink.append({ ...event, id: 'e0', state: 'submitted', attempt: 1 }); + await sink.append({ ...event, id: 'e00', state: 'rendered', attempt: 1 }); + assert.deepStrictEqual( + (await sink.eventsOf('c1')).map((e) => e.state), + ['rendered', 'submitted'], + ); + collection.documents.clear(); + await sink.append(event); + await sink.append({ ...event, state: 'failed' }); + await sink.append({ ...event, id: 'e2', state: 'accepted', at: '2026-10-18T09:00:01.000Z' }); + const stored = collection.documents.get('e1'); + assert.deepStrictEqual( + [stored.state, stored.at instanceof Date, stored.rank], + ['queued', true, 0], + ); + assert.deepStrictEqual(await sink.eventsOf('c1'), [ + event, + { ...event, id: 'e2', state: 'accepted', at: '2026-10-18T09:00:01.000Z' }, + ]); + assert.strictEqual(await sink.stateOf('c1'), 'accepted'); + assert.strictEqual(await sink.stateOf('none'), undefined); + await assert.rejects(sink.append({}), /has an id/); + await sink.ensureIndexes(); + assert.deepStrictEqual(collection.indexes, [ + [{ correlationId: 1, at: 1, attempt: 1, rank: 1 }, { name: 'mailer_correlation' }], + [{ jobId: 1 }, { name: 'mailer_job', sparse: true }], + ]); + // A sink keeps the memory sink's contract. + const memory = createMemoryAuditSink(); + memory.append(event); + assert.deepStrictEqual(memory.events[0], (await sink.eventsOf('c1'))[0]); + assert.throws(() => createMongoAuditSink(), /createMongoAuditSink needs a collection/); +}); + +test('MongoDB: a key is reserved once, by one atomic step, and fenced by its token', async () => { + let time = 1000; + const collection = fakeCollection(); + const store = createMongoIdempotencyStore({ collection, now: () => time }); + const { token } = await store.reserve('k', 'f1', 100); + assert.match(token, /^[0-9a-f-]{36}$/); + assert.deepStrictEqual(await store.reserve('k', 'f1', 100), { + record: { state: 'pending', fingerprint: 'f1' }, + }); + // A lease that ran out frees the key for another. + time += 101; + const second = await store.reserve('k', 'f1', 100); + assert.notStrictEqual(second.token, token); + // The first holder can neither release nor complete it now. + await store.release('k', token); + await store.complete( + 'k', + token, + { fingerprint: 'f1', receipt: { successful: true, messageId: 'late' } }, + 1000, + ); + assert.strictEqual(collection.documents.get('k').token, second.token); + await store.complete( + 'k', + second.token, + { fingerprint: 'f1', receipt: { successful: true, messageId: 'm1' } }, + 1000, + ); + assert.deepStrictEqual(await store.reserve('k', 'f1', 100), { + record: { state: 'done', fingerprint: 'f1', receipt: { successful: true, messageId: 'm1' } }, + }); + // A failed send frees its key. + const third = await store.reserve('free', 'f', 100); + await store.release('free', third.token); + assert.strictEqual(collection.documents.has('free'), false); + // A key not held is completed as given. + await store.complete( + 'new', + 'tok', + { fingerprint: 'f', receipt: { successful: true, messageId: 'x' } }, + 50, + ); + assert.strictEqual(collection.documents.get('new').state, 'done'); + await store.ensureIndexes(); + assert.deepStrictEqual(collection.indexes, [ + [{ expiresAt: 1 }, { name: 'mailer_expires', expireAfterSeconds: 0 }], + ]); +}); + +test('MongoDB: the store surfaces what is not a duplicate key, and reads a key dropped meanwhile again', async () => { + const outage = Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }); + const broken = createMongoIdempotencyStore({ + collection: fakeCollection({ failOn: { updateOne: outage } }), + }); + await assert.rejects(broken.reserve('k', 'f', 10), /connection reset/); + await assert.rejects( + broken.complete('k', 't', { fingerprint: 'f', receipt: {} }, 10), + /connection reset/, + ); + // A key held at the upsert and gone at the read: tried again, then refused. + const collection = fakeCollection(); + const store = createMongoIdempotencyStore({ collection }); + collection.updateOne = async () => { + throw Object.assign(new Error('E11000'), { code: 11000 }); + }; + collection.findOne = async () => null; + await assert.rejects(store.reserve('k', 'f', 10), /could not be reserved in 3 tries/); + assert.throws(() => createMongoIdempotencyStore(), /needs a collection/); + assert.throws(() => createMongoIdempotencyStore({ collection: fakeCollection(), now: 1 }), /now/); +}); + +test('MongoDB: IdempotencyTransport sends a key once over the shared store', async () => { + const store = createMongoIdempotencyStore({ collection: fakeCollection() }); + const inner = fakeTransport(); + const transport = new IdempotencyTransport(inner, { store }); + const { createMessage } = require('../../index.js'); + const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 't' }, + idempotencyKey: 'once', + }); + const first = await transport.send(message); + const again = await transport.send(message); + assert.strictEqual(inner.calls.length, 1); + assert.strictEqual(again.messageId, first.messageId); +}); From dd7f07116c7e30485e048c97ac2c47b1803e0d4f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 22:21:32 +0300 Subject: [PATCH 04/83] chore(release): v1.2.0 The version is 1.2.0 in package.json and src/version.js, and the [Unreleased] entry of CHANGELOG.md becomes 1.2.0 (2026-10-18), with its Tooling and comparison link. The status of README, CLAUDE.md, the docs and llms.txt names what 1.2 adds instead of calling it unreleased; the comparison gives this package 1.2.0 and its test count; and the threat model of SECURITY.md covers the jobs of the queue, the templates and the audit. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 23 ++++++++++++++++++++--- CLAUDE.md | 20 ++++++++++---------- README.md | 4 ++-- SECURITY.md | 21 ++++++++++++++++++--- docs/comparison.json | 4 ++-- docs/guide/comparison.md | 4 ++-- docs/guide/getting-started.md | 4 ++-- docs/guide/why.md | 10 +++++----- docs/index.md | 2 +- docs/public/llms.txt | 11 +++++------ package.json | 2 +- src/version.js | 2 +- 12 files changed, 69 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 921ed83..2e210e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -The reliability layer: a persistent queue whose workers render templates, and the audit of the -states of a message. Everything is additive. +## [1.2.0] - 2026-10-18 + +The reliability layer of the roadmap, on the transport of 1.1: a persistent queue, on BullMQ or a +transactional outbox in MongoDB, whose workers render templates of the engine the application +uses and know a permanent failure from a temporary one, and the audit of every state of a +message. Everything is additive: nothing of 1.1 changes, and the package still has no runtime +dependency, BullMQ, MongoDB and the template engines being passed in. ### Added @@ -59,6 +64,17 @@ states of a message. Everything is additive. - **The root**: the contracts of the reliability layer as types (`QueueDriver`, `QueueSource`, `QueueJob`, `QueueOutcome`, `MessageState`, `AuditEvent`, `AuditSink`), with no code. +### Tooling + +- **Opt-in suites on the real services:** `pnpm test:bullmq` on BullMQ and Valkey + (`tests/bullmq/docker-compose.yml`) and `pnpm test:mongodb` on a MongoDB replica set + (`tests/mongodb/docker-compose.yml`), outside the gates, as `test:mailpit` is. +- **devDependencies for the tests:** the nine template engines, juice and css-inline, BullMQ, + ioredis and the MongoDB driver; `src/` requires none of them, as + `tests/unit/dependencies.test.js` checks. +- **Shared internals:** the backoff, the classification of thrown errors, the fingerprint of a + message and the path check moved to `src/internal/`, with no change of behavior. + ## [1.1.0] - 2026-10-18 The features of three libraries of the [comparison](https://mailerat.vercel.app/guide/comparison) @@ -668,6 +684,7 @@ for code written against upyo 0.7: `./resend`, `./sendgrid`, `./mailgun`, `./plunk` and `./jmap` used to retry no 4xx at all, and waited without jitter, the first three without a cap. -[unreleased]: https://github.com/Alexis-Technologies/mailer/compare/v1.1.0...HEAD +[unreleased]: https://github.com/Alexis-Technologies/mailer/compare/v1.2.0...HEAD +[1.2.0]: https://github.com/Alexis-Technologies/mailer/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/Alexis-Technologies/mailer/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/Alexis-Technologies/mailer/releases/tag/v1.0.0 diff --git a/CLAUDE.md b/CLAUDE.md index a6b3065..4efa1ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,19 +20,19 @@ Nodemailer, emailjs or Upyo, made of two parts: of delivery events, suppression, idempotency and failover between providers. 1.1 has the model of delivery events with the verified webhooks of fifteen providers (`./webhooks`) and the idempotency decorator whose store interface the outbox is to implement (`./idempotency`). - Unreleased: the queue (`./queue`, a driver-neutral contract, its worker and a memory queue; - `./bullmq`, the BullMQ driver), templates by name over nine engines passed in (`./templates`) - and the audit of the states of a message (`./audit`). + 1.2 adds the queue (`./queue`, a driver-neutral contract, its worker and a memory queue; + `./bullmq`, the BullMQ driver; `./mongodb`, the transactional outbox with its audit sink and + idempotency store), templates by name over nine engines passed in (`./templates`) and the + audit of the states of a message (`./audit`). It has zero runtime dependencies and runs on Node.js ≥ 22, and on Bun and Deno through their Node compatibility (see "Platform"; no browser entry). -**Status: 1.1 is the transport and the first parts of the reliability layer.** The root entry -exports the message model and the transport contract of upyo, with the delivery events of 1.1 -and, unreleased, the contracts of the reliability layer as types; each transport, decorator and -helper is a subpath, and their API is under semver from 1.0.0. The MongoDB outbox, the -crypto-shredding of the audit and suppression have no API yet; their direction is in "Roadmap" -below. +**Status: 1.2 is the transport and the reliability layer on it.** The root entry exports the +message model and the transport contract of upyo, with the delivery events of 1.1 and the +contracts of the reliability layer of 1.2 as types; each transport, decorator, driver and helper +is a subpath, and their API is under semver from 1.0.0. The crypto-shredding of the audit and +suppression have no API yet; their direction is in "Roadmap" below. It is maintained by Alexis Technologies and set up like the sibling libraries `@alexify/metaschema` and `@alexify/protoarray` (the scaffold follows protoarray's @@ -909,7 +909,7 @@ model of states and events. `tenantId` is in the context from the start. providers. - **A persistent queue:** the outbox in MongoDB (transactional outbox, no Redis needed) and a BullMQ driver (Valkey) for throughput and delays; priorities (auth codes before newsletters). - (Unreleased: `./queue`, `./bullmq` and `./mongodb`, the outbox as a `QueueSource` with its + (Done in 1.2: `./queue`, `./bullmq` and `./mongodb`, the outbox as a `QueueSource` with its audit sink and idempotency store, and the memory queue.) - **The audit log** of every state, with a keyed HMAC plus an encrypted block for the recipient (crypto-shredding), so the right to erasure does not fight an append-only log. 3–6 records per diff --git a/README.md b/README.md index fd641da..58f2caf 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,14 @@ render templates of the engine you use and know a permanent failure from a tempo audit of every state of a message, one model of delivery events and idempotency; next suppression. Zero runtime dependencies. -> **Status: 1.1 is the transport layer and the first parts of the reliability layer.** 1.0 ported +> **Status: 1.2 is the transport layer and the reliability layer on it.** 1.0 ported > the transport from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds what > [nodemailer](https://github.com/nodemailer/nodemailer), > [unemail](https://github.com/productdevbook/unemail) and > [sently](https://github.com/omqkhafi/sently) did better: sendmail, ten more HTTP providers, > attachments from paths and URLs, one-click List-Unsubscribe, verified webhooks into one model of > delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and -> mirroring. Unreleased: the queue (`./queue`, with a BullMQ driver in `./bullmq` and a +> mirroring. 1.2 adds the queue (`./queue`, with a BullMQ driver in `./bullmq` and a > transactional outbox in `./mongodb`), templates by name over nine engines (`./templates`) and > the audit of the states of a message (`./audit`). Suppression is next. diff --git a/SECURITY.md b/SECURITY.md index 77cfa7a..cefc62b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,9 +28,9 @@ timeline once the report is confirmed. ## Threat model -The transport layer, the delivery events and their webhooks are built; the queue and the audit -log are being designed, and this section will be completed with them. The principles all of it is -designed around: +The transport layer, the delivery events and their webhooks, the queue with its drivers, +templates and the audit of states are built; suppression and the crypto-shredding of the audit +are next. The principles all of it is designed around: - **Message content is untrusted data.** Addresses, display names, subjects, headers, file names and bodies can come from users. A CR or LF in any header field is rejected rather than encoded @@ -70,6 +70,21 @@ designed around: match, compared in constant time; within a window of time where a time is signed, and once by its id. An SNS message (SES, Scaleway) is taken only from a listed topic, signed with the certificate of an SNS host or of the trust chain the application pins. +- **A job of the queue is untrusted when it is read.** Whoever can write to its store (Redis, + MongoDB) could have written it, so the worker checks its shape and its hash before it acts on + it, and `checkJob` can refuse a sender. A job names no file or URL: an attachment is its bytes + or a reference that the application's `resolveAttachment` reads. A job's data is stored as it is, + a token of a reset link among it: the store is the application's to restrict (Redis ACLs and + TLS, MongoDB roles). +- **Templates are named, never pathed.** A template, include, partial or layout is read only + through the loader of the renderer, by a name with no `..` or leading `/`; the file loader + compares real paths, so a symbolic link cannot lead out of its directory. No engine reads a file + itself, MJML's `mj-include` is off unless asked for, output is escaped by default (Liquid + included), and the CSS inliner fetches no remote stylesheet. Templates are code: EJS, Eta, Pug + and Nunjucks run JavaScript, so templates that tenants write belong to Liquid, Handlebars or + Mustache. +- **The audit holds no recipient.** An event names its message by `correlationId`, with the + hashes of its job and message; the text of an error in it has its addresses redacted. - **The audit log can forget a person** (planned). Recipients are stored as a keyed hash and an encrypted block whose key can be destroyed, so a request to erase someone's data does not require editing an append-only log. diff --git a/docs/comparison.json b/docs/comparison.json index 1ef3b4f..32f2f55 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -495,13 +495,13 @@ "text": "MIT" }, "version": { - "text": "1.1.0" + "text": "1.2.0" }, "lastCommit": { "text": "2026-10 (this release)" }, "tests": { - "text": "167 / ~2340" + "text": "177 / ~2560" }, "maintainer": { "text": "1 author, Alexis Technologies" diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index 9846564..0344175 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -114,9 +114,9 @@ it. - + - +
Rate limiting✅ token bucket per key, FIFO, shared store✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry
Circuit breaker✅ one probe in half-open❌❌❌✅ half-open without a single probe⚠️ cooldownMs❌❌—❌❌❌❌❌❌
Timeout / AbortSignal✅ signal + timeout over the body⚠️ timeouts, no AbortSignal⚠️ timeout, no signal✅ signal + timeout⚠️ signal for HTTP, not SMTP⚠️ SMTP connect only⚠️ socket timeouts✅ signal on connect and send—⚠️ signal, no timeout⚠️ global signal only⚠️ timeout, no signal⚠️ 180 s timeout⚠️ hard 10 s✅ abortSignal + requestTimeout
Persistent queue (survives a restart)✅ BullMQ driver, memory queue, any outbox as a QueueSource3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌
Persistent queue (survives a restart)✅ MongoDB outbox (transactional), BullMQ driver, memory queue3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌
Messages
Attachments✅ path, URL, data URI, stream, bytes, Blob4✅ path, URL, Buffer, stream, data-URI⚠️ path, stream, no URL⚠️ bytes, Blob, File, factory⚠️ bytes, string, URL⚠️ bytes + path⚠️ base64 only⚠️ string, bytes, Blob⚠️ base64 only⚠️ base64 or URL⚠️ base64, helper for Buffer✅ Buffer, Blob, stream, 50 MB✅ base64✅ string / Buffer✅ Uint8Array or Raw MIME
Streaming large attachments⚠️ SMTP, sendmail; HTTP providers buffer⚠️ SMTP yes; DKIM and SES buffer✅⚠️ SMTP only❌❌❌❌❌❌❌⚠️ accepts a stream❌❌❌
OpenTelemetry✅ decorator: spans + metrics❌❌✅ decorator: spans + metrics⚠️ withLogger⚠️ hooks + consoleObserver❌❌❌❌❌❌❌❌⚠️ middlewareStack
Hooks / middleware✅ ./hooks, decorators, a pino-like logger✅ use(compile|stream)⚠️ logger function⚠️ decorators, LogTape✅ Middleware pipeline✅ hooks + plugins⚠️ logger✅ onEvent⚠️ dump()⚠️ console.error⚠️ setClient❌❌❌✅ logger + middlewareStack
Mock / test mode✅ ./mock✅ jsonTransport, streamTransport, Ethereal⚠️ Message#read()✅ @upyo/mock✅ mock + Mailpit⚠️ preview, Mailpit❌❌—❌✅ sandboxMode✅ testmode⚠️ Sandbox server✅ Sandbox API⚠️ TestRenderEmailTemplate
Local audit log / event journal⚠️ state model + sinks; crypto-shredding next10❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌
Local audit log / event journal⚠️ state model + sinks, MongoDB among them; crypto-shredding next10❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌
Quality, license, maturity
TypeScript types✅ hand-written, tsd✅ built in✅✅✅✅✅✅✅✅✅ hand-written .d.ts✅✅✅✅ generated
Runtime dependencies0000–1 (@noble/hashes)0000422+ (axios)3 (axios)02 (axios, qs)9
Runtime dependencies0000–1 (@noble/hashes)0000422+ (axios)3 (axios)02 (axios, qs)9
Module formatCJS, ESM importESM + CJSESM onlyESM + CJS + JSRESM onlyESM onlyESM + CJSESM + CJS + JSRESM + CJSESM + CJSCJS onlyESM + CJSCJS onlyCJS onlyESM + CJS
LicenseMITMIT-0MITMITMITMITMITMITMITMITMITMITMITMITApache-2.0
Version (2026-10-07)1.1.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0
Version (2026-10-07)1.2.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0
Last commit2026-10 (this release)2026-10-072026-08-262026-09-092026-09-022026-08-192025-11-152026-09-082026-01-282026-10-062025-09-192026-10-072026-07-102026-10-072026-10-06
Tests: files / cases167 / ~234061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo)
Tests: files / cases177 / ~256061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo)
Owner / maintenance risk1 author, Alexis Technologies1 maintainerno SECURITY.md1 main author1 author1 author, 0.1 → 1.2 in ~80 days1 author1 author, 0.xno SECURITY.mdvendorvendor, old toolchainvendorvendorvendorvendor
diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9c62fa9..bc886de 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -4,8 +4,8 @@ Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds sendmail, ten more HTTP providers, verified webhooks into delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and -mirroring. Unreleased: [the queue](/guide/queue), [templates by name](/messages/templates) -and [the audit](/guide/audit). +mirroring; 1.2 adds [the queue](/guide/queue) on BullMQ or a MongoDB outbox, +[templates by name](/messages/templates) and [the audit](/guide/audit). ::: ## Installation diff --git a/docs/guide/why.md b/docs/guide/why.md index 1e8769f..e7e2941 100644 --- a/docs/guide/why.md +++ b/docs/guide/why.md @@ -3,9 +3,9 @@ ::: tip Status Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0, and 1.1 adds what nodemailer, unemail and sently did better, delivery events and the decorators of a -closed loop. Unreleased on `main`: the [queue](/guide/queue) with its BullMQ driver, -[templates by name](/messages/templates) and the [audit](/guide/audit) of the states of a message; -the MongoDB outbox and suppression are next. +closed loop; 1.2 adds the [queue](/guide/queue), on BullMQ or a +[MongoDB outbox](/guide/outbox), [templates by name](/messages/templates) and the +[audit](/guide/audit) of the states of a message. Suppression is next. ::: ## Two parts @@ -20,8 +20,8 @@ of two parts: signing, Amazon SES v2 (signed with SigV4, without the AWS SDK) and seventeen more HTTP providers on `fetch`, JMAP, and decorators for failover, retries, telemetry and logs. 2. **A reliability layer on top of it,** in part here. One model of delivery events read from - verified webhooks, idempotency keys, a circuit breaker and rate limits came in 1.1; the queue, - templates and the audit of states are unreleased on `main`; an outbox in MongoDB, the + verified webhooks, idempotency keys, a circuit breaker and rate limits came in 1.1; the queue + with its BullMQ driver and MongoDB outbox, templates and the audit of states in 1.2; the crypto-shredding of the audit and suppression are next. ## The gap it fills diff --git a/docs/index.md b/docs/index.md index ce36f14..d4ac91b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,7 +60,7 @@ The transport layer is here, ported from [upyo](https://github.com/dahlia/upyo) what [nodemailer](https://github.com/nodemailer/nodemailer), [unemail](https://github.com/productdevbook/unemail) and [sently](https://github.com/omqkhafi/sently) did better ported in 1.1: one package, with a subpath -per transport and decorator. Delivery events are here too, and, unreleased, the +per transport and decorator. Delivery events are here too, and, from 1.2, the [queue](/guide/queue), [templates](/messages/templates) and the [audit](/guide/audit). See [Why @alexify/mailer?](/guide/why) for the plan. ::: diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 207d89b..8e3d563 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -3,15 +3,14 @@ > Zero-dependency email for Node.js, Bun and Deno, maintained by Alexis Technologies. Its own SMTP > client, sendmail, MIME builder, DKIM signing, eighteen HTTP providers and JMAP, with a > reliability layer on top that none of the common Node.js email libraries has: one model of -> delivery events from verified webhooks and idempotency; unreleased, a persistent queue with a -> BullMQ driver, templates over nine engines and an audit of message states; an outbox in MongoDB -> and suppression next. +> delivery events from verified webhooks, idempotency, a persistent queue on BullMQ or a +> transactional MongoDB outbox, templates over nine engines and an audit of message states; +> suppression next. Key facts: - Status: 1.0 is the transport layer, ported from upyo 0.7.0 (MIT); 1.1 ports what nodemailer, - unemail and sently did better and adds delivery events with verified webhooks; the queue - (`/queue`, `/bullmq`, `/mongodb`), templates (`/templates`) and the audit (`/audit`) are - unreleased + unemail and sently did better and adds delivery events with verified webhooks; 1.2 adds the + queue (`/queue`, `/bullmq`, `/mongodb`), templates (`/templates`) and the audit (`/audit`) - Package (npm): `@alexify/mailer`, installed with `npm install @alexify/mailer` - Node.js >= 22, tested on Bun and Deno too; CommonJS with ESM named-import interop; hand-written d.ts per entry diff --git a/package.json b/package.json index 1491f10..199fe48 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alexify/mailer", - "version": "1.1.0", + "version": "1.2.0", "description": "Zero-dependency email for Node.js, Bun and Deno: its own SMTP client, sendmail, MIME builder and DKIM, eighteen HTTP providers and JMAP, a persistent queue with a BullMQ driver, templates over nine engines, an audit of message states, verified webhooks into one model of delivery events, and decorators for failover, retries, circuit breaking, rate limits, idempotency and OpenTelemetry", "license": "MIT", "type": "commonjs", diff --git a/src/version.js b/src/version.js index f3e9eba..e6bda00 100644 --- a/src/version.js +++ b/src/version.js @@ -3,4 +3,4 @@ // require of package.json would bundle the whole manifest into ./otel. // tests/unit/version.test.js keeps it equal to package.json, and the release // process bumps both. -module.exports = { name: '@alexify/mailer', version: '1.1.0' }; +module.exports = { name: '@alexify/mailer', version: '1.2.0' }; From 5f75515a6ab23a244929e8b4be4263e6b9050485 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:38:11 +0300 Subject: [PATCH 05/83] feat(queue): fix the fields of a version 1 job The hash of a job covers every field but enqueuedAt and itself, so a field added to version 1 changed the hash an older worker recomputed, and during a rolling deploy that worker failed the job for good (queue.hash_mismatch). The fields of version 1 are now a fixed set: a field beyond it makes the job wait for a worker that reads it (queue.unsupported_version). mac, kid and trace are reserved outside the hash for the signature and the trace context. The set of fields takes ./queue from 14.4 to 14.6 KB min+gzip: its budget goes from 14.5 to 15 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- docs/guide/queue.md | 7 +++++ scripts/size-budgets.json | 2 +- src/queue/index.d.ts | 12 ++++++-- src/queue/job.js | 47 ++++++++++++++++++++++++++++-- src/queue/parse.js | 16 +++++++++-- tests/unit/queue-job.test.js | 55 ++++++++++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e210e6..0e64ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. minutes, and never sends again a message of unknown delivery. `scheduledAt` is a delay of the queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and - `createMemoryQueue()` is one in memory for tests. + `createMemoryQueue()` is one in memory for tests. The fields of a job of version 1 are fixed: + a later version, or a field this one does not know, makes the job wait for a worker that reads + it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 4d6212d..58cd3c0 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -80,6 +80,13 @@ not the worker: same message; - a job is 256 KiB at most. +A job is of version 1 (`v: 1`), and its fields are those of that version. A +worker that meets a later version, or a field it does not know, gives the job +back for a worker that reads it (`queue.unsupported_version`) instead of +failing it for a hash it cannot recompute: during a rolling deploy, the jobs +the new producers write wait for the new workers. A field is never added to +version 1; a new field is a new version. + ## Once per key The id of a job is a UUID of its namespace, tenant and `idempotencyKey`: the diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index e6e5d89..23332ef 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 14.5, + "./queue": 15, "./bullmq": 3.5, "./mongodb": 5.5 } diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 58d3870..cb9ea52 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -144,8 +144,14 @@ interface MailJobDataBase { readonly notBefore?: string; readonly expiresAt?: string; readonly enqueuedAt: string; - /** The SHA-256 of the job, all but `enqueuedAt` and itself. */ + /** The SHA-256 of the job, all but `enqueuedAt`, `mac`, `kid`, `trace` and itself. */ readonly hash: string; + /** The HMAC-SHA256 of the job under the key `kid`, when the queue signs its jobs. */ + readonly mac?: string; + /** The id of the signing key of `mac`. */ + readonly kid?: string; + /** The W3C `traceparent` of the producer, when the queue is given one. */ + readonly trace?: string; } /** The data of a template job, as stored. */ @@ -201,7 +207,9 @@ export declare function prepareMailJob( /** * Checks the data of a job read from its queue. * - * @throws {MailQueueError} `queue.invalid_job`, `queue.unsupported_version` or `queue.hash_mismatch`. + * @throws {MailQueueError} `queue.invalid_job`; `queue.unsupported_version` + * for a later version, or a field this version does not know; or + * `queue.hash_mismatch`. */ export declare function parseMailJob(data: unknown): MailJobData; diff --git a/src/queue/job.js b/src/queue/job.js index fba1f79..1c20727 100644 --- a/src/queue/job.js +++ b/src/queue/job.js @@ -18,8 +18,49 @@ const { toJson } = require('./json.js'); const MAIL_JOB_VERSION = 1; const MAIL_JOB_NAME = 'mail'; const KINDS = new Set(['template', 'content']); -// What the hash of a job leaves out: when it was enqueued, and itself. -const UNHASHED = new Set(['enqueuedAt', 'hash']); +// What the hash of a job leaves out: when it was enqueued, itself, and what +// is written over it once it is made: the HMAC and its key id, and the trace +// context of the producer. +const UNHASHED = new Set(['enqueuedAt', 'hash', 'mac', 'kid', 'trace']); +// The fields of a job of version 1. A field beyond them was written by a +// later version, whose worker reads it: a job here does not fail its hash for +// it, it waits (see parse.js). A field more is a new version. +const V1_KEYS = new Set([ + 'v', + 'kind', + 'template', + 'context', + 'locale', + 'subject', + 'content', + 'from', + 'to', + 'cc', + 'bcc', + 'replyTo', + 'attachments', + 'priority', + 'tags', + 'headers', + 'list', + 'messageId', + 'date', + 'inReplyTo', + 'references', + 'calendar', + 'tracking', + 'metadata', + 'idempotencyKey', + 'tenantId', + 'correlationId', + 'notBefore', + 'expiresAt', + 'enqueuedAt', + 'hash', + 'mac', + 'kid', + 'trace', +]); const MAX_KEY_LENGTH = 256; const DEFAULTS = Object.freeze({ namespace: 'mail', @@ -272,4 +313,4 @@ const prepareMailJob = (input, options = {}) => { return { data, request }; }; -module.exports = { MAIL_JOB_NAME, MAIL_JOB_VERSION, UNHASHED, prepareMailJob }; +module.exports = { MAIL_JOB_NAME, MAIL_JOB_VERSION, UNHASHED, V1_KEYS, prepareMailJob }; diff --git a/src/queue/parse.js b/src/queue/parse.js index 9adcebb..693f2f3 100644 --- a/src/queue/parse.js +++ b/src/queue/parse.js @@ -4,7 +4,7 @@ const { MailQueueError } = require('./errors.js'); const { hashJson } = require('./hash.js'); -const { MAIL_JOB_VERSION, UNHASHED } = require('./job.js'); +const { MAIL_JOB_VERSION, UNHASHED, V1_KEYS } = require('./job.js'); const invalid = (message) => new MailQueueError('queue.invalid_job', `Invalid job: ${message}`); @@ -69,7 +69,8 @@ const checkAttachments = (attachments) => { * @param data The data of the job. * @returns The same data, checked. * @throws {MailQueueError} `queue.invalid_job` if it is not the data of a - * mail job, `queue.unsupported_version` if a later version wrote it, and + * mail job, `queue.unsupported_version` if a later version wrote it (its + * version, or a field this one does not know), and * `queue.hash_mismatch` if it is not what was enqueued. */ const parseMailJob = (data) => { @@ -83,6 +84,16 @@ const parseMailJob = (data) => { } throw invalid(`its version is ${JSON.stringify(data.v)}.`); } + // A field this version does not know was written by a later one: the job + // waits for its worker, rather than failing here for a hash of other fields. + for (const key of Object.keys(data)) { + if (!V1_KEYS.has(key)) { + throw new MailQueueError( + 'queue.unsupported_version', + `The job holds the field ${JSON.stringify(key)}, which this worker does not read.`, + ); + } + } checkBody(data); checkAddress(data.from, 'from'); checkAddresses(data.to, 'to', true); @@ -92,6 +103,7 @@ const parseMailJob = (data) => { checkAttachments(data.attachments); checkStrings(data, ['idempotencyKey', 'messageId', 'correlationId', 'hash', 'enqueuedAt']); checkStrings(data, ['subject', 'locale', 'tenantId', 'date', 'notBefore', 'expiresAt']); + checkStrings(data, ['mac', 'kid', 'trace']); for (const name of ['messageId', 'correlationId', 'hash', 'enqueuedAt']) { if (data[name] === undefined) throw invalid(`it has no ${name}.`); } diff --git a/tests/unit/queue-job.test.js b/tests/unit/queue-job.test.js index 4bd68a0..b0d0640 100644 --- a/tests/unit/queue-job.test.js +++ b/tests/unit/queue-job.test.js @@ -350,3 +350,58 @@ test('Queue: a job read from its store is checked, its hash among it', () => { assert.strictEqual(parseMailJob(content), content); assert.throws(() => parseMailJob({ ...content, content: null }), /no subject or content/); }); + +test('Queue: a job holds only the fields of its version', () => { + const { V1_KEYS } = require('../../src/queue/job.js'); + const template = prepare({ + idempotencyKey: 'all', + cc: 'c@example.com', + bcc: 'd@example.com', + replyTo: 'e@example.com', + subject: 'Hi', + attachments: [{ filename: 'a.txt', content: new Uint8Array([1]) }], + priority: 'low', + tags: ['t'], + headers: { 'X-A': 'b' }, + list: { unsubscribe: { url: 'https://example.com/u' } }, + messageId: '', + date: new Date(NOW), + inReplyTo: '', + references: '', + calendar: { content: 'BEGIN:VCALENDAR\r\nMETHOD:REQUEST\r\nEND:VCALENDAR' }, + tracking: { opens: false }, + metadata: { a: 'b' }, + tenantId: 'acme', + locale: 'uk', + scheduledAt: new Date(NOW + 1000), + expiresAt: new Date(NOW + 2000), + }).data; + const content = prepareMailJob({ + kind: 'content', + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { html: '

h

', text: 't' }, + }).data; + for (const data of [template, content]) { + for (const key of Object.keys(data)) assert.ok(V1_KEYS.has(key), key); + } +}); + +test('Queue: a field a later version adds makes the job wait, not fail its hash', () => { + const { data } = prepare({ idempotencyKey: 'later' }); + assert.throws( + () => parseMailJob({ ...data, priorityClass: 'bulk' }), + (error) => + error instanceof MailQueueError && + error.code === 'queue.unsupported_version' && + /priorityClass/.test(error.message), + ); + // What is written over a job once it is made does not change its hash. + const traced = { ...data, trace: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' }; + assert.strictEqual(parseMailJob(traced), traced); + assert.throws( + () => parseMailJob({ ...data, trace: 1 }), + (error) => error.code === 'queue.invalid_job', + ); +}); From 6b63d0e54dca24c0d5856f95325fcabb36628b61 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:42:01 +0300 Subject: [PATCH 06/83] feat(queue): sign jobs with HMAC keys A job carried a SHA-256 of its fields, which tells a job that changed in its store but not one that was forged: whoever can write to Redis or MongoDB can recompute it, and the worker sent their message from the application's domain. signingKeys, on the queue and its workers (and on prepareMailJob and parseMailJob), sign each job with an HMAC-SHA256 of the same canonical form (kid, mac, outside the hash) and refuse, unsent, a job none of them signed (queue.signature_invalid), compared in constant time. The first key signs and each verifies, for rotation. The keyring takes ./queue from 14.6 to 15.3 KB min+gzip: its budget goes from 15 to 15.5 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + SECURITY.md | 5 +- docs/guide/queue.md | 22 +++++ scripts/size-budgets.json | 2 +- src/queue/errors.js | 3 +- src/queue/handler.js | 9 +- src/queue/hash.js | 66 +++++++++------ src/queue/index.d.ts | 30 ++++++- src/queue/job.js | 9 +- src/queue/mail-queue.js | 15 +++- src/queue/parse.js | 12 ++- src/queue/signing.js | 108 ++++++++++++++++++++++++ tests/types/queue.test-d.ts | 26 +++++- tests/unit/queue-security.test.js | 135 ++++++++++++++++++++++++++++++ 14 files changed, 404 insertions(+), 42 deletions(-) create mode 100644 src/queue/signing.js create mode 100644 tests/unit/queue-security.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e64ca3..ea0f781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `createMemoryQueue()` is one in memory for tests. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. + `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends + no job they did not sign (`queue.signature_invalid`): the SHA-256 of a job tells a change, but + whoever writes to the store can recompute it. The first key signs and each verifies, for + rotation. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases diff --git a/SECURITY.md b/SECURITY.md index cefc62b..a75be01 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -72,7 +72,10 @@ are next. The principles all of it is designed around: certificate of an SNS host or of the trust chain the application pins. - **A job of the queue is untrusted when it is read.** Whoever can write to its store (Redis, MongoDB) could have written it, so the worker checks its shape and its hash before it acts on - it, and `checkJob` can refuse a sender. A job names no file or URL: an attachment is its bytes + it, and `checkJob` can refuse a sender. The hash, a SHA-256, tells a job that changed but not + one that was forged, since a writer can recompute it: with `signingKeys`, each job carries an + HMAC-SHA256 under the application's key, and the worker sends no job that one of its keys did + not sign. A job names no file or URL: an attachment is its bytes or a reference that the application's `resolveAttachment` reads. A job's data is stored as it is, a token of a reset link among it: the store is the application's to restrict (Redis ACLs and TLS, MongoDB roles). diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 58cd3c0..b912e18 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -87,6 +87,28 @@ failing it for a hash it cannot recompute: during a rolling deploy, the jobs the new producers write wait for the new workers. A field is never added to version 1; a new field is a new version. +## Signed jobs + +The hash of a job tells a job that changed in its store, but not one that was +forged: whoever can write to Redis or MongoDB can recompute a SHA-256, and a +worker would send their message from your domain, signed with your DKIM key. +Give the queue `signingKeys`, and each job is signed with an HMAC-SHA256 (`kid` +and `mac`); its workers send only jobs that one of the keys signed, and fail +any other with `queue.signature_invalid`, unsent, logged at `error`: + +```js +const queue = createMailQueue({ + driver, + signingKeys: [{ id: '2026-10', secret: process.env.MAIL_JOB_KEY }], // 32 bytes at least +}); +const worker = queue.createWorker({ transport }); // verifies with the queue's keys +``` + +The first key signs and each one verifies. To rotate, put the new key first, +deploy the workers, then the producers, and drop the old key once no job it +signed is left. A worker without keys checks the hash alone, so give every +worker the keys before any producer. + ## Once per key The id of a job is a UUID of its namespace, tenant and `idempotencyKey`: the diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 23332ef..f39418f 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 15, + "./queue": 15.5, "./bullmq": 3.5, "./mongodb": 5.5 } diff --git a/src/queue/errors.js b/src/queue/errors.js index a77ffcd..d824969 100644 --- a/src/queue/errors.js +++ b/src/queue/errors.js @@ -3,7 +3,8 @@ /** * A failure of the queue: `queue.key_reused`, `queue.invalid_job`, - * `queue.unsupported_version` or `queue.hash_mismatch`. + * `queue.unsupported_version`, `queue.hash_mismatch` or + * `queue.signature_invalid`. */ class MailQueueError extends Error { /** diff --git a/src/queue/handler.js b/src/queue/handler.js index 99a5453..7ef18b8 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -20,6 +20,7 @@ const { isProgrammerError, receiptOfThrown } = require('../internal/thrown.js'); const { toUuid } = require('../internal/uuid.js'); const { outcomeOfFailure, reasonOf } = require('./outcome.js'); const { parseMailJob } = require('./parse.js'); +const { keyringOf } = require('./signing.js'); const DEFAULT_BACKOFF = Object.freeze({ baseDelayMilliseconds: 30_000, @@ -61,11 +62,14 @@ const createHandlerConfig = (options = {}) => { if (options.auditStrict !== undefined && typeof options.auditStrict !== 'boolean') { throw new TypeError('auditStrict must be a boolean.'); } + // Checked now, so that a bad key fails the worker, not each job. + keyringOf(options.signingKeys); return { transport, renderer, audit, auditStrict: options.auditStrict ?? false, + signingKeys: options.signingKeys, resolveAttachment: checkOptionalFunction(options.resolveAttachment, 'resolveAttachment'), checkJob: checkOptionalFunction(options.checkJob, 'checkJob'), classify: checkOptionalFunction(options.classify, 'classify'), @@ -412,7 +416,7 @@ const attempt = async (run) => { const readJob = (config, job) => { try { - return { data: parseMailJob(job.data) }; + return { data: parseMailJob(job.data, { signingKeys: config.signingKeys }) }; } catch (error) { const code = error?.code ?? 'queue.invalid_job'; const later = code === 'queue.unsupported_version' && job.attempt < job.maxAttempts; @@ -440,7 +444,8 @@ const readJob = (config, job) => { * Makes the handler of mail jobs, for any queue driver. * * @param options `transport`; `renderer`, for template jobs; `audit`, - * `auditStrict`; `resolveAttachment(ref, { job, attachment, signal })`; + * `auditStrict`; `signingKeys`, when only signed jobs are sent; + * `resolveAttachment(ref, { job, attachment, signal })`; * `checkJob(data, job)`; `classify({ receipt, job })`; `backoff`, * `operatorDelayMilliseconds` (5 minutes), `rateLimitScope` ('job'); * `logger`, `random`, `now`. diff --git a/src/queue/hash.js b/src/queue/hash.js index 8878247..2a9bc1c 100644 --- a/src/queue/hash.js +++ b/src/queue/hash.js @@ -1,53 +1,67 @@ // Not ported: the SHA-256 of a value of JSON, its keys in order, so that the -// same job hashes the same whatever order its fields were written in. +// same job hashes the same whatever order its fields were written in; and the +// HMAC of the same form, which only a holder of the key can make. -const { createHash } = require('node:crypto'); +const { createHash, createHmac } = require('node:crypto'); -const update = (hash, value) => { +const update = (digest, value) => { if (value === null || typeof value !== 'object') { - hash.update(JSON.stringify(value)); + digest.update(JSON.stringify(value)); return; } if (Array.isArray(value)) { - hash.update('['); + digest.update('['); for (let index = 0; index < value.length; index++) { - if (index > 0) hash.update(','); - update(hash, value[index]); + if (index > 0) digest.update(','); + update(digest, value[index]); } - hash.update(']'); + digest.update(']'); return; } - hash.update('{'); + digest.update('{'); let first = true; for (const key of Object.keys(value).toSorted()) { if (value[key] === undefined) continue; - hash.update(`${first ? '' : ','}${JSON.stringify(key)}:`); + digest.update(`${first ? '' : ','}${JSON.stringify(key)}:`); first = false; - update(hash, value[key]); + update(digest, value[key]); } - hash.update('}'); + digest.update('}'); }; -/** - * The SHA-256 of a value of JSON in its canonical form, its keys sorted. - * - * @param value The value. - * @param leaveOut Keys of the top object left out, such as the hash itself. - * @returns The hash, in hexadecimal. - * @internal - */ -const hashJson = (value, leaveOut) => { - const hash = createHash('sha256'); +const digestJson = (digest, value, leaveOut) => { if (leaveOut === undefined) { - update(hash, value); + update(digest, value); } else { const kept = {}; for (const key of Object.keys(value)) { if (!leaveOut.has(key)) kept[key] = value[key]; } - update(hash, kept); + update(digest, kept); } - return hash.digest('hex'); + return digest.digest('hex'); }; -module.exports = { hashJson }; +/** + * The SHA-256 of a value of JSON in its canonical form, its keys sorted. + * + * @param value The value. + * @param leaveOut Keys of the top object left out, such as the hash itself. + * @returns The hash, in hexadecimal. + * @internal + */ +const hashJson = (value, leaveOut) => digestJson(createHash('sha256'), value, leaveOut); + +/** + * The HMAC-SHA256 of a value of JSON in the canonical form of `hashJson`. + * + * @param secret The key. + * @param value The value. + * @param leaveOut Keys of the top object left out. + * @returns The HMAC, in hexadecimal. + * @internal + */ +const macJson = (secret, value, leaveOut) => + digestJson(createHmac('sha256', secret), value, leaveOut); + +module.exports = { hashJson, macJson }; diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index cb9ea52..7110493 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -172,6 +172,17 @@ export interface MailContentJobData extends MailJobDataBase { /** The data of a mail job, as stored: JSON only. */ export type MailJobData = MailTemplateJobData | MailContentJobData; +/** + * A key that signs jobs: an HMAC-SHA256 key of 32 bytes at least. Of the keys + * given, the first signs and each verifies, so a key is rotated by putting + * the new one first and dropping the old one once its jobs are done. + */ +export interface MailJobSigningKey { + /** 1 to 64 of A-Z, a-z, 0-9, `.`, `_` and `-`; written into each job it signs as `kid`. */ + readonly id: string; + readonly secret: string | Uint8Array; +} + /** The options of the jobs of a queue. */ export interface MailJobOptions { /** The scope of the keys of the jobs. @default 'mail' */ @@ -184,6 +195,8 @@ export interface MailJobOptions { readonly maxInlineAttachmentBytes?: number; readonly generateId?: IdGenerator; readonly now?: () => number; + /** Signs each job (`kid`, `mac`) with the first key, for workers that read only signed jobs. */ + readonly signingKeys?: readonly MailJobSigningKey[]; } /** A job of a message: its data, and the request a driver takes. */ @@ -209,16 +222,22 @@ export declare function prepareMailJob( * * @throws {MailQueueError} `queue.invalid_job`; `queue.unsupported_version` * for a later version, or a field this version does not know; or - * `queue.hash_mismatch`. + * `queue.hash_mismatch`; `queue.signature_invalid` if `signingKeys` are given + * and the job is not signed by one of them. + * @throws {TypeError} If a signing key is invalid. */ -export declare function parseMailJob(data: unknown): MailJobData; +export declare function parseMailJob( + data: unknown, + options?: { readonly signingKeys?: readonly MailJobSigningKey[] }, +): MailJobData; /** The code of a {@link MailQueueError}. */ export type MailQueueErrorCode = | 'queue.key_reused' | 'queue.invalid_job' | 'queue.unsupported_version' - | 'queue.hash_mismatch'; + | 'queue.hash_mismatch' + | 'queue.signature_invalid'; /** A failure of the queue, by its code. */ export declare class MailQueueError extends Error { @@ -275,6 +294,11 @@ export interface MailWorkerOptions { readonly audit?: AuditSink; /** A sink that fails before the send makes the job retried, rather than sent unrecorded. @default false */ readonly auditStrict?: boolean; + /** + * Only jobs signed by one of these keys are sent; any other fails with + * `queue.signature_invalid`. A worker of a queue takes the queue's. + */ + readonly signingKeys?: readonly MailJobSigningKey[]; readonly backoff?: MailBackoff; /** The wait before an attempt again after an `auth` or `configuration` failure. @default 300000 */ readonly operatorDelayMilliseconds?: number; diff --git a/src/queue/job.js b/src/queue/job.js index 1c20727..bc3f798 100644 --- a/src/queue/job.js +++ b/src/queue/job.js @@ -14,6 +14,7 @@ const { toBase64 } = require('../internal/base64.js'); const { toUuid } = require('../internal/uuid.js'); const { hashJson } = require('./hash.js'); const { toJson } = require('./json.js'); +const { keyringOf, signJob } = require('./signing.js'); const MAIL_JOB_VERSION = 1; const MAIL_JOB_NAME = 'mail'; @@ -204,6 +205,7 @@ const resolveConfig = (options) => { } } if (!isText(config.namespace, 128)) throw new TypeError('namespace must be a short string.'); + config.keyring = keyringOf(config.signingKeys); return config; }; @@ -239,9 +241,11 @@ const fieldsOf = (input, probe, attachments, messageId) => { * @param input `{ kind: 'template', template, context, locale }` or * `{ kind: 'content', subject, content }`, with the fields of a message. * @param options `namespace` ('mail'), `generateId`, `now`, `maxAttempts` - * (5), `maxJobBytes` (256 KiB), `maxInlineAttachmentBytes` (256 KiB). + * (5), `maxJobBytes` (256 KiB), `maxInlineAttachmentBytes` (256 KiB); + * `signingKeys`, whose first key signs the job (`kid`, `mac`). * @returns `{ data, request }`. - * @throws {TypeError} If a field is invalid, as `createMessage` finds it. + * @throws {TypeError} If a field is invalid, as `createMessage` finds it, or + * a signing key. * @throws {RangeError} If the job or its attachments are too large. */ const prepareMailJob = (input, options = {}) => { @@ -295,6 +299,7 @@ const prepareMailJob = (input, options = {}) => { if (expiresAt !== undefined) data.expiresAt = expiresAt.toISOString(); data.enqueuedAt = new Date(now).toISOString(); data.hash = hashJson(data, UNHASHED); + if (config.keyring !== undefined) signJob(data, config.keyring, UNHASHED); const bytes = Buffer.byteLength(JSON.stringify(data)); if (bytes > config.maxJobBytes) { throw new RangeError(`The job holds ${bytes} bytes, more than ${config.maxJobBytes}.`); diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 0e7556a..b72dd0a 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -8,6 +8,7 @@ const { toUuid } = require('../internal/uuid.js'); const { MailQueueError } = require('./errors.js'); const { createMailJobHandler } = require('./handler.js'); const { prepareMailJob } = require('./job.js'); +const { keyringOf } = require('./signing.js'); const checkDriver = (driver) => { if (typeof driver?.enqueue !== 'function' || typeof driver?.consume !== 'function') { @@ -74,7 +75,8 @@ const afterEnqueue = async (queue, { data, request }, result, enqueueOptions) => * * @param options `driver`; `audit` and `auditStrict`; `namespace` ('mail'), * the scope of the keys; `maxAttempts` (5), `maxJobBytes` and - * `maxInlineAttachmentBytes` (256 KiB); `logger`, `generateId`, `now`. + * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job + * and which its workers verify; `logger`, `generateId`, `now`. * @returns `{ driver, enqueue, enqueueMany, createWorker }`. * @throws {TypeError} If an option is invalid. */ @@ -86,6 +88,7 @@ const createMailQueue = (options = {}) => { } const auditStrict = options.auditStrict ?? false; if (typeof auditStrict !== 'boolean') throw new TypeError('auditStrict must be a boolean.'); + keyringOf(options.signingKeys); const now = options.now ?? Date.now; const jobOptions = { namespace: options.namespace, @@ -93,6 +96,7 @@ const createMailQueue = (options = {}) => { maxJobBytes: options.maxJobBytes, maxInlineAttachmentBytes: options.maxInlineAttachmentBytes, generateId: options.generateId, + signingKeys: options.signingKeys, now, }; for (const name of Object.keys(jobOptions)) { @@ -131,7 +135,14 @@ const createMailQueue = (options = {}) => { return answers; }, createWorker(workerOptions = {}) { - const handler = createMailJobHandler({ audit, auditStrict, logger, now, ...workerOptions }); + const handler = createMailJobHandler({ + audit, + auditStrict, + logger, + now, + signingKeys: options.signingKeys, + ...workerOptions, + }); const consumer = driver.consume(handler, { concurrency: workerOptions.concurrency, signal: workerOptions.signal, diff --git a/src/queue/parse.js b/src/queue/parse.js index 693f2f3..f7e655c 100644 --- a/src/queue/parse.js +++ b/src/queue/parse.js @@ -5,6 +5,7 @@ const { MailQueueError } = require('./errors.js'); const { hashJson } = require('./hash.js'); const { MAIL_JOB_VERSION, UNHASHED, V1_KEYS } = require('./job.js'); +const { keyringOf, verifyJob } = require('./signing.js'); const invalid = (message) => new MailQueueError('queue.invalid_job', `Invalid job: ${message}`); @@ -67,13 +68,17 @@ const checkAttachments = (attachments) => { * Checks the data of a job as read from its queue. * * @param data The data of the job. + * @param options `signingKeys`: a job must then be signed by one of them. * @returns The same data, checked. * @throws {MailQueueError} `queue.invalid_job` if it is not the data of a * mail job, `queue.unsupported_version` if a later version wrote it (its - * version, or a field this one does not know), and - * `queue.hash_mismatch` if it is not what was enqueued. + * version, or a field this one does not know), `queue.hash_mismatch` if it is + * not what was enqueued, and `queue.signature_invalid` if it is not signed by + * a key of `signingKeys`. + * @throws {TypeError} If a signing key is invalid. */ -const parseMailJob = (data) => { +const parseMailJob = (data, options = {}) => { + const keyring = keyringOf(options.signingKeys); if (!isObject(data)) throw invalid('its data is no object.'); if (data.v !== MAIL_JOB_VERSION) { if (Number.isInteger(data.v) && data.v > MAIL_JOB_VERSION) { @@ -114,6 +119,7 @@ const parseMailJob = (data) => { 'The job is not what was enqueued: its hash differs.', ); } + if (keyring !== undefined) verifyJob(data, keyring, UNHASHED); return data; }; diff --git a/src/queue/signing.js b/src/queue/signing.js new file mode 100644 index 0000000..7a1891a --- /dev/null +++ b/src/queue/signing.js @@ -0,0 +1,108 @@ +// Not ported: the signature of a job, an HMAC-SHA256 of its canonical form +// under a key of the application's. Its SHA-256 tells a job that changed, but +// whoever can write to the store of the queue can recompute it, and so make +// the worker send a message of their own from the application's domain; the +// HMAC only a holder of a key can make. + +const { Buffer } = require('node:buffer'); +const { timingSafeEqual } = require('node:crypto'); +const { MailQueueError } = require('./errors.js'); +const { macJson } = require('./hash.js'); + +const MIN_SECRET_BYTES = 32; +const KEY_ID = /^[A-Za-z0-9._-]{1,64}$/; +const MAC = /^[0-9a-f]{64}$/; + +// The keyring of each array of keys given, checked once. +const keyrings = new WeakMap(); + +const secretBytes = (secret) => + typeof secret === 'string' ? Buffer.byteLength(secret) : secret.byteLength; + +const checkKey = (key, keys) => { + const id = key?.id; + const secret = key?.secret; + if (typeof id !== 'string' || !KEY_ID.test(id)) { + throw new TypeError('The id of a signing key is 1 to 64 of A-Z, a-z, 0-9, ".", "_" and "-".'); + } + if ( + (typeof secret !== 'string' && !(secret instanceof Uint8Array)) || + secretBytes(secret) < MIN_SECRET_BYTES + ) { + throw new TypeError( + `The secret of the signing key ${id} must hold ${MIN_SECRET_BYTES} bytes at least.`, + ); + } + if (keys.has(id)) throw new TypeError(`The signing key ${id} is given twice.`); + keys.set(id, secret); +}; + +/** + * Checks the signing keys of a queue or a worker: the first signs, and each + * verifies, so that a key is rotated by putting the new one first and + * dropping the old one once no job it signed is left. + * + * @param signingKeys `[{ id, secret }]`, a secret of 32 bytes at least, or + * nothing. + * @returns The keyring, `{ signer, keys }`, or `undefined` for none. + * @throws {TypeError} If a key is invalid. + * @internal + */ +const keyringOf = (signingKeys) => { + if (signingKeys === undefined) return undefined; + if (!Array.isArray(signingKeys) || signingKeys.length === 0) { + throw new TypeError('signingKeys must be a non-empty array of { id, secret }.'); + } + const known = keyrings.get(signingKeys); + if (known !== undefined) return known; + const keys = new Map(); + for (const key of signingKeys) checkKey(key, keys); + const keyring = { signer: signingKeys[0].id, keys }; + keyrings.set(signingKeys, keyring); + return keyring; +}; + +/** + * Signs the data of a job with the first key: `kid` and `mac`. + * + * @param data The data, its hash made. + * @param keyring The keyring. + * @param leaveOut The fields the hash leaves out, `mac` and `kid` among them. + * @internal + */ +const signJob = (data, keyring, leaveOut) => { + data.kid = keyring.signer; + data.mac = macJson(keyring.keys.get(keyring.signer), data, leaveOut); +}; + +const invalid = (message) => new MailQueueError('queue.signature_invalid', message); + +/** + * Verifies the signature of the data of a job read from its store. + * + * @param data The data. + * @param keyring The keyring. + * @param leaveOut The fields the hash leaves out. + * @throws {MailQueueError} `queue.signature_invalid` if the job is not + * signed, by a key the keyring holds, or its signature does not verify. + * @internal + */ +const verifyJob = (data, keyring, leaveOut) => { + if (typeof data.mac !== 'string' || typeof data.kid !== 'string') { + throw invalid('The job is not signed, and this worker reads only signed jobs.'); + } + const secret = keyring.keys.get(data.kid); + if (secret === undefined) { + throw invalid( + `The job is signed with the key ${JSON.stringify(data.kid.slice(0, 64))}, ` + + 'which this worker does not hold.', + ); + } + if (!MAC.test(data.mac)) throw invalid('The signature of the job is not an HMAC-SHA256.'); + const expected = Buffer.from(macJson(secret, data, leaveOut), 'hex'); + if (!timingSafeEqual(expected, Buffer.from(data.mac, 'hex'))) { + throw invalid('The signature of the job does not verify: it is not what was enqueued.'); + } +}; + +module.exports = { keyringOf, signJob, verifyJob }; diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index a672a19..a028b5e 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -121,11 +121,35 @@ const prepared = prepareMailJob({ expectType(prepared.data); expectType(prepared.request.id); expectType(parseMailJob({})); +const keys = [ + { id: 'k1', secret: 'a'.repeat(32) }, + { id: 'k0', secret: new Uint8Array(32) }, +]; +expectType(parseMailJob({}, { signingKeys: keys })); +expectType( + prepareMailJob( + { + kind: 'content', + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 't' }, + }, + { signingKeys: keys }, + ).data.mac, +); +createMailQueue({ driver, signingKeys: keys }); +createMailJobHandler({ transport, signingKeys: keys }); +expectError(createMailJobHandler({ transport, signingKeys: [{ id: 'k', secret: 1 }] })); expectType<1>(MAIL_JOB_VERSION); expectType<'mail'>(MAIL_JOB_NAME); const error = new MailQueueError('queue.key_reused', 'x'); expectType< - 'queue.key_reused' | 'queue.invalid_job' | 'queue.unsupported_version' | 'queue.hash_mismatch' + | 'queue.key_reused' + | 'queue.invalid_job' + | 'queue.unsupported_version' + | 'queue.hash_mismatch' + | 'queue.signature_invalid' >(error.code); expectError(new MailQueueError('nope', 'x')); new QueueLeaseLostError('j'); diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js new file mode 100644 index 0000000..4bc505d --- /dev/null +++ b/tests/unit/queue-security.test.js @@ -0,0 +1,135 @@ +// Not ported: tests of what keeps a job of ./queue to what its producer +// enqueued, against whoever can write to the store of the queue. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { + MailQueueError, + createMailJobHandler, + createMailQueue, + createMemoryQueue, + parseMailJob, + prepareMailJob, +} = require('../../src/queue/index.js'); +const { hashJson } = require('../../src/queue/hash.js'); +const { UNHASHED } = require('../../src/queue/job.js'); +const { fakeTransport } = require('../helpers/decorators/fake.js'); +const { manualClock } = require('../helpers/queue/clock.js'); + +const NOW = Date.parse('2026-10-18T09:00:00Z'); +const KEY_A = { id: 'a', secret: 'a'.repeat(32) }; +const KEY_B = { id: 'b', secret: new Uint8Array(32).fill(7) }; +const input = { + kind: 'content', + subject: 'Your code', + content: { text: 'Your code is 123456' }, + from: 'App ', + to: 'ada@example.com', + idempotencyKey: 'code:1', +}; +const prepare = (options) => prepareMailJob(input, { now: () => NOW, ...options }); +const copyOf = (data) => JSON.parse(JSON.stringify(data)); +const isCode = (code) => (error) => error instanceof MailQueueError && error.code === code; + +// What a writer to the store can do: change the job and recompute its hash. +const forge = (data) => { + const forged = copyOf(data); + forged.to = [{ address: 'victim@example.org' }]; + forged.content = { text: 'Reset your password at https://evil.example' }; + forged.hash = hashJson(forged, UNHASHED); + return forged; +}; + +test('Queue: a signed job carries its key id and HMAC, outside its hash', () => { + const plain = prepare().data; + const signed = prepare({ signingKeys: [KEY_A] }).data; + assert.strictEqual(signed.kid, 'a'); + assert.match(signed.mac, /^[0-9a-f]{64}$/); + assert.strictEqual(signed.hash, plain.hash); + const copy = copyOf(signed); + assert.strictEqual(parseMailJob(copy, { signingKeys: [KEY_A] }), copy); +}); + +test('Queue: a job forged with its hash recomputed is refused under signing keys', () => { + const signed = prepare({ signingKeys: [KEY_A] }).data; + const forged = forge(signed); + // Its SHA-256 alone does not tell it: anyone who writes to the store can make it. + assert.strictEqual(parseMailJob(forged).to[0].address, 'victim@example.org'); + assert.throws( + () => parseMailJob(forged, { signingKeys: [KEY_A] }), + (error) => { + assert.ok(isCode('queue.signature_invalid')(error)); + assert.match(error.message, /does not verify/); + return true; + }, + ); + const unsigned = forge(prepare().data); + assert.throws( + () => parseMailJob(unsigned, { signingKeys: [KEY_A] }), + (error) => isCode('queue.signature_invalid')(error) && /not signed/.test(error.message), + ); + const badMac = { ...copyOf(signed), mac: 'z'.repeat(64) }; + assert.throws( + () => parseMailJob(badMac, { signingKeys: [KEY_A] }), + isCode('queue.signature_invalid'), + ); +}); + +test('Queue: signing keys rotate: the first signs, each verifies', () => { + const old = copyOf(prepare({ signingKeys: [KEY_A] }).data); + const rotated = copyOf(prepare({ signingKeys: [KEY_B, KEY_A] }).data); + assert.strictEqual(rotated.kid, 'b'); + for (const data of [old, rotated]) { + assert.strictEqual(parseMailJob(data, { signingKeys: [KEY_B, KEY_A] }), data); + } + // Once the old key is dropped, what it signed is refused. + assert.throws( + () => parseMailJob(old, { signingKeys: [KEY_B] }), + (error) => isCode('queue.signature_invalid')(error) && /"a"/.test(error.message), + ); +}); + +test('Queue: signing keys are checked', () => { + const refusals = [ + [[], /non-empty array/], + ['key', /non-empty array/], + [[{ id: 'a', secret: 'short' }], /32 bytes/], + [[{ id: 'a b', secret: KEY_A.secret }], /id of a signing key/], + [[{ id: 'a', secret: 42 }], /32 bytes/], + [[KEY_A, { ...KEY_A }], /given twice/], + ]; + for (const [signingKeys, pattern] of refusals) { + assert.throws(() => prepare({ signingKeys }), pattern); + assert.throws(() => createMailJobHandler({ transport: fakeTransport(), signingKeys }), pattern); + assert.throws(() => createMailQueue({ driver: createMemoryQueue(), signingKeys }), pattern); + } +}); + +test('Queue: a worker with signing keys sends a signed job and fails a forged one unsent', async () => { + const clock = manualClock(NOW); + const driver = createMemoryQueue({ now: clock.now, sleep: clock.sleep }); + const queue = createMailQueue({ driver, now: clock.now, signingKeys: [KEY_A] }); + const transport = fakeTransport(); + await queue.enqueue(input); + await using worker = queue.createWorker({ transport }); + await driver.drain(); + assert.strictEqual(transport.calls.length, 1); + assert.ok(worker); + + const records = []; + const logger = { error: (record, message) => records.push([message, record]) }; + const handler = createMailJobHandler({ transport, signingKeys: [KEY_A], logger, now: () => NOW }); + const signed = prepare({ signingKeys: [KEY_A] }); + const job = { + id: signed.request.id, + name: 'mail', + data: forge(signed.data), + attempt: 1, + maxAttempts: 5, + redelivered: false, + }; + const outcome = await handler(job, new AbortController().signal); + assert.deepStrictEqual([outcome.type, outcome.code], ['fail', 'queue.signature_invalid']); + assert.strictEqual(transport.calls.length, 1); + assert.deepStrictEqual(records[0][0], 'queue.job.failed'); + assert.strictEqual(records[0][1].code, 'queue.signature_invalid'); +}); From 7307d049a3dff5875ca9e7d0fe400895c6b4f779 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:45:25 +0300 Subject: [PATCH 07/83] feat(queue): seal the fields of a job with a codec A job held its message in clear in Redis or MongoDB, its addresses, its context (a link of a password reset among it) and its attachments, for as long as the job was kept and in every backup. A codec of the application's (encode, decode, such as AES-GCM under its key or KMS), on the queue and its workers, seals those fields as enc; the version, the hash, by which a duplicate is still told, the signature and the trace context stay in clear, and no other field may stand beside enc. A worker that cannot open a sealed job gives it back (queue.codec_missing, queue.codec_failed) unless the codec says its failure is for good (retryable: false). The checks of the queue's options move out of createMailQueue, which had grown past 80 lines. The envelope takes ./queue from 15.3 to 15.8 KB min+gzip: its budget goes from 15.5 to 16 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- SECURITY.md | 6 +- docs/guide/outbox.md | 3 +- docs/guide/queue.md | 36 ++++++++ scripts/size-budgets.json | 2 +- src/queue/envelope.js | 119 ++++++++++++++++++++++++ src/queue/errors.js | 4 +- src/queue/handler.js | 32 +++++-- src/queue/index.d.ts | 29 +++++- src/queue/mail-queue.js | 61 ++++++++----- src/queue/parse.js | 6 ++ tests/types/queue.test-d.ts | 6 ++ tests/unit/queue-security.test.js | 144 ++++++++++++++++++++++++++++++ 13 files changed, 416 insertions(+), 37 deletions(-) create mode 100644 src/queue/envelope.js diff --git a/CHANGELOG.md b/CHANGELOG.md index ea0f781..269f136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,10 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends no job they did not sign (`queue.signature_invalid`): the SHA-256 of a job tells a change, but whoever writes to the store can recompute it. The first key signs and each verifies, for - rotation. + rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the application's key) seals the + fields of each job in its store, its addresses, context and attachments, leaving the version, + hash, signature and trace context in clear; a worker that cannot open a sealed job gives it + back (`queue.codec_missing`, `queue.codec_failed`). - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases diff --git a/SECURITY.md b/SECURITY.md index a75be01..48f5aa7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -77,8 +77,10 @@ are next. The principles all of it is designed around: HMAC-SHA256 under the application's key, and the worker sends no job that one of its keys did not sign. A job names no file or URL: an attachment is its bytes or a reference that the application's `resolveAttachment` reads. A job's data is stored as it is, - a token of a reset link among it: the store is the application's to restrict (Redis ACLs and - TLS, MongoDB roles). + a token of a reset link among it, unless a `codec` of the application's seals it (an envelope + encryption: the store then holds the version, the hash and the signature in clear and the + rest encrypted); the store is the application's to restrict (Redis ACLs and TLS, MongoDB + roles). - **Templates are named, never pathed.** A template, include, partial or layout is read only through the loader of the renderer, by a name with no `..` or leading `/`; the file loader compares real paths, so a symbolic link cannot lead out of its directory. No engine reads a file diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 852db97..a9f53ff 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -106,7 +106,8 @@ deduplication. A key of the store goes when its time passes. ``` `outbox.get(id)` reads one. The data of a job is stored as it is, a token of a -reset link among it: restrict who reads the collection. +reset link among it, unless the queue seals it with a +[codec](/guide/queue#sealed-jobs): restrict who reads the collection. ## The audit and the keys diff --git a/docs/guide/queue.md b/docs/guide/queue.md index b912e18..b355293 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -109,6 +109,42 @@ deploy the workers, then the producers, and drop the old key once no job it signed is left. A worker without keys checks the hash alone, so give every worker the keys before any producer. +## Sealed jobs + +A job holds its message in clear: its addresses, its context (a link of a +password reset among it) and its attachments, in Redis or MongoDB for as long +as the job is kept, and in their backups. Give the queue a `codec`, and the +store holds what it made of them: + +```js +const { createCipheriv, createDecipheriv, randomBytes } = require('node:crypto'); +const key = Buffer.from(process.env.MAIL_DATA_KEY, 'base64'); // 32 bytes + +const codec = { + encode(text) { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + const sealed = Buffer.concat([cipher.update(text, 'utf8'), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), sealed]).toString('base64'); + }, + decode(text) { + const bytes = Buffer.from(text, 'base64'); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAuthTag(bytes.subarray(12, 28)); + return Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString(); + }, +}; +const queue = createMailQueue({ driver, codec, signingKeys }); +``` + +A sealed job keeps in clear only `v`, `hash` (a duplicate is still told by +it), `enqueuedAt`, `kid`, `mac` and `trace`; the rest is `enc`. Its workers +take the queue's codec. A worker without it, or whose codec throws (a key out +of reach), gives the job back for a worker that can open it +(`queue.codec_missing`, `queue.codec_failed`); a codec that throws an error +with `retryable: false` fails it. Encryption is not a signature: a writer who +can seal can still forge, so seal with `signingKeys` too. + ## Once per key The id of a job is a UUID of its namespace, tenant and `idempotencyKey`: the diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index f39418f..b257e5a 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 15.5, + "./queue": 16, "./bullmq": 3.5, "./mongodb": 5.5 } diff --git a/src/queue/envelope.js b/src/queue/envelope.js new file mode 100644 index 0000000..13e6c0f --- /dev/null +++ b/src/queue/envelope.js @@ -0,0 +1,119 @@ +// Not ported: the data of a job sealed by a codec of the application's, such +// as an envelope encryption under its key or KMS. The store then holds what +// the codec made of the fields of the message, its addresses, context and +// attachments, and in clear only what a driver and a dashboard read: the +// version, the hash by which a duplicate is told, the signature and the trace +// context. + +const { messageOf } = require('../internal/decorator.js'); +const { MailQueueError } = require('./errors.js'); + +// The fields a sealed job keeps in clear, beside `enc`. +const ENVELOPE = new Set(['v', 'hash', 'enqueuedAt', 'mac', 'kid', 'trace']); + +/** + * Checks a codec: `encode(text)` and `decode(text)`, sync or async, each + * giving a string. + * + * @param codec The codec, or nothing. + * @returns The codec. + * @throws {TypeError} If it is not one. + * @internal + */ +const checkCodec = (codec) => { + if (codec === undefined) return undefined; + if (typeof codec?.encode !== 'function' || typeof codec?.decode !== 'function') { + throw new TypeError('codec must have encode(text) and decode(text).'); + } + return codec; +}; + +/** + * Whether the data of a job is sealed. + * + * @internal + */ +const isSealed = (data) => + typeof data === 'object' && data !== null && typeof data.enc === 'string'; + +/** + * Seals the data of a job: the fields of its message as the codec encodes + * their JSON, the envelope in clear. + * + * @param data The data of the job, its hash and signature made. + * @param codec The codec. + * @returns The sealed data. + * @throws {TypeError} If the codec gives no string. + * @internal + */ +const sealMailJob = async (data, codec) => { + const envelope = {}; + const fields = {}; + for (const key of Object.keys(data)) { + if (ENVELOPE.has(key)) envelope[key] = data[key]; + else fields[key] = data[key]; + } + const enc = await codec.encode(JSON.stringify(fields)); + if (typeof enc !== 'string' || enc === '') { + throw new TypeError('codec.encode must give a string.'); + } + envelope.enc = enc; + return envelope; +}; + +const invalid = (message) => new MailQueueError('queue.invalid_job', `Invalid job: ${message}`); + +/** + * Opens the data of a job read from its store: sealed data as the codec + * decodes it, unsealed data as it is. + * + * @param data The data of the job. + * @param codec The codec, if any. + * @returns The data in clear, to be checked by `parseMailJob`. + * @throws {MailQueueError} `queue.codec_missing` for sealed data and no + * codec; `queue.codec_failed` if the codec throws, with its error as + * `cause`; `queue.invalid_job` if what it gives is not the fields of a job. + * @internal + */ +const openMailJob = async (data, codec) => { + if (!isSealed(data)) return data; + if (codec === undefined) { + throw new MailQueueError( + 'queue.codec_missing', + 'The job is sealed, and this worker has no codec to open it.', + ); + } + let text; + try { + text = await codec.decode(data.enc); + } catch (error) { + throw new MailQueueError( + 'queue.codec_failed', + `The codec could not open the job: ${messageOf(error)}`, + { cause: error }, + ); + } + let fields; + try { + fields = JSON.parse(text); + } catch { + throw invalid('the codec gave no JSON.'); + } + if (typeof fields !== 'object' || fields === null || Array.isArray(fields)) { + throw invalid('the codec gave no object.'); + } + const opened = {}; + for (const key of Object.keys(fields)) { + if (ENVELOPE.has(key) || key === 'enc') throw invalid(`its sealed fields hold ${key}.`); + opened[key] = fields[key]; + } + // A field in clear beside `enc` would stand over the sealed one. + for (const key of Object.keys(data)) { + if (key === 'enc') continue; + if (!ENVELOPE.has(key)) throw invalid(`a sealed job holds ${key} in clear.`); + opened[key] = data[key]; + } + return opened; +}; + +module.exports = { checkCodec, openMailJob, sealMailJob }; diff --git a/src/queue/errors.js b/src/queue/errors.js index d824969..db60495 100644 --- a/src/queue/errors.js +++ b/src/queue/errors.js @@ -3,8 +3,8 @@ /** * A failure of the queue: `queue.key_reused`, `queue.invalid_job`, - * `queue.unsupported_version`, `queue.hash_mismatch` or - * `queue.signature_invalid`. + * `queue.unsupported_version`, `queue.hash_mismatch`, + * `queue.signature_invalid`, `queue.codec_missing` or `queue.codec_failed`. */ class MailQueueError extends Error { /** diff --git a/src/queue/handler.js b/src/queue/handler.js index 7ef18b8..f6860e6 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -19,6 +19,7 @@ const { isCallerAbort } = require('../internal/sleep.js'); const { isProgrammerError, receiptOfThrown } = require('../internal/thrown.js'); const { toUuid } = require('../internal/uuid.js'); const { outcomeOfFailure, reasonOf } = require('./outcome.js'); +const { checkCodec, openMailJob } = require('./envelope.js'); const { parseMailJob } = require('./parse.js'); const { keyringOf } = require('./signing.js'); @@ -30,6 +31,13 @@ const DEFAULT_BACKOFF = Object.freeze({ const DEFAULT_OPERATOR_DELAY_MS = 5 * 60_000; // The errors of a template that fails again the same way. const TEMPLATE_CODES = new Set(['MAILER_TEMPLATE_NOT_FOUND', 'MAILER_TEMPLATE_RENDER']); +// The jobs this worker cannot read yet: of a later version, or sealed for a +// codec it lacks or that fails. They wait for a worker that can. +const WAITING_CODES = new Set([ + 'queue.unsupported_version', + 'queue.codec_missing', + 'queue.codec_failed', +]); const checkBackoff = (given) => { const backoff = { ...DEFAULT_BACKOFF, ...given }; @@ -70,6 +78,7 @@ const createHandlerConfig = (options = {}) => { audit, auditStrict: options.auditStrict ?? false, signingKeys: options.signingKeys, + codec: checkCodec(options.codec), resolveAttachment: checkOptionalFunction(options.resolveAttachment, 'resolveAttachment'), checkJob: checkOptionalFunction(options.checkJob, 'checkJob'), classify: checkOptionalFunction(options.classify, 'classify'), @@ -414,25 +423,33 @@ const attempt = async (run) => { return sendStep(run, message); }; -const readJob = (config, job) => { +// A codec that says its failure is for good (a ciphertext that does not +// authenticate) fails the job; any other failure may be an outage of its key. +const isWaiting = (error) => + WAITING_CODES.has(error?.code) && + !(error.code === 'queue.codec_failed' && error.cause?.retryable === false); + +const readJob = async (config, job) => { try { - return { data: parseMailJob(job.data, { signingKeys: config.signingKeys }) }; + const data = await openMailJob(job.data, config.codec); + return { data: parseMailJob(data, { signingKeys: config.signingKeys }) }; } catch (error) { const code = error?.code ?? 'queue.invalid_job'; - const later = code === 'queue.unsupported_version' && job.attempt < job.maxAttempts; + const later = isWaiting(error) && job.attempt < job.maxAttempts; config.log.write( later ? 'warn' : 'error', () => ({ jobId: job.id, attempt: job.attempt, code }), later ? 'queue.job.retrying' : 'queue.job.failed', ); - // A job a later version wrote waits for a worker of that version. + // A job a later version wrote waits for a worker of that version, and a + // sealed one for a worker with its codec. if (later) { return { outcome: { type: 'retry', delayMilliseconds: config.operatorDelayMilliseconds, code, - reason: error.message, + reason: reasonOf(error.message), }, }; } @@ -444,7 +461,8 @@ const readJob = (config, job) => { * Makes the handler of mail jobs, for any queue driver. * * @param options `transport`; `renderer`, for template jobs; `audit`, - * `auditStrict`; `signingKeys`, when only signed jobs are sent; + * `auditStrict`; `signingKeys`, when only signed jobs are sent; `codec`, + * which opens sealed jobs; * `resolveAttachment(ref, { job, attachment, signal })`; * `checkJob(data, job)`; `classify({ receipt, job })`; `backoff`, * `operatorDelayMilliseconds` (5 minutes), `rateLimitScope` ('job'); @@ -456,7 +474,7 @@ const readJob = (config, job) => { const createMailJobHandler = (options) => { const config = createHandlerConfig(options); return async (job, signal) => { - const read = readJob(config, job); + const read = await readJob(config, job); if (read.outcome !== undefined) return read.outcome; const run = { config, diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 7110493..75599b3 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -183,6 +183,18 @@ export interface MailJobSigningKey { readonly secret: string | Uint8Array; } +/** + * Seals the fields of a job in its store, such as an envelope encryption + * under the application's key or KMS: `encode` gets their JSON and gives the + * text stored as `enc`, `decode` gives the JSON back. The version, the hash, + * the signature and the trace context stay in clear. + */ +export interface MailJobCodec { + encode(text: string): string | Promise; + /** Throws an error with `retryable: false` when the text can never be opened, so the job fails rather than waits. */ + decode(text: string): string | Promise; +} + /** The options of the jobs of a queue. */ export interface MailJobOptions { /** The scope of the keys of the jobs. @default 'mail' */ @@ -218,9 +230,10 @@ export declare function prepareMailJob( ): PreparedMailJob; /** - * Checks the data of a job read from its queue. + * Checks the data of a job read from its queue, opened if it was sealed. * - * @throws {MailQueueError} `queue.invalid_job`; `queue.unsupported_version` + * @throws {MailQueueError} `queue.invalid_job`; `queue.codec_missing` for a + * sealed job; `queue.unsupported_version` * for a later version, or a field this version does not know; or * `queue.hash_mismatch`; `queue.signature_invalid` if `signingKeys` are given * and the job is not signed by one of them. @@ -237,7 +250,9 @@ export type MailQueueErrorCode = | 'queue.invalid_job' | 'queue.unsupported_version' | 'queue.hash_mismatch' - | 'queue.signature_invalid'; + | 'queue.signature_invalid' + | 'queue.codec_missing' + | 'queue.codec_failed'; /** A failure of the queue, by its code. */ export declare class MailQueueError extends Error { @@ -299,6 +314,12 @@ export interface MailWorkerOptions { * `queue.signature_invalid`. A worker of a queue takes the queue's. */ readonly signingKeys?: readonly MailJobSigningKey[]; + /** + * Opens sealed jobs. A sealed job this worker cannot open waits for one + * that can (`queue.codec_missing`, `queue.codec_failed`). A worker of a + * queue takes the queue's. + */ + readonly codec?: MailJobCodec; readonly backoff?: MailBackoff; /** The wait before an attempt again after an `auth` or `configuration` failure. @default 300000 */ readonly operatorDelayMilliseconds?: number; @@ -323,6 +344,8 @@ export interface MailQueueOptions extends Omit { readonly audit?: AuditSink; /** A sink that fails makes the enqueue fail, and the job retried before its send. @default false */ readonly auditStrict?: boolean; + /** Seals the fields of each job in the store; its workers open them. */ + readonly codec?: MailJobCodec; readonly logger?: Logger; readonly now?: () => number; } diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index b72dd0a..ac08797 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -5,6 +5,7 @@ const { warnThrown } = require('../internal/callback.js'); const { createLog } = require('../internal/log.js'); const { toUuid } = require('../internal/uuid.js'); +const { checkCodec, sealMailJob } = require('./envelope.js'); const { MailQueueError } = require('./errors.js'); const { createMailJobHandler } = require('./handler.js'); const { prepareMailJob } = require('./job.js'); @@ -70,19 +71,10 @@ const afterEnqueue = async (queue, { data, request }, result, enqueueOptions) => return { ...answer, duplicate: result?.duplicate }; }; -/** - * Makes a mail queue over a driver. - * - * @param options `driver`; `audit` and `auditStrict`; `namespace` ('mail'), - * the scope of the keys; `maxAttempts` (5), `maxJobBytes` and - * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job - * and which its workers verify; `logger`, `generateId`, `now`. - * @returns `{ driver, enqueue, enqueueMany, createWorker }`. - * @throws {TypeError} If an option is invalid. - */ -const createMailQueue = (options = {}) => { +// The options of a queue, checked: a bad one fails the queue, not an enqueue. +const createQueueConfig = (options) => { const driver = checkDriver(options.driver); - const { audit, logger } = options; + const { audit } = options; if (audit !== undefined && typeof audit?.append !== 'function') { throw new TypeError('The audit sink of a mail queue must have append().'); } @@ -102,23 +94,51 @@ const createMailQueue = (options = {}) => { for (const name of Object.keys(jobOptions)) { if (jobOptions[name] === undefined) delete jobOptions[name]; } - const queue = { + return { + driver, audit, auditStrict, - log: createLog(logger, { component: 'queue', queue: driver.id }), + codec: checkCodec(options.codec), + now, + jobOptions, + log: createLog(options.logger, { component: 'queue', queue: driver.id }), + }; +}; + +/** + * Makes a mail queue over a driver. + * + * @param options `driver`; `audit` and `auditStrict`; `namespace` ('mail'), + * the scope of the keys; `maxAttempts` (5), `maxJobBytes` and + * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job + * and which its workers verify; `codec`, which seals the fields of each job + * in its store and opens them in its workers; `logger`, `generateId`, `now`. + * @returns `{ driver, enqueue, enqueueMany, createWorker }`. + * @throws {TypeError} If an option is invalid. + */ +const createMailQueue = (options = {}) => { + const queue = createQueueConfig(options); + const { driver, codec } = queue; + // The job, its data sealed when the queue has a codec: the driver stores + // the sealed data, the audit reads the clear. + const prepare = async (input) => { + const prepared = prepareMailJob(input, queue.jobOptions); + if (codec === undefined) return prepared; + const data = await sealMailJob(prepared.data, codec); + return { data: prepared.data, request: { ...prepared.request, data } }; }; return { driver, async enqueue(input, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); - const prepared = prepareMailJob(input, jobOptions); + const prepared = await prepare(input); const result = await driver.enqueue(prepared.request, enqueueOptions); return afterEnqueue(queue, prepared, result, enqueueOptions); }, async enqueueMany(inputs, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); const prepared = []; - for (const input of inputs) prepared.push(prepareMailJob(input, jobOptions)); + for (const input of inputs) prepared.push(await prepare(input)); let results; if (typeof driver.enqueueMany === 'function') { const requests = []; @@ -136,11 +156,12 @@ const createMailQueue = (options = {}) => { }, createWorker(workerOptions = {}) { const handler = createMailJobHandler({ - audit, - auditStrict, - logger, - now, + audit: queue.audit, + auditStrict: queue.auditStrict, + logger: options.logger, + now: queue.now, signingKeys: options.signingKeys, + codec, ...workerOptions, }); const consumer = driver.consume(handler, { diff --git a/src/queue/parse.js b/src/queue/parse.js index f7e655c..75babd4 100644 --- a/src/queue/parse.js +++ b/src/queue/parse.js @@ -80,6 +80,12 @@ const checkAttachments = (attachments) => { const parseMailJob = (data, options = {}) => { const keyring = keyringOf(options.signingKeys); if (!isObject(data)) throw invalid('its data is no object.'); + if (typeof data.enc === 'string') { + throw new MailQueueError( + 'queue.codec_missing', + 'The job is sealed: open it with its codec before it is read.', + ); + } if (data.v !== MAIL_JOB_VERSION) { if (Number.isInteger(data.v) && data.v > MAIL_JOB_VERSION) { throw new MailQueueError( diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index a028b5e..103e67f 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -139,6 +139,10 @@ expectType( ).data.mac, ); createMailQueue({ driver, signingKeys: keys }); +const codec = { encode: (text: string) => text, decode: async (text: string) => text }; +createMailQueue({ driver, codec }); +createMailJobHandler({ transport, codec }); +expectError(createMailQueue({ driver, codec: { encode: (text: string) => text } })); createMailJobHandler({ transport, signingKeys: keys }); expectError(createMailJobHandler({ transport, signingKeys: [{ id: 'k', secret: 1 }] })); expectType<1>(MAIL_JOB_VERSION); @@ -150,6 +154,8 @@ expectType< | 'queue.unsupported_version' | 'queue.hash_mismatch' | 'queue.signature_invalid' + | 'queue.codec_missing' + | 'queue.codec_failed' >(error.code); expectError(new MailQueueError('nope', 'x')); new QueueLeaseLostError('j'); diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index 4bc505d..0dccbde 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -133,3 +133,147 @@ test('Queue: a worker with signing keys sends a signed job and fails a forged on assert.deepStrictEqual(records[0][0], 'queue.job.failed'); assert.strictEqual(records[0][1].code, 'queue.signature_invalid'); }); + +// An envelope encryption as an application would write it: AES-256-GCM. +const aesCodec = (key = Buffer.alloc(32, 9)) => { + const { createCipheriv, createDecipheriv, randomBytes } = require('node:crypto'); + return { + encode(text) { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + const sealed = Buffer.concat([cipher.update(text, 'utf8'), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), sealed]).toString('base64'); + }, + async decode(text) { + const bytes = Buffer.from(text, 'base64'); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAuthTag(bytes.subarray(12, 28)); + return Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString(); + }, + }; +}; + +const runOf = (data, extra = {}) => ({ + id: 'job-1', + name: 'mail', + data, + attempt: 1, + maxAttempts: 5, + redelivered: false, + ...extra, +}); + +test('Queue: with a codec the store holds no address, and the worker opens the job', async () => { + const clock = manualClock(NOW); + const driver = createMemoryQueue({ now: clock.now, sleep: clock.sleep }); + const codec = aesCodec(); + const queue = createMailQueue({ driver, now: clock.now, codec, signingKeys: [KEY_A] }); + const transport = fakeTransport(); + const withFile = { + ...input, + attachments: [{ filename: 'a.txt', content: new Uint8Array([65]) }], + }; + const enqueued = await queue.enqueue(withFile); + const [stored] = driver.list(); + assert.deepStrictEqual(Object.keys(stored.data).toSorted(), [ + 'enc', + 'enqueuedAt', + 'hash', + 'kid', + 'mac', + 'v', + ]); + const text = JSON.stringify(stored); + for (const secret of ['ada@example.com', 'app@example.com', '123456', 'a.txt']) { + assert.ok(!text.includes(secret), secret); + } + // A duplicate is still told by the hash in clear. + assert.strictEqual((await queue.enqueue(withFile)).duplicate, true); + await using worker = queue.createWorker({ transport }); + await driver.drain(); + assert.strictEqual(transport.calls.length, 1); + const { message } = transport.calls[0]; + assert.strictEqual(message.recipients[0].address, 'ada@example.com'); + assert.strictEqual(message.messageId, enqueued.messageId); + assert.ok(worker); + assert.throws(() => parseMailJob(stored.data), isCode('queue.codec_missing')); +}); + +test('Queue: a sealed job waits for a worker with its codec, and fails one that does not open', async () => { + const codec = aesCodec(); + const queue = createMailQueue({ driver: createMemoryQueue(), codec }); + await queue.enqueue(input); + const sealed = queue.driver.list()[0].data; + const transport = fakeTransport(); + const plain = createMailJobHandler({ transport, now: () => NOW }); + const waiting = await plain(runOf(sealed), new AbortController().signal); + assert.deepStrictEqual([waiting.type, waiting.code], ['retry', 'queue.codec_missing']); + const last = await plain(runOf(sealed, { attempt: 5 }), new AbortController().signal); + assert.deepStrictEqual([last.type, last.code], ['fail', 'queue.codec_missing']); + // Another key: the ciphertext does not authenticate, which may be an outage of the key... + const other = createMailJobHandler({ transport, codec: aesCodec(Buffer.alloc(32, 1)) }); + const outage = await other(runOf(sealed), new AbortController().signal); + assert.deepStrictEqual([outage.type, outage.code], ['retry', 'queue.codec_failed']); + // ...unless the codec says it is for good. + const final = createMailJobHandler({ + transport, + codec: { + encode: (text) => text, + decode() { + throw Object.assign(new Error('bad tag'), { retryable: false }); + }, + }, + }); + const refused = await final(runOf(sealed), new AbortController().signal); + assert.deepStrictEqual([refused.type, refused.code], ['fail', 'queue.codec_failed']); + assert.strictEqual(transport.calls.length, 0); +}); + +test('Queue: what a codec opens is checked, and a field in clear is refused', async () => { + const identity = { encode: (text) => text, decode: (text) => text }; + const handler = createMailJobHandler({ transport: fakeTransport(), codec: identity }); + const outcomeOf = async (data) => (await handler(runOf(data), new AbortController().signal)).code; + const envelope = { v: 1, hash: 'h', enqueuedAt: '2026-10-18T09:00:00.000Z' }; + assert.strictEqual(await outcomeOf({ ...envelope, enc: 'not json' }), 'queue.invalid_job'); + assert.strictEqual(await outcomeOf({ ...envelope, enc: '[1]' }), 'queue.invalid_job'); + assert.strictEqual(await outcomeOf({ ...envelope, enc: '{"hash":"x"}' }), 'queue.invalid_job'); + const queue = createMailQueue({ driver: createMemoryQueue(), codec: identity }); + await queue.enqueue(input); + const sealed = queue.driver.list()[0].data; + const overridden = { ...sealed, to: [{ address: 'victim@example.org' }] }; + assert.strictEqual(await outcomeOf(overridden), 'queue.invalid_job'); + // A writer who can seal (a codec without a key) is still stopped by the signature. + const signedQueue = createMailQueue({ + driver: createMemoryQueue(), + codec: identity, + signingKeys: [KEY_A], + }); + await signedQueue.enqueue(input); + const signed = signedQueue.driver.list()[0].data; + const opened = { ...JSON.parse(signed.enc), ...signed }; + delete opened.enc; + const forged = forge(opened); + const { v, hash, enqueuedAt, kid, mac, ...fields } = forged; + const resealed = { v, hash, enqueuedAt, kid, mac, enc: JSON.stringify(fields) }; + const verifying = createMailJobHandler({ + transport: fakeTransport(), + codec: identity, + signingKeys: [KEY_A], + }); + const outcome = await verifying(runOf(resealed), new AbortController().signal); + assert.strictEqual(outcome.code, 'queue.signature_invalid'); +}); + +test('Queue: a codec is checked, and must give a string', async () => { + assert.throws(() => createMailQueue({ driver: createMemoryQueue(), codec: {} }), /codec must/); + assert.throws( + () => createMailJobHandler({ transport: fakeTransport(), codec: { encode() {} } }), + /codec must/, + ); + const queue = createMailQueue({ + driver: createMemoryQueue(), + codec: { encode: () => 42, decode: (text) => text }, + }); + await assert.rejects(queue.enqueue(input), /codec.encode must give a string/); + assert.strictEqual(queue.driver.list().length, 0); +}); From 0017a9e148103e7965de6c17fe8075d221e0f3f7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:47:49 +0300 Subject: [PATCH 08/83] feat(queue): derive job ids under a secret The id of a job, a SHA-256 of its namespace, tenant and key, goes into the audit, the logs, the provider's idempotency key and the Message-ID, where addresses do not; but a key such as welcome:ada@example.com is found again from it by hashing the addresses one knows. idSecret (32 bytes at least) derives the id with an HMAC instead (toKeyedUuid). The examples keyed by a reset token now key by the id of the request, and the docs say a key is an id, never an address or a secret. ./queue stays within its budget (16.0 of 16 KB min+gzip). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++- README.md | 2 +- docs/guide/outbox.md | 6 +++-- docs/guide/queue.md | 10 +++++++- src/internal/uuid.js | 27 +++++++++++++++++---- src/queue/handler.js | 3 ++- src/queue/index.d.ts | 12 +++++++++- src/queue/job.js | 40 +++++++++++++++++++++++++++---- src/queue/mail-queue.js | 7 ++++-- tests/types/queue.test-d.ts | 13 +++++++++- tests/unit/queue-security.test.js | 30 +++++++++++++++++++++++ 11 files changed, 134 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 269f136..bf4a06c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the application's key) seals the fields of each job in its store, its addresses, context and attachments, leaving the version, hash, signature and trace context in clear; a worker that cannot open a sealed job gives it - back (`queue.codec_missing`, `queue.codec_failed`). + back (`queue.codec_missing`, `queue.codec_failed`). `idSecret` derives the id of a job from its + key with an HMAC, so an id that goes into the audit and the logs cannot be traced back to a key + that names a person. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases diff --git a/README.md b/README.md index 58f2caf..483e513 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ await queue.enqueue({ context: { url }, from: 'no-reply@example.com', to: user.email, - idempotencyKey: `reset-password:${token}`, + idempotencyKey: `reset-password:${resetRequest.id}`, // an id, never an address or a token }); queue.createWorker({ diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index a9f53ff..0908400 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -8,6 +8,7 @@ the keys of [`IdempotencyTransport`](/transports/idempotency) in MongoDB. The package depends on no MongoDB driver: you pass in its collections. ```js +const { randomUUID } = require('node:crypto'); const { MongoClient } = require('mongodb'); const { createMailQueue } = require('@alexify/mailer/queue'); const { IdempotencyTransport } = require('@alexify/mailer/idempotency'); @@ -30,7 +31,8 @@ const queue = createMailQueue({ driver: outbox, audit }); // The producer: the job and its audit event go in the transaction. const session = client.startSession(); await session.withTransaction(async () => { - await users.updateOne({ _id: user.id }, { $set: { resetToken } }, { session }); + const resetId = randomUUID(); // the key names the request, never its token + await db.collection('resets').insertOne({ _id: resetId, userId: user.id, token }, { session }); await queue.enqueue( { kind: 'template', @@ -38,7 +40,7 @@ await session.withTransaction(async () => { context: { url }, from: 'no-reply@example.com', to: user.email, - idempotencyKey: `reset-password:${resetToken}`, + idempotencyKey: `reset-password:${resetId}`, }, { transaction: session }, ); diff --git a/docs/guide/queue.md b/docs/guide/queue.md index b355293..f572daf 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -33,7 +33,7 @@ await queue.enqueue({ locale: user.locale, from: 'Hirevel ', to: user.email, - idempotencyKey: `reset-password:${token}`, + idempotencyKey: `reset-password:${resetRequest.id}`, priority: 'high', expiresAt: new Date(Date.now() + 15 * 60_000), }); @@ -159,6 +159,14 @@ attempt again is of unknown delivery (`delivery_unknown`), not a second send. The rendering must then be the same for the same context: no `Date.now()` in a template, or an attempt again is refused as another message. +The id of a job is a hash of its key, and it goes where addresses do not: into +the audit, the logs, the provider's idempotency key and the Message-ID. A key +is an id of what the message tells of (`reset-password:${resetRequest.id}`), +never an address or a token: a SHA-256 of `welcome:ada@example.com` is found +again by hashing the addresses one knows. `idSecret` (32 bytes at least) +derives the id with an HMAC instead; it is the same on every producer and never +rotated, as another secret makes other ids and the same message another job. + ## What the queue does with an attempt | The attempt | The queue | The state | diff --git a/src/internal/uuid.js b/src/internal/uuid.js index a3a9473..f3b8bf0 100644 --- a/src/internal/uuid.js +++ b/src/internal/uuid.js @@ -3,7 +3,7 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind // Changed in the port: the hash is the synchronous SHA-256 of node:crypto. -const { createHash } = require('node:crypto'); +const { createHash, createHmac } = require('node:crypto'); const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -15,9 +15,7 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; * @returns The UUID. * @internal */ -const toUuid = (key) => { - if (UUID.test(key)) return key.toLowerCase(); - const bytes = createHash('sha256').update(key, 'utf8').digest(); +const uuidOf = (bytes) => { // The version and variant nibbles, as a provider that checks the shape wants. bytes[6] = (bytes[6] & 0x0f) | 0x40; bytes[8] = (bytes[8] & 0x3f) | 0x80; @@ -25,4 +23,23 @@ const toUuid = (key) => { return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; }; -module.exports = { toUuid }; +const toUuid = (key) => { + if (UUID.test(key)) return key.toLowerCase(); + return uuidOf(createHash('sha256').update(key, 'utf8').digest()); +}; + +// Changed in the port: a key under a secret, an HMAC-SHA256 rather than a +// hash anyone can recompute, so that a key that names a person (an address) +// cannot be found again from its UUID by trying the addresses one knows. +/** + * A key as a UUID under a secret. + * + * @param key The key. + * @param secret The secret. + * @returns The UUID, the same for the same key and secret. + * @internal + */ +const toKeyedUuid = (key, secret) => + uuidOf(createHmac('sha256', secret).update(key, 'utf8').digest()); + +module.exports = { toKeyedUuid, toUuid }; diff --git a/src/queue/handler.js b/src/queue/handler.js index f6860e6..656c3c8 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -327,7 +327,8 @@ const buildMessage = (run, fields, attachments) => { tags: data.tags, headers: data.headers, list: data.list, - // The job's id: scoped to the tenant, of a fixed length, with no address. + // The job's id: scoped to the tenant, of a fixed length, a hash of the + // key (under `idSecret`, one no address can be found from). idempotencyKey: job.id, messageId: data.messageId, date: data.date === undefined ? undefined : new Date(data.date), diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 75599b3..4f26adf 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -60,6 +60,8 @@ export interface MailJobFieldsInput { /** * The key the job is enqueued once under: an enqueue again with it is the * same job, and with it for another message is refused. Made when absent. + * An id of what the message tells of (`reset:${request.id}`), never an + * address or a secret: the id of the job is a hash of it. */ readonly idempotencyKey?: string; readonly messageId?: string; @@ -209,6 +211,14 @@ export interface MailJobOptions { readonly now?: () => number; /** Signs each job (`kid`, `mac`) with the first key, for workers that read only signed jobs. */ readonly signingKeys?: readonly MailJobSigningKey[]; + /** + * The secret, 32 bytes at least, under which the id of a job is derived + * from its key (an HMAC rather than a SHA-256), so that an id cannot be + * traced back to a key that names a person. The same on every producer, + * and never rotated: another secret makes other ids, and the same message + * another job. + */ + readonly idSecret?: string | Uint8Array; } /** A job of a message: its data, and the request a driver takes. */ @@ -396,7 +406,7 @@ export interface MailQueue { * const queue = createMailQueue({ driver: createMemoryQueue(), audit }); * await queue.enqueue({ * kind: 'template', template: 'reset-password', context: { url }, - * from: 'app@example.com', to: user.email, idempotencyKey: `reset:${token}`, + * from: 'app@example.com', to: user.email, idempotencyKey: `reset:${resetRequest.id}`, * }); * const worker = queue.createWorker({ transport, renderer: templates }); * ``` diff --git a/src/queue/job.js b/src/queue/job.js index bc3f798..aa2d46c 100644 --- a/src/queue/job.js +++ b/src/queue/job.js @@ -11,7 +11,7 @@ const { createMessage } = require('../core/message.js'); const { generateSenderMessageId } = require('../core/message-id.js'); const { parseAddress } = require('../core/address.js'); const { toBase64 } = require('../internal/base64.js'); -const { toUuid } = require('../internal/uuid.js'); +const { toKeyedUuid, toUuid } = require('../internal/uuid.js'); const { hashJson } = require('./hash.js'); const { toJson } = require('./json.js'); const { keyringOf, signJob } = require('./signing.js'); @@ -63,6 +63,7 @@ const V1_KEYS = new Set([ 'trace', ]); const MAX_KEY_LENGTH = 256; +const MIN_ID_SECRET_BYTES = 32; const DEFAULTS = Object.freeze({ namespace: 'mail', maxAttempts: 5, @@ -197,6 +198,22 @@ const correlationIdOf = (input, jobId) => { return given ?? inMetadata ?? jobId; }; +/** + * Checks the secret the ids of jobs are derived under: 32 bytes at least. + * + * @throws {TypeError} If it is not one. + * @internal + */ +const checkIdSecret = (secret) => { + if (secret === undefined) return; + let bytes = 0; + if (typeof secret === 'string') bytes = Buffer.byteLength(secret); + else if (secret instanceof Uint8Array) bytes = secret.byteLength; + if (bytes < MIN_ID_SECRET_BYTES) { + throw new TypeError(`idSecret must hold ${MIN_ID_SECRET_BYTES} bytes at least.`); + } +}; + const resolveConfig = (options) => { const config = { ...DEFAULTS, ...options }; for (const name of ['maxAttempts', 'maxJobBytes', 'maxInlineAttachmentBytes']) { @@ -206,6 +223,7 @@ const resolveConfig = (options) => { } if (!isText(config.namespace, 128)) throw new TypeError('namespace must be a short string.'); config.keyring = keyringOf(config.signingKeys); + checkIdSecret(config.idSecret); return config; }; @@ -242,10 +260,11 @@ const fieldsOf = (input, probe, attachments, messageId) => { * `{ kind: 'content', subject, content }`, with the fields of a message. * @param options `namespace` ('mail'), `generateId`, `now`, `maxAttempts` * (5), `maxJobBytes` (256 KiB), `maxInlineAttachmentBytes` (256 KiB); - * `signingKeys`, whose first key signs the job (`kid`, `mac`). + * `signingKeys`, whose first key signs the job (`kid`, `mac`); `idSecret`, + * under which its id is derived from its key. * @returns `{ data, request }`. * @throws {TypeError} If a field is invalid, as `createMessage` finds it, or - * a signing key. + * a signing key or `idSecret`. * @throws {RangeError} If the job or its attachments are too large. */ const prepareMailJob = (input, options = {}) => { @@ -257,7 +276,11 @@ const prepareMailJob = (input, options = {}) => { const tenantId = checkKey(input.tenantId, 'tenantId'); const keyed = checkKey(input.idempotencyKey, 'idempotencyKey'); const idempotencyKey = keyed ?? generateId('idempotency-key', config.generateId); - const jobId = toUuid(`${config.namespace}\u0000${tenantId ?? ''}\u0000${idempotencyKey}`); + // A UUID of the key: under `idSecret`, one that cannot be traced back to a + // key that names a person. + const scoped = `${config.namespace}\u0000${tenantId ?? ''}\u0000${idempotencyKey}`; + const jobId = + config.idSecret === undefined ? toUuid(scoped) : toKeyedUuid(scoped, config.idSecret); const correlationId = correlationIdOf(input, jobId); const scheduledAt = checkDateField(input.scheduledAt, 'scheduledAt'); const expiresAt = checkDateField(input.expiresAt, 'expiresAt'); @@ -318,4 +341,11 @@ const prepareMailJob = (input, options = {}) => { return { data, request }; }; -module.exports = { MAIL_JOB_NAME, MAIL_JOB_VERSION, UNHASHED, V1_KEYS, prepareMailJob }; +module.exports = { + MAIL_JOB_NAME, + MAIL_JOB_VERSION, + UNHASHED, + V1_KEYS, + checkIdSecret, + prepareMailJob, +}; diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index ac08797..1b7254e 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -8,7 +8,7 @@ const { toUuid } = require('../internal/uuid.js'); const { checkCodec, sealMailJob } = require('./envelope.js'); const { MailQueueError } = require('./errors.js'); const { createMailJobHandler } = require('./handler.js'); -const { prepareMailJob } = require('./job.js'); +const { checkIdSecret, prepareMailJob } = require('./job.js'); const { keyringOf } = require('./signing.js'); const checkDriver = (driver) => { @@ -81,6 +81,7 @@ const createQueueConfig = (options) => { const auditStrict = options.auditStrict ?? false; if (typeof auditStrict !== 'boolean') throw new TypeError('auditStrict must be a boolean.'); keyringOf(options.signingKeys); + checkIdSecret(options.idSecret); const now = options.now ?? Date.now; const jobOptions = { namespace: options.namespace, @@ -89,6 +90,7 @@ const createQueueConfig = (options) => { maxInlineAttachmentBytes: options.maxInlineAttachmentBytes, generateId: options.generateId, signingKeys: options.signingKeys, + idSecret: options.idSecret, now, }; for (const name of Object.keys(jobOptions)) { @@ -112,7 +114,8 @@ const createQueueConfig = (options) => { * the scope of the keys; `maxAttempts` (5), `maxJobBytes` and * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job * and which its workers verify; `codec`, which seals the fields of each job - * in its store and opens them in its workers; `logger`, `generateId`, `now`. + * in its store and opens them in its workers; `idSecret`, under which the id + * of a job is derived from its key; `logger`, `generateId`, `now`. * @returns `{ driver, enqueue, enqueueMany, createWorker }`. * @throws {TypeError} If an option is invalid. */ diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 103e67f..73ebc7e 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -138,7 +138,18 @@ expectType( { signingKeys: keys }, ).data.mac, ); -createMailQueue({ driver, signingKeys: keys }); +createMailQueue({ driver, signingKeys: keys, idSecret: 's'.repeat(32) }); +prepareMailJob( + { + kind: 'content', + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 't' }, + }, + { idSecret: new Uint8Array(32) }, +); +expectError(createMailQueue({ driver, idSecret: 42 })); const codec = { encode: (text: string) => text, decode: async (text: string) => text }; createMailQueue({ driver, codec }); createMailJobHandler({ transport, codec }); diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index 0dccbde..264cf0e 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -277,3 +277,33 @@ test('Queue: a codec is checked, and must give a string', async () => { await assert.rejects(queue.enqueue(input), /codec.encode must give a string/); assert.strictEqual(queue.driver.list().length, 0); }); + +test('Queue: under idSecret the id of a job cannot be found from the addresses one knows', () => { + const { toUuid } = require('../../src/internal/uuid.js'); + const keyed = { ...input, idempotencyKey: 'welcome:ada@example.com' }; + const guess = toUuid('mail\u0000\u0000welcome:ada@example.com'); + // Without a secret the id is a hash anyone can recompute from a guess. + assert.strictEqual(prepareMailJob(keyed).request.id, guess); + const secret = 's'.repeat(32); + const first = prepareMailJob(keyed, { idSecret: secret }); + const again = prepareMailJob(keyed, { idSecret: Buffer.from(secret) }); + assert.notStrictEqual(first.request.id, guess); + assert.match( + first.request.id, + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + // The same key and secret: the same job, its Message-ID too. + assert.strictEqual(again.request.id, first.request.id); + assert.strictEqual(again.data.messageId, first.data.messageId); + assert.notStrictEqual( + prepareMailJob(keyed, { idSecret: 't'.repeat(32) }).request.id, + first.request.id, + ); + for (const idSecret of ['short', 42, new Uint8Array(8)]) { + assert.throws(() => prepareMailJob(keyed, { idSecret }), /idSecret must hold 32 bytes/); + assert.throws( + () => createMailQueue({ driver: createMemoryQueue(), idSecret }), + /idSecret must hold 32 bytes/, + ); + } +}); From 0e4f8b3dc1731ac64c30e196277b81b326b07496 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:49:26 +0300 Subject: [PATCH 09/83] refactor: share the bounded wait and the priorities of queue sources settledWithin was written twice (the polling driver, the BullMQ driver), and the ranks of priorities and the shape of a job request twice (the memory queue, the MongoDB outbox). They move to src/internal/settle.js and src/internal/priority.js; the outbox keeps its own rule that an id has no colon. The modules of their own take ./queue from 16.0 to 16.1 KB min+gzip: its budget goes from 16 to 16.5 KB. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/bullmq/bullmq-driver.js | 14 +------------- src/internal/polling-driver.js | 14 +------------- src/internal/priority.js | 34 ++++++++++++++++++++++++++++++++++ src/internal/settle.js | 24 ++++++++++++++++++++++++ src/mongodb/outbox.js | 20 ++++---------------- src/queue/memory-queue.js | 21 +++------------------ 7 files changed, 68 insertions(+), 61 deletions(-) create mode 100644 src/internal/priority.js create mode 100644 src/internal/settle.js diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index b257e5a..6ddf7ca 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 16, + "./queue": 16.5, "./bullmq": 3.5, "./mongodb": 5.5 } diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index a944254..138d50e 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -6,6 +6,7 @@ const { createLog } = require('../internal/log.js'); const { checkPositiveInteger } = require('../internal/options.js'); +const { settledWithin } = require('../internal/settle.js'); const { MAILER_BACKOFF_TYPE, createBullMQBackoffStrategy, @@ -115,19 +116,6 @@ const duplicateOf = async (queue, request) => { return answer; }; -// Settles within the time, or says it did not. -const settledWithin = async (promise, milliseconds) => { - let timer; - const late = new Promise((resolve) => { - timer = setTimeout(resolve, milliseconds, 'timeout'); - }); - try { - return await Promise.race([promise, late]); - } finally { - clearTimeout(timer); - } -}; - const startWorker = (config, queue, handler, consumeOptions, log) => { const { Worker } = config.bullmq; if (typeof Worker !== 'function') { diff --git a/src/internal/polling-driver.js b/src/internal/polling-driver.js index 0643173..ce1f271 100644 --- a/src/internal/polling-driver.js +++ b/src/internal/polling-driver.js @@ -7,6 +7,7 @@ const { combineSignals } = require('../core/abort-signal.js'); const { warnThrown } = require('./callback.js'); const { createLog } = require('./log.js'); const { checkOptionalFunction, checkPositiveInteger } = require('./options.js'); +const { settledWithin } = require('./settle.js'); const { sleep } = require('./sleep.js'); const DEFAULTS = Object.freeze({ @@ -174,19 +175,6 @@ const runLease = async (source, handler, lease, jobsSignal, config, log) => { } }; -// Whether a promise settles within a time: `'timeout'` when it does not. -const settledWithin = async (promise, milliseconds) => { - let timer; - const late = new Promise((resolve) => { - timer = setTimeout(resolve, milliseconds, 'timeout'); - }); - try { - return await Promise.race([promise, late]); - } finally { - clearTimeout(timer); - } -}; - const startConsumer = (source, handler, concurrency, outerSignal, config, log) => { const stopping = new AbortController(); const jobs = new AbortController(); diff --git a/src/internal/priority.js b/src/internal/priority.js new file mode 100644 index 0000000..d014c96 --- /dev/null +++ b/src/internal/priority.js @@ -0,0 +1,34 @@ +// Not ported: the priorities of the jobs of a queue as the sources that lend +// them sort them (the memory queue, a store's outbox): high, then normal, then +// low; and the shape of a request such a source takes. + +const RANK = new Map([ + ['high', 0], + ['normal', 1], + ['low', 2], +]); + +/** + * The rank of a priority, 0 the first: `undefined` for none. + * + * @internal + */ +const rankOf = (priority) => RANK.get(priority); + +/** + * Whether a value is the request of a job: an id, a name, its data, a + * priority and its attempts. + * + * @internal + */ +const isJobRequest = (request) => + typeof request?.id === 'string' && + request.id !== '' && + typeof request.name === 'string' && + typeof request.data === 'object' && + request.data !== null && + RANK.has(request.priority) && + Number.isInteger(request.maxAttempts) && + request.maxAttempts >= 1; + +module.exports = { isJobRequest, rankOf }; diff --git a/src/internal/settle.js b/src/internal/settle.js new file mode 100644 index 0000000..d908094 --- /dev/null +++ b/src/internal/settle.js @@ -0,0 +1,24 @@ +// Not ported: a wait for a promise bounded in time, for the close() of the +// queue drivers, which wait for the jobs in flight and then stop waiting. + +/** + * Whether a promise settles within a time: what it gives, or `'timeout'`. + * + * @param promise The promise, whose rejection is the caller's to handle. + * @param milliseconds The time. + * @returns What it gave, or `'timeout'`. + * @internal + */ +const settledWithin = async (promise, milliseconds) => { + let timer; + const late = new Promise((resolve) => { + timer = setTimeout(resolve, milliseconds, 'timeout'); + }); + try { + return await Promise.race([promise, late]); + } finally { + clearTimeout(timer); + } +}; + +module.exports = { settledWithin }; diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 572aefc..5ce56c2 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -10,27 +10,15 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { createLog } = require('../internal/log.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { isJobRequest, rankOf } = require('../internal/priority.js'); const { checkCollection, documentOf, sessionOf } = require('./collection.js'); -const RANK = new Map([ - ['high', 0], - ['normal', 1], - ['low', 2], -]); // The document of a pause of the whole queue: no job id has a colon. const PAUSE_ID = 'mailer:pause'; const DEFAULT_RETAIN_SECONDS = 7 * 24 * 60 * 60; -const isRequest = (request) => - typeof request?.id === 'string' && - request.id !== '' && - !request.id.includes(':') && - typeof request.name === 'string' && - typeof request.data === 'object' && - request.data !== null && - RANK.has(request.priority) && - Number.isInteger(request.maxAttempts) && - request.maxAttempts >= 1; +// A job id has no colon, which the pause's id has. +const isRequest = (request) => isJobRequest(request) && !request.id.includes(':'); const documentOfRequest = (request, now) => ({ _id: request.id, @@ -38,7 +26,7 @@ const documentOfRequest = (request, now) => ({ data: request.data, hash: request.hash, priority: request.priority, - rank: RANK.get(request.priority), + rank: rankOf(request.priority), maxAttempts: request.maxAttempts, enqueuedAt: new Date(request.enqueuedAt ?? now), availableAt: new Date(request.availableAt ?? request.enqueuedAt ?? now), diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 3e1bdbc..97245df 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -7,22 +7,7 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); - -const RANK = new Map([ - ['high', 0], - ['normal', 1], - ['low', 2], -]); - -const isRequest = (request) => - typeof request?.id === 'string' && - request.id !== '' && - typeof request.name === 'string' && - typeof request.data === 'object' && - request.data !== null && - RANK.has(request.priority) && - Number.isInteger(request.maxAttempts) && - request.maxAttempts >= 1; +const { isJobRequest, rankOf } = require('../internal/priority.js'); // The data of a job as a store gives it back: a copy, never the object // enqueued. @@ -30,7 +15,7 @@ const stored = (data) => JSON.parse(JSON.stringify(data)); // Whether `a` runs before `b`. const before = (a, b) => { - const rank = RANK.get(a.priority) - RANK.get(b.priority); + const rank = rankOf(a.priority) - rankOf(b.priority); if (rank !== 0) return rank < 0; if (a.availableAt !== b.availableAt) return a.availableAt < b.availableAt; return a.seq < b.seq; @@ -87,7 +72,7 @@ class MemorySource { } async enqueue(request) { - if (!isRequest(request)) throw new TypeError('The memory queue takes a job request.'); + if (!isJobRequest(request)) throw new TypeError('The memory queue takes a job request.'); const held = this.jobs.get(request.id); if (held !== undefined) return { id: request.id, duplicate: true, existingHash: held.hash }; if (this.#open.size >= this.#config.maxJobs) { From 001e4e9a877eea853af13b184e9520a8805ab636 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:52:02 +0300 Subject: [PATCH 10/83] test(queue): hold every queue source to one contract The memory queue and the MongoDB outbox were each tested on their own, so nothing kept their semantics one, and a source of an application's (an outbox in Postgres) had nothing to be held to. tests/helpers/queue/ source-contract.js is the contract as tests: the order of priorities, when a job is due, acknowledgements and failures, retries, releases that spend no attempt, leases that run out and are extended, fencing by the token, and the pause. It runs on the memory queue and on the outbox over a fake collection in pnpm test, and on a replica set in pnpm test:mongodb. The memory queue gives its source, as the outbox does. Co-Authored-By: Claude Opus 5.5 --- docs/guide/queue.md | 6 + src/queue/index.d.ts | 2 + src/queue/memory-queue.js | 4 +- tests/helpers/queue/source-contract.js | 168 +++++++++++++++++++++++ tests/mongodb/outbox.mongodb.test.js | 21 +++ tests/types/queue.test-d.ts | 1 + tests/unit/queue-source-contract.test.js | 12 ++ 7 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 tests/helpers/queue/source-contract.js create mode 100644 tests/unit/queue-source-contract.test.js diff --git a/docs/guide/queue.md b/docs/guide/queue.md index f572daf..1705a3a 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -229,3 +229,9 @@ lends its jobs (`reserve`, `extend`, `ack`, `retry`, `fail`, `release`), such as an outbox in your database. It keeps the lease of a job while it runs, gives a job whose lease ran out to another worker as `redelivered`, and on `close()` waits for the jobs in flight, then aborts them. + +A source of your own is held to the same semantics as the memory queue and the +MongoDB outbox: the order of priorities, leases, redelivery, fencing by the +token, releases that spend no attempt, and the pause. The repository's +`tests/helpers/queue/source-contract.js` is that contract as tests; copy it, +and run it on your source (`createMemoryQueue().source` is one to compare). diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 4f26adf..65d21ac 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -469,6 +469,8 @@ export interface MemoryQueueOptions extends PollingDriverOptions { /** The queue in memory. */ export interface MemoryQueue extends QueueDriver { + /** The source the polling driver reads: what a store's outbox implements, in memory. */ + readonly source: QueueSource; list(): readonly MemoryQueueJob[]; get(id: string): MemoryQueueJob | undefined; /** Settles when no job is ready now and none runs. */ diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 97245df..a1cc81e 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -240,7 +240,8 @@ const viewOf = (entry, now) => { * @param options `now`, `sleep`, `leaseMilliseconds` (60000), * `pollIntervalMilliseconds` (1000), `maxJobs` (100000), `retainFinished` * (10000), `logger`. - * @returns The driver, with `list()`, `get(id)`, `drain()` and `wake()`. + * @returns The driver, with `source`, `list()`, `get(id)`, `drain()` and + * `wake()`. * @throws {TypeError} If an option is invalid. */ const createMemoryQueue = (options = {}) => { @@ -253,6 +254,7 @@ const createMemoryQueue = (options = {}) => { const driver = createPollingDriver(source, { ...options, now: config.now }); return { ...driver, + source, list() { const views = []; const now = config.now(); diff --git a/tests/helpers/queue/source-contract.js b/tests/helpers/queue/source-contract.js new file mode 100644 index 0000000..146b084 --- /dev/null +++ b/tests/helpers/queue/source-contract.js @@ -0,0 +1,168 @@ +// Not ported: the contract of a QueueSource, one set of cases every source +// that lends its jobs is held to (the memory queue, the MongoDB outbox on a +// fake collection and on MongoDB itself), so that their semantics stay one: +// the order of priorities, leases, redelivery, fencing, releases that spend +// no attempt, and the pause. +const { test } = require('node:test'); +const assert = require('node:assert'); + +const START = Date.parse('2026-10-18T09:00:00Z'); +const LEASE = 60_000; + +const requestOf = (id, extra = {}) => ({ + id, + name: 'mail', + data: { v: 1, id }, + hash: `hash-${id}`, + priority: 'normal', + maxAttempts: 5, + enqueuedAt: START, + ...extra, +}); + +/** + * Registers the cases of the contract for a source. + * + * @param name The name of the source, in the names of the tests. + * @param makeSource `({ now }) => Promise<{ source, cleanup? }>`: a new, empty + * source on the clock given. + * @param options `skip`, to register the cases skipped. + */ +const sourceContract = (name, makeSource, options = {}) => { + const contract = (title, body) => + test(`Queue source (${name}): ${title}`, { skip: options.skip }, async (t) => { + let time = START; + const clock = { + now: () => time, + advance(milliseconds) { + time += milliseconds; + }, + }; + const made = await makeSource({ now: clock.now }); + if (made.cleanup !== undefined) t.after(made.cleanup); + await body(made.source, clock); + }); + + contract('a job is lent once, its first attempt, its data a copy', async (source) => { + const request = requestOf('a'); + assert.deepStrictEqual(await source.enqueue(request), { id: 'a', duplicate: false }); + const { lease } = await source.reserve(LEASE); + assert.strictEqual(lease.job.id, 'a'); + assert.strictEqual(lease.job.name, 'mail'); + assert.deepStrictEqual(lease.job.data, request.data); + assert.notStrictEqual(lease.job.data, request.data); + assert.strictEqual(lease.job.attempt, 1); + assert.strictEqual(lease.job.maxAttempts, 5); + assert.strictEqual(lease.job.redelivered, false); + assert.strictEqual(typeof lease.token, 'string'); + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + }); + + contract('an id enqueued again is a duplicate, with the hash it holds', async (source) => { + await source.enqueue(requestOf('a')); + const again = await source.enqueue(requestOf('a', { hash: 'other' })); + assert.deepStrictEqual(again, { id: 'a', duplicate: true, existingHash: 'hash-a' }); + }); + + contract('jobs go by priority, then time, then order', async (source) => { + await source.enqueue(requestOf('low', { priority: 'low' })); + await source.enqueue(requestOf('n1')); + await source.enqueue(requestOf('n0', { availableAt: START - 1000 })); + // Of two due at once, the one enqueued first: a store orders them by time. + await source.enqueue(requestOf('n2', { enqueuedAt: START + 1, availableAt: START })); + await source.enqueue(requestOf('high', { priority: 'high' })); + const order = []; + for (let index = 0; index < 5; index++) { + const { lease } = await source.reserve(LEASE); + order.push(lease.job.id); + await source.ack(lease, null); + } + assert.deepStrictEqual(order, ['high', 'n0', 'n1', 'n2', 'low']); + }); + + contract('a job not yet due tells when it is', async (source, clock) => { + await source.enqueue(requestOf('later', { availableAt: START + 5000 })); + assert.deepStrictEqual(await source.reserve(LEASE), { nextAvailableAt: START + 5000 }); + clock.advance(5000); + assert.strictEqual((await source.reserve(LEASE)).lease.job.id, 'later'); + }); + + contract('an acknowledged or failed job does not come back', async (source, clock) => { + await source.enqueue(requestOf('done')); + await source.enqueue(requestOf('failed')); + for (let index = 0; index < 2; index++) { + const { lease } = await source.reserve(LEASE); + if (lease.job.id === 'done') await source.ack(lease, { provider: 'p' }); + else await source.fail(lease, { code: 'x', reason: 'y' }); + } + clock.advance(LEASE * 2); + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + }); + + contract('a retry comes back when due, its next attempt', async (source, clock) => { + await source.enqueue(requestOf('a')); + const first = (await source.reserve(LEASE)).lease; + await source.retry(first, START + 1000, { code: 'x', reason: 'y' }); + assert.deepStrictEqual(await source.reserve(LEASE), { nextAvailableAt: START + 1000 }); + clock.advance(1000); + const second = (await source.reserve(LEASE)).lease; + assert.strictEqual(second.job.attempt, 2); + assert.strictEqual(second.job.redelivered, false); + }); + + contract('a release spends no attempt', async (source, clock) => { + await source.enqueue(requestOf('a')); + const first = (await source.reserve(LEASE)).lease; + await source.release(first, START + 500); + clock.advance(500); + const again = (await source.reserve(LEASE)).lease; + assert.strictEqual(again.job.attempt, 1); + await source.release(again); + assert.strictEqual((await source.reserve(LEASE)).lease.job.attempt, 1); + }); + + contract('a lease that runs out lends the job again, redelivered', async (source, clock) => { + await source.enqueue(requestOf('a')); + const lost = (await source.reserve(LEASE)).lease; + clock.advance(LEASE / 2); + assert.strictEqual(await source.extend(lost, LEASE), true); + clock.advance(LEASE / 2 + 1); + // Extended, it is still held. + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + clock.advance(LEASE); + const again = (await source.reserve(LEASE)).lease; + assert.strictEqual(again.job.id, 'a'); + assert.strictEqual(again.job.attempt, 1); + assert.strictEqual(again.job.redelivered, true); + assert.notStrictEqual(again.token, lost.token); + }); + + contract('a lease lost writes nothing over the next one', async (source, clock) => { + await source.enqueue(requestOf('a')); + const lost = (await source.reserve(LEASE)).lease; + clock.advance(LEASE + 1); + const held = (await source.reserve(LEASE)).lease; + assert.strictEqual(await source.extend(lost, LEASE), false); + await source.ack(lost, null); + await source.fail(lost, { code: 'x', reason: 'y' }); + await source.release(lost); + await source.retry(lost, START, { code: 'x', reason: 'y' }); + // The job is still the next holder's. + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + assert.strictEqual(await source.extend(held, LEASE), true); + await source.ack(held, null); + clock.advance(LEASE * 3); + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + }); + + contract('a pause holds every job until its time, and only grows', async (source, clock) => { + await source.enqueue(requestOf('a')); + await source.pause(START + 2000); + await source.pause(START + 1000); + assert.deepStrictEqual(await source.reserve(LEASE), { nextAvailableAt: START + 2000 }); + clock.advance(2000); + assert.strictEqual((await source.reserve(LEASE)).lease.job.id, 'a'); + }); +}; + +module.exports = { LEASE, START, requestOf, sourceContract }; diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index 5028608..ce8629d 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -194,3 +194,24 @@ test('MongoDB (replica set): a change stream wakes the workers at once', { skip await until(async () => (await outbox.get(id))?.state === 'completed', 5000); assert.ok(Date.now() - started < 3000, 'woken by the change stream'); }); + +const { sourceContract } = require('../helpers/queue/source-contract.js'); + +sourceContract( + 'mongodb, a replica set', + async ({ now }) => { + const client = new MongoClient(url); + await client.connect(); + const db = client.db(`mailer_test_${randomUUID().replaceAll('-', '')}`); + const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), now }); + await outbox.ensureIndexes(); + return { + source: outbox.source, + cleanup: async () => { + await db.dropDatabase(); + await client.close(); + }, + }; + }, + { skip }, +); diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 73ebc7e..90c9f0b 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -31,6 +31,7 @@ declare const transport: Transport; declare const engine: TemplateEngine; const driver = createMemoryQueue({ maxJobs: 10, pollIntervalMilliseconds: 100 }); expectType(driver); +expectType(driver.source); expectAssignable(driver); const queue = createMailQueue({ driver, diff --git a/tests/unit/queue-source-contract.test.js b/tests/unit/queue-source-contract.test.js new file mode 100644 index 0000000..b76feff --- /dev/null +++ b/tests/unit/queue-source-contract.test.js @@ -0,0 +1,12 @@ +// Not ported: the contract of a QueueSource held by the memory queue and by +// the MongoDB outbox on a collection in memory (tests/mongodb holds it on +// MongoDB itself). +const { createMongoOutbox } = require('../../src/mongodb/index.js'); +const { createMemoryQueue } = require('../../src/queue/index.js'); +const { fakeCollection } = require('../helpers/mongodb/fake-collection.js'); +const { sourceContract } = require('../helpers/queue/source-contract.js'); + +sourceContract('memory', async ({ now }) => ({ source: createMemoryQueue({ now }).source })); +sourceContract('mongodb', async ({ now }) => ({ + source: createMongoOutbox({ collection: fakeCollection(), now }).source, +})); From b2d850b04b78265a275b99ba3ca26c414b01172a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:54:33 +0300 Subject: [PATCH 11/83] perf(queue): keep the jobs of the memory queue on heaps Each reserve of the memory queue looked through every open job twice, once for the leases that ran out and once for the next job, so draining its default 100000 jobs was quadratic (30000 took 6.6 s). The jobs waiting are on a heap by when they are due, those due on one by the order they run in, and the leases on one by when they run out (src/internal/heap.js), stale nodes skipped by a version; the ids of finished jobs are dropped from the head of their list rather than shifted. The contract of a source holds. The heap takes ./queue from 16.1 to 16.5 KB min+gzip: its budget goes from 16.5 to 17 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- scripts/size-budgets.json | 2 +- src/internal/heap.js | 65 +++++++++++++ src/queue/memory-queue.js | 165 ++++++++++++++++++++++---------- tests/unit/queue-memory.test.js | 29 ++++++ 5 files changed, 214 insertions(+), 50 deletions(-) create mode 100644 src/internal/heap.js diff --git a/CHANGELOG.md b/CHANGELOG.md index bf4a06c..c7dd130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. minutes, and never sends again a message of unknown delivery. `scheduledAt` is a delay of the queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and - `createMemoryQueue()` is one in memory for tests. The fields of a job of version 1 are fixed: + `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs + O(log n) however many wait, its `source` held to the same contract as the outbox. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 6ddf7ca..89be9dd 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 16.5, + "./queue": 17, "./bullmq": 3.5, "./mongodb": 5.5 } diff --git a/src/internal/heap.js b/src/internal/heap.js new file mode 100644 index 0000000..6c1d477 --- /dev/null +++ b/src/internal/heap.js @@ -0,0 +1,65 @@ +// Not ported: a binary heap, the least item on top by a comparison of the +// caller's, for the memory queue, which takes its next job, its next due time +// and its next lease to run out in O(log n) rather than by looking through +// every job. + +/** + * A binary min-heap. + * + * @internal + */ +class Heap { + #items = []; + #before; + + /** + * @param before `(a, b) => boolean`: whether `a` comes before `b`. + */ + constructor(before) { + this.#before = before; + } + + get size() { + return this.#items.length; + } + + /** The least item, left on the heap. */ + peek() { + return this.#items[0]; + } + + push(item) { + const items = this.#items; + let index = items.length; + items.push(item); + while (index > 0) { + const parent = (index - 1) >> 1; + if (!this.#before(items[index], items[parent])) break; + [items[index], items[parent]] = [items[parent], items[index]]; + index = parent; + } + } + + /** Takes the least item off the heap. */ + pop() { + const items = this.#items; + const top = items[0]; + const last = items.pop(); + if (items.length === 0) return top; + items[0] = last; + let index = 0; + for (;;) { + const left = index * 2 + 1; + const right = left + 1; + let least = index; + if (left < items.length && this.#before(items[left], items[least])) least = left; + if (right < items.length && this.#before(items[right], items[least])) least = right; + if (least === index) break; + [items[index], items[least]] = [items[least], items[index]]; + index = least; + } + return top; + } +} + +module.exports = { Heap }; diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index a1cc81e..8a900a2 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -2,9 +2,11 @@ // the polling driver reads, with the semantics a store's outbox has: jobs by // priority, then time, then order; a lease per run, its attempt counted; a job // whose lease ran out comes back as redelivered; an id is a duplicate while -// its job is kept. +// its job is kept. Its jobs are on heaps, so that a reserve costs O(log n) +// however many wait. const { randomUUID } = require('node:crypto'); +const { Heap } = require('../internal/heap.js'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); @@ -13,20 +15,44 @@ const { isJobRequest, rankOf } = require('../internal/priority.js'); // enqueued. const stored = (data) => JSON.parse(JSON.stringify(data)); -// Whether `a` runs before `b`. -const before = (a, b) => { - const rank = rankOf(a.priority) - rankOf(b.priority); - if (rank !== 0) return rank < 0; +// Of two ready jobs, whether `a` runs before `b`: by priority, then time, +// then order. +const readyBefore = (a, b) => { + if (a.rank !== b.rank) return a.rank < b.rank; if (a.availableAt !== b.availableAt) return a.availableAt < b.availableAt; return a.seq < b.seq; }; +const dueBefore = (a, b) => + a.availableAt < b.availableAt || (a.availableAt === b.availableAt && a.seq < b.seq); +const leaseBefore = (a, b) => a.leaseUntil < b.leaseUntil; + +// A node of a heap: what its job was when it was pushed. It is stale once the +// job changed since (its version), and is then skipped when it comes up. +const nodeOf = (entry) => ({ + entry, + version: entry.version, + rank: entry.rank, + availableAt: entry.availableAt, + seq: entry.seq, + leaseUntil: entry.leaseUntil, +}); + +const isCurrent = (node, state) => + node.version === node.entry.version && node.entry.state === state; class MemorySource { id = 'memory'; jobs = new Map(); - // The jobs not finished, the ones reserve() looks through. - #open = new Map(); + // The jobs waiting, by when they are due; those due, by the order they run + // in; and the leases, by when they run out. + #delayed = new Heap(dueBefore); + #ready = new Heap(readyBefore); + #leases = new Heap(leaseBefore); + #open = 0; + #active = 0; + // The ids of the jobs finished, the oldest at #finishedHead. #finished = []; + #finishedHead = 0; #listeners = new Set(); #waiters = new Set(); #seq = 0; @@ -42,13 +68,29 @@ class MemorySource { for (const waiter of this.#waiters) waiter(); } + // A job waits again, its earlier nodes stale. + #wait(entry) { + entry.state = 'waiting'; + entry.token = undefined; + entry.version += 1; + this.#delayed.push(nodeOf(entry)); + } + #finish(entry, state) { entry.state = state; entry.token = undefined; - this.#open.delete(entry.id); + entry.version += 1; + this.#active -= 1; + this.#open -= 1; this.#finished.push(entry.id); - while (this.#finished.length > this.#config.retainFinished) { - this.jobs.delete(this.#finished.shift()); + while (this.#finished.length - this.#finishedHead > this.#config.retainFinished) { + this.jobs.delete(this.#finished[this.#finishedHead]); + this.#finished[this.#finishedHead] = undefined; + this.#finishedHead += 1; + } + if (this.#finishedHead > 1024 && this.#finishedHead * 2 > this.#finished.length) { + this.#finished = this.#finished.slice(this.#finishedHead); + this.#finishedHead = 0; } this.#notify(); } @@ -61,13 +103,45 @@ class MemorySource { // A job whose lease ran out comes back, its attempt run again. #expire(now) { - for (const entry of this.#open.values()) { - if (entry.state === 'active' && entry.leaseUntil <= now) { - entry.state = 'waiting'; - entry.token = undefined; - entry.attempt -= 1; - entry.redelivered = true; - } + for (;;) { + const node = this.#leases.peek(); + if (node === undefined || node.leaseUntil > now) return; + this.#leases.pop(); + if (!isCurrent(node, 'active')) continue; + const { entry } = node; + entry.attempt -= 1; + entry.redelivered = true; + this.#active -= 1; + this.#wait(entry); + } + } + + // The jobs that came due go from the delayed heap to the ready one. + #promote(now) { + for (;;) { + const node = this.#delayed.peek(); + if (node === undefined) return; + const current = isCurrent(node, 'waiting'); + if (current && node.availableAt > now) return; + this.#delayed.pop(); + if (current) this.#ready.push(node); + } + } + + #nextReady() { + for (;;) { + const node = this.#ready.peek(); + if (node === undefined || isCurrent(node, 'waiting')) return node; + this.#ready.pop(); + } + } + + #nextDue() { + for (;;) { + const node = this.#delayed.peek(); + if (node === undefined) return undefined; + if (isCurrent(node, 'waiting')) return node.availableAt; + this.#delayed.pop(); } } @@ -75,7 +149,7 @@ class MemorySource { if (!isJobRequest(request)) throw new TypeError('The memory queue takes a job request.'); const held = this.jobs.get(request.id); if (held !== undefined) return { id: request.id, duplicate: true, existingHash: held.hash }; - if (this.#open.size >= this.#config.maxJobs) { + if (this.#open >= this.#config.maxJobs) { throw new RangeError(`The memory queue holds ${this.#config.maxJobs} jobs.`); } const entry = { @@ -84,6 +158,7 @@ class MemorySource { data: stored(request.data), hash: request.hash, priority: request.priority, + rank: rankOf(request.priority), maxAttempts: request.maxAttempts, enqueuedAt: request.enqueuedAt, availableAt: request.availableAt ?? request.enqueuedAt ?? this.#config.now(), @@ -93,40 +168,34 @@ class MemorySource { redelivered: false, token: undefined, leaseUntil: 0, + version: 0, }; this.jobs.set(entry.id, entry); - this.#open.set(entry.id, entry); + this.#open += 1; + this.#delayed.push(nodeOf(entry)); this.#notify(); return { id: request.id, duplicate: false }; } - // The next job ready now, and when the next one is due. - #next(now) { - let next; - let nextAvailableAt; - for (const entry of this.#open.values()) { - if (entry.state === 'active') continue; - if (entry.availableAt > now) { - if (nextAvailableAt === undefined || entry.availableAt < nextAvailableAt) { - nextAvailableAt = entry.availableAt; - } - } else if (next === undefined || before(entry, next)) { - next = entry; - } - } - return { next, nextAvailableAt }; - } - async reserve(leaseMilliseconds) { const now = this.#config.now(); this.#expire(now); if (this.#pausedUntil > now) return { nextAvailableAt: this.#pausedUntil }; - const { next, nextAvailableAt } = this.#next(now); - if (next === undefined) return nextAvailableAt === undefined ? {} : { nextAvailableAt }; + this.#promote(now); + const node = this.#nextReady(); + if (node === undefined) { + const due = this.#nextDue(); + return due === undefined ? {} : { nextAvailableAt: due }; + } + this.#ready.pop(); + const next = node.entry; next.state = 'active'; next.attempt += 1; next.token = randomUUID(); next.leaseUntil = now + leaseMilliseconds; + next.version += 1; + this.#active += 1; + this.#leases.push(nodeOf(next)); const job = { id: next.id, name: next.name, @@ -145,6 +214,8 @@ class MemorySource { const entry = this.#holds(lease); if (entry === undefined) return false; entry.leaseUntil = this.#config.now() + leaseMilliseconds; + entry.version += 1; + this.#leases.push(nodeOf(entry)); return true; } @@ -158,10 +229,10 @@ class MemorySource { async retry(lease, availableAt, failure) { const entry = this.#holds(lease); if (entry === undefined) return; - entry.state = 'waiting'; - entry.token = undefined; + this.#active -= 1; entry.availableAt = availableAt; entry.failure = failure; + this.#wait(entry); this.#notify(); } @@ -175,10 +246,10 @@ class MemorySource { async release(lease, availableAt) { const entry = this.#holds(lease); if (entry === undefined) return; - entry.state = 'waiting'; - entry.token = undefined; + this.#active -= 1; entry.attempt -= 1; entry.availableAt = availableAt ?? this.#config.now(); + this.#wait(entry); this.#notify(); } @@ -192,12 +263,10 @@ class MemorySource { } #busy(now) { - for (const entry of this.#open.values()) { - if (entry.state === 'active' || (entry.availableAt <= now && this.#pausedUntil <= now)) { - return true; - } - } - return false; + if (this.#active > 0) return true; + if (this.#pausedUntil > now) return false; + this.#promote(now); + return this.#nextReady() !== undefined; } // Settles when no job is ready now and none runs. diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 71418dc..4090b94 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -376,3 +376,32 @@ test('Queue: a job that runs longer than its lease keeps it, and runs once', asy assert.deepStrictEqual(runs, [false]); assert.strictEqual(queue.get('long').state, 'completed'); }); + +test('Queue: the memory queue lends each of many jobs without looking through them all', async () => { + // Looking through every open job on each reserve made draining 30000 of + // them take some 9e8 steps: seconds, where heaps take milliseconds. + const { source } = createMemoryQueue({ now: () => 0, retainFinished: 10 }); + const count = 30_000; + for (let index = 0; index < count; index++) { + await source.enqueue({ + id: `j${index}`, + name: 'mail', + data: {}, + hash: 'h', + priority: index % 3 === 0 ? 'high' : 'normal', + maxAttempts: 1, + enqueuedAt: 0, + }); + } + const started = performance.now(); + let previous = 'high'; + for (let index = 0; index < count; index++) { + const { lease } = await source.reserve(60_000); + const rank = Number(lease.job.id.slice(1)) % 3 === 0 ? 'high' : 'normal'; + assert.ok(previous === rank || previous === 'high', 'high before normal'); + previous = rank; + await source.ack(lease, null); + } + assert.deepStrictEqual(await source.reserve(60_000), {}); + assert.ok(performance.now() - started < 3000, `${performance.now() - started} ms`); +}); From abfb4c9d5305a448d8fff09a19a34f897cf53c04 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:56:09 +0300 Subject: [PATCH 12/83] fix(queue): tell a settle that the token of its lease fenced off ack, retry, fail and release of a source are fenced by the token of the lease, but the outbox ignored matchedCount and the memory queue returned nothing, so a run whose lease ran out between its send and its settling was dropped silently, though another worker held the job and could send it again. They give false when nothing was written, and the polling driver logs queue.settle_fenced at warn. QueueSource types it as boolean | void, so a source of an application's that gives nothing still works. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- docs/guide/queue.md | 5 +++- index.d.ts | 13 ++++++--- src/internal/polling-driver.js | 11 +++++++- src/mongodb/outbox.js | 4 ++- src/queue/memory-queue.js | 12 ++++++--- tests/helpers/queue/source-contract.js | 11 ++++---- tests/unit/queue-memory.test.js | 37 ++++++++++++++++++++++++++ 8 files changed, 80 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7dd130..c1fba30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs - O(log n) however many wait, its `source` held to the same contract as the outbox. The fields of a job of version 1 are fixed: + O(log n) however many wait, its `source` held to the same contract as the outbox. A source's + settling writes give `false` when the token of the lease no longer holds the job, and the driver + logs `queue.settle_fenced`. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 1705a3a..96ecd22 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -228,7 +228,10 @@ clock of the tests. It is the polling driver over a source in memory: lends its jobs (`reserve`, `extend`, `ack`, `retry`, `fail`, `release`), such as an outbox in your database. It keeps the lease of a job while it runs, gives a job whose lease ran out to another worker as `redelivered`, and on `close()` -waits for the jobs in flight, then aborts them. +waits for the jobs in flight, then aborts them. A source's `ack`, `retry`, +`fail` and `release` are fenced by the token of the lease and give `false` +when it was lost: the driver then logs `queue.settle_fenced` at `warn`, as +another worker holds the job and what this run did may be done again. A source of your own is held to the same semantics as the memory queue and the MongoDB outbox: the order of priorities, leases, redelivery, fencing by the diff --git a/index.d.ts b/index.d.ts index a39e60b..2ce7bdb 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1590,11 +1590,16 @@ export interface QueueSource { ): Promise<{ readonly lease?: QueueLease; readonly nextAvailableAt?: number }>; /** false: the lease was lost (it ran out, another took it). */ extend(lease: QueueLease, leaseMilliseconds: number): Promise; - ack(lease: QueueLease, result?: JsonObject): Promise; - retry(lease: QueueLease, availableAt: number, failure: QueueFailure): Promise; - fail(lease: QueueLease, failure: QueueFailure): Promise; + /** + * Settles the job of a lease. Each settling write is fenced by the token: + * `false` when the lease was lost and nothing was written (another worker + * holds the job, and what this run did may be done twice). + */ + ack(lease: QueueLease, result?: JsonObject): Promise; + retry(lease: QueueLease, availableAt: number, failure: QueueFailure): Promise; + fail(lease: QueueLease, failure: QueueFailure): Promise; /** Back to ready at `availableAt` (now by default), its attempt not counted. */ - release(lease: QueueLease, availableAt?: number): Promise; + release(lease: QueueLease, availableAt?: number): Promise; /** The whole queue waits until then; without it, a retry of the queue is one of the job. */ pause?(until: number): Promise; /** Wakes the readers when a job may be ready: an enqueue, a change stream. */ diff --git a/src/internal/polling-driver.js b/src/internal/polling-driver.js index ce1f271..1956d01 100644 --- a/src/internal/polling-driver.js +++ b/src/internal/polling-driver.js @@ -164,7 +164,16 @@ const runLease = async (source, handler, lease, jobsSignal, config, log) => { return; } try { - await settle(source, lease, outcome, config); + const settled = await settle(source, lease, outcome, config); + // The lease ran out between the run and its settling: another worker + // holds the job, and what this run did may be done again. + if (settled === false) { + log.write( + 'warn', + () => ({ jobId: lease.job.id, outcome: outcome?.type }), + 'queue.settle_fenced', + ); + } } catch (error) { // The lease runs out, and the job comes back. log.write( diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 5ce56c2..15a1009 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -75,7 +75,9 @@ class MongoOutboxSource { $set: { ...set, token: null, leaseUntil: null, updatedAt: new Date(this.#config.now()) }, }; if (inc !== undefined) update.$inc = inc; - await this.#collection.updateOne(leased(lease), update); + const result = await this.#collection.updateOne(leased(lease), update); + // false: the lease was lost, and the job is another worker's. + return (result?.matchedCount ?? 0) > 0; } async enqueue(request, options = {}) { diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 8a900a2..65429a5 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -221,36 +221,40 @@ class MemorySource { async ack(lease, result) { const entry = this.#holds(lease); - if (entry === undefined) return; + if (entry === undefined) return false; entry.result = result; this.#finish(entry, 'completed'); + return true; } async retry(lease, availableAt, failure) { const entry = this.#holds(lease); - if (entry === undefined) return; + if (entry === undefined) return false; this.#active -= 1; entry.availableAt = availableAt; entry.failure = failure; this.#wait(entry); this.#notify(); + return true; } async fail(lease, failure) { const entry = this.#holds(lease); - if (entry === undefined) return; + if (entry === undefined) return false; entry.failure = failure; this.#finish(entry, 'failed'); + return true; } async release(lease, availableAt) { const entry = this.#holds(lease); - if (entry === undefined) return; + if (entry === undefined) return false; this.#active -= 1; entry.attempt -= 1; entry.availableAt = availableAt ?? this.#config.now(); this.#wait(entry); this.#notify(); + return true; } async pause(until) { diff --git a/tests/helpers/queue/source-contract.js b/tests/helpers/queue/source-contract.js index 146b084..1c7a882 100644 --- a/tests/helpers/queue/source-contract.js +++ b/tests/helpers/queue/source-contract.js @@ -143,14 +143,15 @@ const sourceContract = (name, makeSource, options = {}) => { clock.advance(LEASE + 1); const held = (await source.reserve(LEASE)).lease; assert.strictEqual(await source.extend(lost, LEASE), false); - await source.ack(lost, null); - await source.fail(lost, { code: 'x', reason: 'y' }); - await source.release(lost); - await source.retry(lost, START, { code: 'x', reason: 'y' }); + // Each settling write of a lost lease writes nothing, and says so. + assert.strictEqual(await source.ack(lost, null), false); + assert.strictEqual(await source.fail(lost, { code: 'x', reason: 'y' }), false); + assert.strictEqual(await source.release(lost), false); + assert.strictEqual(await source.retry(lost, START, { code: 'x', reason: 'y' }), false); // The job is still the next holder's. assert.strictEqual((await source.reserve(LEASE)).lease, undefined); assert.strictEqual(await source.extend(held, LEASE), true); - await source.ack(held, null); + assert.strictEqual(await source.ack(held, null), true); clock.advance(LEASE * 3); assert.strictEqual((await source.reserve(LEASE)).lease, undefined); }); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 4090b94..15f5c93 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -405,3 +405,40 @@ test('Queue: the memory queue lends each of many jobs without looking through th assert.deepStrictEqual(await source.reserve(60_000), {}); assert.ok(performance.now() - started < 3000, `${performance.now() - started} ms`); }); + +test('Queue: a settle its source fenced off is told, as the job may run twice', async () => { + const records = []; + const logger = { warn: (record, message) => records.push([message, record]) }; + let reserved = false; + const source = { + id: 'fenced', + enqueue: async () => ({ id: 'x', duplicate: false }), + reserve: async () => { + if (reserved) return {}; + reserved = true; + const job = { + id: 'j', + name: 'mail', + data: {}, + attempt: 1, + maxAttempts: 1, + redelivered: false, + }; + return { lease: { job, token: 't' } }; + }, + extend: async () => true, + // The lease ran out between the run and its settling. + ack: async () => false, + retry: async () => true, + fail: async () => true, + release: async () => undefined, + }; + const driver = createPollingDriver(source, { pollIntervalMilliseconds: 5, logger }); + const consumer = driver.consume(async () => ({ type: 'ack' })); + while (records.length === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + await consumer.close(); + assert.deepStrictEqual(records[0], [ + 'queue.settle_fenced', + { component: 'queue', queue: 'fenced', jobId: 'j', outcome: 'ack' }, + ]); +}); From c578ac50604837b3eeee4f0d11d4b809d2911daa Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 23:58:32 +0300 Subject: [PATCH 13/83] fix(queue): give up a job whose runs keep ending unsettled A job whose lease ran out was lent again with its attempt counted down, so a job that killed its worker (out of memory) or blocked its event loop came back for ever and took every worker down in turn; BullMQ has maxStalledCount, the outbox and the memory queue had nothing. Each source counts the runs that end unsettled (redeliveries, never counted down; BullMQ's stalledCounter), and the worker gives up a job past maxRedeliveries (3) before it renders, as delivery_unknown (queue.redelivery_exhausted), since one of those runs may have sent it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++++- docs/guide/outbox.md | 1 + docs/guide/queue.md | 1 + index.d.ts | 6 ++++++ src/bullmq/processor.js | 1 + src/internal/options.js | 19 ++++++++++++++++- src/mongodb/outbox.js | 5 ++++- src/queue/handler.js | 29 +++++++++++++++++++++++--- src/queue/index.d.ts | 7 +++++++ src/queue/memory-queue.js | 3 +++ tests/helpers/queue/source-contract.js | 23 ++++++++++++++++++++ tests/types/queue.test-d.ts | 2 +- tests/unit/bullmq.test.js | 1 + tests/unit/mongodb.test.js | 1 + tests/unit/queue-handler.test.js | 23 ++++++++++++++++++++ 15 files changed, 120 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1fba30..ed01f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,10 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs O(log n) however many wait, its `source` held to the same contract as the outbox. A source's settling writes give `false` when the token of the lease no longer holds the job, and the driver - logs `queue.settle_fenced`. The fields of a job of version 1 are fixed: + logs `queue.settle_fenced`. A source counts the runs of a job that end unsettled + (`redeliveries`, never counted down, BullMQ's `stalledCounter`), and the worker gives up a job + past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does + not take every worker down in turn for ever. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 0908400..cb49144 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -101,6 +101,7 @@ deduplication. A key of the store goes when its time passes. state: 'ready', // 'active', 'completed', 'failed' attempts: 0, availableAt: ISODate(…), enqueuedAt: ISODate(…), token: null, leaseUntil: null, redelivered: false, + redeliveries: 0, // the runs that ended unsettled, never counted down failure: { code, reason }, // of the last failed attempt result: { provider, providerMessageId }, finishedAt: ISODate(…), diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 96ecd22..d7c75f1 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -183,6 +183,7 @@ rotated, as another secret makes other ids and the same message another job. | a job past `expiresAt`, or whose retry would come after it | failed (`queue.expired`) | `failed` | | a job written by a later version of the worker | retried, for a worker of that version | `queued` | | the worker stops before the send | given back, its attempt not spent | — | +| a job whose runs ended unsettled more than `maxRedeliveries` times (3): it kills or blocks its worker | failed before it renders (`queue.redelivery_exhausted`) | `delivery_unknown` | The backoff is exponential with full jitter: 30 seconds, doubled per attempt, an hour at most (`backoff`). A `Retry-After` is a minimum, with a little jitter diff --git a/index.d.ts b/index.d.ts index 2ce7bdb..0806982 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1518,6 +1518,12 @@ export interface QueueJob { readonly enqueuedAt?: number; /** A run of this attempt before did not finish: the worker stopped, or its lease ran out. */ readonly redelivered: boolean; + /** + * How many runs of this job did not finish, never counted down: a job that + * kills its worker each time is told by it. Absent from a source that does + * not count them. + */ + readonly redeliveries?: number; } /** What a handler makes of a run of a job. */ export type QueueOutcome = diff --git a/src/bullmq/processor.js b/src/bullmq/processor.js index b777030..573b30e 100644 --- a/src/bullmq/processor.js +++ b/src/bullmq/processor.js @@ -15,6 +15,7 @@ const jobOf = (job) => ({ maxAttempts: job.opts?.attempts ?? 1, enqueuedAt: job.timestamp, redelivered: (job.stalledCounter ?? 0) > 0, + redeliveries: job.stalledCounter ?? 0, }); // A job given back, by the means its version of BullMQ has. diff --git a/src/internal/options.js b/src/internal/options.js index f75de3b..a11489d 100644 --- a/src/internal/options.js +++ b/src/internal/options.js @@ -37,4 +37,21 @@ const checkPositiveInteger = (value, name) => { return value; }; -module.exports = { checkOptionalFunction, checkPositiveInteger, checkPositiveNumber }; +/** + * Checks a whole number of zero or more. + * + * @internal + */ +const checkNonNegativeInteger = (value, name) => { + if (!Number.isInteger(value) || value < 0) { + throw new TypeError(`${name} must be a whole number of zero or more: ${String(value)}`); + } + return value; +}; + +module.exports = { + checkNonNegativeInteger, + checkOptionalFunction, + checkPositiveInteger, + checkPositiveNumber, +}; diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 15a1009..463eba6 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -33,6 +33,8 @@ const documentOfRequest = (request, now) => ({ state: 'ready', attempts: 0, redelivered: false, + // The runs that did not finish, never counted down, unlike attempts. + redeliveries: 0, token: null, leaseUntil: null, updatedAt: new Date(now), @@ -48,6 +50,7 @@ const jobOf = (document) => { maxAttempts: document.maxAttempts, enqueuedAt: document.enqueuedAt?.getTime?.(), redelivered: document.redelivered === true, + redeliveries: document.redeliveries ?? 0, }; }; @@ -118,7 +121,7 @@ class MongoOutboxSource { { state: 'active', leaseUntil: { $lte: now } }, { $set: { state: 'ready', token: null, leaseUntil: null, redelivered: true }, - $inc: { attempts: -1 }, + $inc: { attempts: -1, redeliveries: 1 }, }, ); const token = randomUUID(); diff --git a/src/queue/handler.js b/src/queue/handler.js index 656c3c8..2fff148 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -14,7 +14,11 @@ const { messageOf } = require('../internal/decorator.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { fingerprintOf } = require('../internal/fingerprint.js'); const { createLog } = require('../internal/log.js'); -const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { + checkNonNegativeInteger, + checkOptionalFunction, + checkPositiveInteger, +} = require('../internal/options.js'); const { isCallerAbort } = require('../internal/sleep.js'); const { isProgrammerError, receiptOfThrown } = require('../internal/thrown.js'); const { toUuid } = require('../internal/uuid.js'); @@ -29,6 +33,9 @@ const DEFAULT_BACKOFF = Object.freeze({ factor: 2, }); const DEFAULT_OPERATOR_DELAY_MS = 5 * 60_000; +// The runs of a job that may end without settling (its worker killed, its +// event loop blocked) before the job is given up. +const DEFAULT_MAX_REDELIVERIES = 3; // The errors of a template that fails again the same way. const TEMPLATE_CODES = new Set(['MAILER_TEMPLATE_NOT_FOUND', 'MAILER_TEMPLATE_RENDER']); // The jobs this worker cannot read yet: of a later version, or sealed for a @@ -88,6 +95,10 @@ const createHandlerConfig = (options = {}) => { 'operatorDelayMilliseconds', ), rateLimitScope, + maxRedeliveries: checkNonNegativeInteger( + options.maxRedeliveries ?? DEFAULT_MAX_REDELIVERIES, + 'maxRedeliveries', + ), random: checkOptionalFunction(options.random, 'random') ?? Math.random, now: checkOptionalFunction(options.now, 'now') ?? Date.now, log: createLog(options.logger, { component: 'queue' }), @@ -397,6 +408,17 @@ const isExpired = (run) => const attempt = async (run) => { const { config, data, job, signal } = run; + // A job whose runs keep ending unsettled kills its worker or blocks it: it + // is given up before it renders, of unknown delivery, as one of those runs + // may have sent it. + if ((job.redeliveries ?? 0) > config.maxRedeliveries) { + return fail( + run, + 'queue.redelivery_exhausted', + `${job.redeliveries} runs of the job ended unsettled, more than ${config.maxRedeliveries}.`, + { state: 'delivery_unknown', category: 'unknown' }, + ); + } if (config.checkJob !== undefined) { try { await config.checkJob(data, job); @@ -466,7 +488,8 @@ const readJob = async (config, job) => { * which opens sealed jobs; * `resolveAttachment(ref, { job, attachment, signal })`; * `checkJob(data, job)`; `classify({ receipt, job })`; `backoff`, - * `operatorDelayMilliseconds` (5 minutes), `rateLimitScope` ('job'); + * `operatorDelayMilliseconds` (5 minutes), `rateLimitScope` ('job'), + * `maxRedeliveries` (3); * `logger`, `random`, `now`. * @returns The handler: `(job, signal) => Promise`, which does * not throw. @@ -488,7 +511,7 @@ const createMailJobHandler = (options) => { if (job.redelivered) { config.log.write( 'warn', - () => ({ jobId: job.id, attempt: job.attempt }), + () => ({ jobId: job.id, attempt: job.attempt, redeliveries: job.redeliveries }), 'queue.job.redelivered', ); } diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 65d21ac..05a93ac 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -335,6 +335,13 @@ export interface MailWorkerOptions { readonly operatorDelayMilliseconds?: number; /** `queue`: a rate limit makes the whole queue wait, where the driver can. @default 'job' */ readonly rateLimitScope?: 'job' | 'queue'; + /** + * The runs of a job that may end unsettled (its worker killed, its event + * loop blocked) before the job is given up as `delivery_unknown` + * (`queue.redelivery_exhausted`), so that one job cannot kill every worker + * in turn for ever. @default 3 + */ + readonly maxRedeliveries?: number; readonly logger?: Logger; readonly random?: () => number; readonly now?: () => number; diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 65429a5..8cc9e20 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -111,6 +111,7 @@ class MemorySource { const { entry } = node; entry.attempt -= 1; entry.redelivered = true; + entry.redeliveries += 1; this.#active -= 1; this.#wait(entry); } @@ -166,6 +167,7 @@ class MemorySource { state: 'waiting', attempt: 0, redelivered: false, + redeliveries: 0, token: undefined, leaseUntil: 0, version: 0, @@ -204,6 +206,7 @@ class MemorySource { maxAttempts: next.maxAttempts, enqueuedAt: next.enqueuedAt, redelivered: next.redelivered, + redeliveries: next.redeliveries, }; next.redelivered = false; this.#notify(); diff --git a/tests/helpers/queue/source-contract.js b/tests/helpers/queue/source-contract.js index 1c7a882..8d9457b 100644 --- a/tests/helpers/queue/source-contract.js +++ b/tests/helpers/queue/source-contract.js @@ -124,6 +124,7 @@ const sourceContract = (name, makeSource, options = {}) => { contract('a lease that runs out lends the job again, redelivered', async (source, clock) => { await source.enqueue(requestOf('a')); const lost = (await source.reserve(LEASE)).lease; + assert.strictEqual(lost.job.redeliveries, 0); clock.advance(LEASE / 2); assert.strictEqual(await source.extend(lost, LEASE), true); clock.advance(LEASE / 2 + 1); @@ -137,6 +138,28 @@ const sourceContract = (name, makeSource, options = {}) => { assert.notStrictEqual(again.token, lost.token); }); + contract( + 'the runs that end unsettled are counted, and never counted down', + async (source, clock) => { + await source.enqueue(requestOf('poison')); + for (let run = 0; run < 3; run++) { + const { lease } = await source.reserve(LEASE); + assert.strictEqual(lease.job.redeliveries, run); + // The attempt is run again: only the count of lost runs grows. + assert.strictEqual(lease.job.attempt, 1); + clock.advance(LEASE + 1); + } + const { lease } = await source.reserve(LEASE); + assert.strictEqual(lease.job.redeliveries, 3); + // A release or a retry counts nothing. + await source.release(lease); + const released = (await source.reserve(LEASE)).lease; + assert.strictEqual(released.job.redeliveries, 3); + await source.retry(released, START, { code: 'x', reason: 'y' }); + assert.strictEqual((await source.reserve(LEASE)).lease.job.redeliveries, 3); + }, + ); + contract('a lease lost writes nothing over the next one', async (source, clock) => { await source.enqueue(requestOf('a')); const lost = (await source.reserve(LEASE)).lease; diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 90c9f0b..4d917ad 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -153,7 +153,7 @@ prepareMailJob( expectError(createMailQueue({ driver, idSecret: 42 })); const codec = { encode: (text: string) => text, decode: async (text: string) => text }; createMailQueue({ driver, codec }); -createMailJobHandler({ transport, codec }); +createMailJobHandler({ transport, codec, maxRedeliveries: 0 }); expectError(createMailQueue({ driver, codec: { encode: (text: string) => text } })); createMailJobHandler({ transport, signingKeys: keys }); expectError(createMailJobHandler({ transport, signingKeys: [{ id: 'k', secret: 1 }] })); diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index 6ffc426..5dd0cb6 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -334,6 +334,7 @@ test('BullMQ: the processor gives the handler the run of the job and every signa maxAttempts: 1, enqueuedAt: 1000, redelivered: true, + redeliveries: 1, }); bullmqSignal.abort('bullmq'); assert.strictEqual(runs[0].signal.reason, 'bullmq'); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index c50abb4..b15f240 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -110,6 +110,7 @@ test('MongoDB: jobs are lent by priority, then time, their attempts counted', as 'maxAttempts', 'enqueuedAt', 'redelivered', + 'redeliveries', ]); await source.ack(lease); assert.deepStrictEqual(await source.reserve(1000), {}); diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index 8e36515..c944f9e 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -531,3 +531,26 @@ test('Queue: the handler checks its options', () => { } assert.throws(() => createMailJobHandler(), /needs a transport/); }); + +test('Queue: a job whose runs keep ending unsettled is given up, unsent', async () => { + const { handler, audit, transport } = handlerOf(); + const allowed = await run(handler, jobOf({}, { redelivered: true, redeliveries: 3 })); + assert.strictEqual(allowed.type, 'ack'); + const poison = await run( + handler, + jobOf({ idempotencyKey: 'p' }, { redelivered: true, redeliveries: 4 }), + ); + assert.deepStrictEqual([poison.type, poison.code], ['fail', 'queue.redelivery_exhausted']); + assert.strictEqual(transport.calls.length, 1); + const last = audit.events.at(-1); + assert.strictEqual(last.state, 'delivery_unknown'); + assert.strictEqual(last.error.code, 'queue.redelivery_exhausted'); + // A source that counts nothing is never given up for it. + const { handler: strict } = handlerOf({ maxRedeliveries: 0 }); + assert.strictEqual((await run(strict, jobOf({ idempotencyKey: 'q' }))).type, 'ack'); + const once = await run(strict, jobOf({ idempotencyKey: 'r' }, { redeliveries: 1 })); + assert.strictEqual(once.code, 'queue.redelivery_exhausted'); + for (const maxRedeliveries of [-1, 1.5, '3']) { + assert.throws(() => handlerOf({ maxRedeliveries }), /maxRedeliveries must be a whole number/); + } +}); From 233a4983b98601addafa0e44944a218f558646bf Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:02:16 +0300 Subject: [PATCH 14/83] fix(queue): stop a run whose lease the store does not extend When extend() threw (the store out of reach), the polling driver only logged it, and the run went on past its lease: another worker took the job and sent it beside this one, the token fencing only the writes to the store, not the send. The driver keeps when the lease was last extended, and when the next keep would come after the lease runs out, it aborts the run with QueueLeaseLostError, before it sends. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/guide/queue.md | 5 ++- src/internal/polling-driver.js | 9 +++++ tests/unit/queue-memory.test.js | 66 +++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed01f74..7cc4689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs O(log n) however many wait, its `source` held to the same contract as the outbox. A source's settling writes give `false` when the token of the lease no longer holds the job, and the driver - logs `queue.settle_fenced`. A source counts the runs of a job that end unsettled + logs `queue.settle_fenced`; a run whose lease the store does not extend is aborted before the + lease runs out, not after another worker took the job. A source counts the runs of a job that end unsettled (`redeliveries`, never counted down, BullMQ's `stalledCounter`), and the worker gives up a job past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does not take every worker down in turn for ever. The fields of a job of version 1 are fixed: diff --git a/docs/guide/queue.md b/docs/guide/queue.md index d7c75f1..cd1615d 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -229,7 +229,10 @@ clock of the tests. It is the polling driver over a source in memory: lends its jobs (`reserve`, `extend`, `ack`, `retry`, `fail`, `release`), such as an outbox in your database. It keeps the lease of a job while it runs, gives a job whose lease ran out to another worker as `redelivered`, and on `close()` -waits for the jobs in flight, then aborts them. A source's `ack`, `retry`, +waits for the jobs in flight, then aborts them. When the store does not answer +the extensions of a lease, the run is aborted before the lease runs out +(`QueueLeaseLostError`), so that another worker that takes the job does not +send it beside this one. A source's `ack`, `retry`, `fail` and `release` are fenced by the token of the lease and give `false` when it was lost: the driver then logs `queue.settle_fenced` at `warn`, as another worker holds the job and what this run did may be done again. diff --git a/src/internal/polling-driver.js b/src/internal/polling-driver.js index 1956d01..b5fdff1 100644 --- a/src/internal/polling-driver.js +++ b/src/internal/polling-driver.js @@ -128,16 +128,25 @@ const runLease = async (source, handler, lease, jobsSignal, config, log) => { const lost = new AbortController(); const { signal, cleanup } = combineSignals(jobsSignal, lost.signal); const every = Math.max(1, Math.floor(config.leaseMilliseconds / 3)); + // When the lease was last taken or extended: it runs out a lease after. + let keptAt = config.now(); const keep = async () => { try { const kept = await source.extend(lease, config.leaseMilliseconds); if (!kept) lost.abort(new QueueLeaseLostError(lease.job.id)); + else keptAt = config.now(); } catch (error) { log.write( 'warn', () => ({ jobId: lease.job.id, error: error?.message }), 'queue.lease.extend_failed', ); + // The store does not answer: past the next keep the lease will have run + // out, and another worker may take the job. The run stops now, before + // it sends what the other would send too. + if (config.now() - keptAt + every >= config.leaseMilliseconds) { + lost.abort(new QueueLeaseLostError(lease.job.id)); + } } }; const keeper = setInterval(keep, every); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 15f5c93..d0432fb 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -442,3 +442,69 @@ test('Queue: a settle its source fenced off is told, as the job may run twice', { component: 'queue', queue: 'fenced', jobId: 'j', outcome: 'ack' }, ]); }); + +test('Queue: a run whose lease cannot be extended stops before the lease runs out', async () => { + let reserved = false; + const settled = []; + const records = []; + const source = { + id: 'unkept', + enqueue: async () => ({ id: 'x', duplicate: false }), + reserve: async () => { + if (reserved) return {}; + reserved = true; + const job = { + id: 'j', + name: 'mail', + data: {}, + attempt: 1, + maxAttempts: 1, + redelivered: false, + }; + return { lease: { job, token: 't' } }; + }, + // The store is out of reach: no extend answers. + extend: async () => { + throw new Error('connection reset'); + }, + ack: async () => settled.push('ack'), + retry: async () => settled.push('retry'), + fail: async () => settled.push('fail'), + release: async () => settled.push('release'), + }; + const logger = { warn: (record, message) => records.push(message) }; + const driver = createPollingDriver(source, { + leaseMilliseconds: 60, + pollIntervalMilliseconds: 5, + logger, + }); + const started = Date.now(); + let onStopped; + const stoppedOnce = new Promise((resolve) => { + onStopped = resolve; + }); + const consumer = driver.consume( + (job, signal) => + new Promise((resolve) => { + signal.addEventListener( + 'abort', + () => { + onStopped({ reason: signal.reason, after: Date.now() - started }); + resolve({ type: 'ack' }); + }, + { once: true }, + ); + }), + ); + const stopped = await Promise.race([ + stoppedOnce, + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + await consumer.close({ timeoutMilliseconds: 50 }); + assert.ok(stopped?.reason instanceof QueueLeaseLostError); + // Two keeps failed (at 20 and 40 ms): the third would come after the lease. + assert.ok(stopped.after < 60, `${stopped.after} ms`); + assert.deepStrictEqual(settled, []); + assert.ok(records.includes('queue.lease.extend_failed')); + assert.ok(records.includes('queue.lease.lost')); +}); From 17a7650285f20c1e652b0f9335c839dbafcb637b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:18:29 +0300 Subject: [PATCH 15/83] fix(mongodb): take a lease back only past the skew of the workers' clocks The outbox wrote leases and took them back by the clock of each worker, so a worker whose clock was behind wrote a lease another already saw as run out, and the job ran in both. A lease is taken back clockSkewTolerance- Milliseconds (5000) after it ran out by the taker's clock, and the lease must be more than three times longer. The opt-in suite registers its cleanup before anything that may throw, so a failed setup no longer holds the run open. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- docs/guide/outbox.md | 5 ++++ src/mongodb/index.d.ts | 6 +++++ src/mongodb/outbox.js | 33 ++++++++++++++++++++++---- tests/helpers/queue/source-contract.js | 6 +++-- tests/mongodb/outbox.mongodb.test.js | 19 +++++++++------ tests/unit/mongodb.test.js | 27 ++++++++++++++++++++- 7 files changed, 83 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc4689..4f549cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. its indexes and the TTL of finished jobs. `createMongoAuditSink` (each event once by its id, in the transaction) and `createMongoIdempotencyStore` (each step atomic, for `IdempotencyTransport` across processes) go with it. `pnpm test:mongodb` runs them on a - replica set. + replica set. A lease is taken back `clockSkewToleranceMilliseconds` (5 s) after it ran out, so + that a worker whose clock is behind does not run a job beside its holder. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index cb49144..cbe4b8c 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -73,6 +73,11 @@ The outbox is a `QueueSource`, read by the polling driver of `./queue`: - a rate limit of the whole queue (`rateLimitScope: 'queue'`) pauses every worker, through a document of its own. +Leases are written by the clock of each worker, so a lease is taken back only +`clockSkewToleranceMilliseconds` (5 s) after it ran out, lest a worker whose +clock is behind still holds it; keep the clocks of the workers in sync (NTP), +within that tolerance, and the lease more than three times longer. + Workers poll (`pollIntervalMilliseconds`, a second). With `changeStream: true`, on a replica set, a change of the collection wakes them at once, and the poll is only the fallback. diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index aaa3dbd..a06ffd1 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -52,6 +52,12 @@ export interface MongoOutboxOptions extends PollingDriverOptions { readonly collection: MongoCollectionLike; /** Wakes the workers on a change of the collection, on a replica set. @default false */ readonly changeStream?: boolean; + /** + * How far the clocks of the workers may be apart: a lease is taken back + * only that long after it ran out. The lease must be more than three times + * longer. @default 5000 + */ + readonly clockSkewToleranceMilliseconds?: number; /** The time a finished job is kept, and so the window of its duplicates. @default 604800 */ readonly retainFinishedSeconds?: number; readonly logger?: Logger; diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 463eba6..802b038 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -9,13 +9,21 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { createLog } = require('../internal/log.js'); -const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { + checkNonNegativeInteger, + checkOptionalFunction, + checkPositiveInteger, +} = require('../internal/options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); const { checkCollection, documentOf, sessionOf } = require('./collection.js'); // The document of a pause of the whole queue: no job id has a colon. const PAUSE_ID = 'mailer:pause'; const DEFAULT_RETAIN_SECONDS = 7 * 24 * 60 * 60; +// How far the clocks of two workers may be apart: a lease is taken back only +// that long after it ran out by the clock of the worker that takes it. +const DEFAULT_CLOCK_SKEW_MS = 5000; +const DEFAULT_LEASE_MS = 60_000; // A job id has no colon, which the pause's id has. const isRequest = (request) => isJobRequest(request) && !request.id.includes(':'); @@ -116,9 +124,12 @@ class MongoOutboxSource { const now = new Date(this.#config.now()); const paused = await this.#paused(now); if (paused !== undefined) return { nextAvailableAt: paused.getTime() }; - // A job whose lease ran out is ready again, its attempt to run again. + // A job whose lease ran out is ready again, its attempt to run again: + // by this worker's clock, as long ago as two clocks may be apart, lest a + // worker whose clock is behind still holds it. + const ranOut = new Date(now.getTime() - this.#config.clockSkewToleranceMilliseconds); await this.#collection.updateMany( - { state: 'active', leaseUntil: { $lte: now } }, + { state: 'active', leaseUntil: { $lte: ranOut } }, { $set: { state: 'ready', token: null, leaseUntil: null, redelivered: true }, $inc: { attempts: -1, redeliveries: 1 }, @@ -213,8 +224,9 @@ const subscribe = (collection, listener, config) => { * @param options `collection`, `db.collection('mail_outbox')`; * `changeStream` (false), to wake the workers on a change (a replica set); * `retainFinishedSeconds` (7 days), the time a finished job is kept, and so - * the window of its duplicates; `now`, `logger`, and the options of the - * polling driver. + * the window of its duplicates; `clockSkewToleranceMilliseconds` (5000), how + * far the clocks of the workers may be apart; `now`, `logger`, and the + * options of the polling driver. * @returns The driver, with `source`, `ensureIndexes()` and `get(id)`. * @throws {TypeError} If an option is invalid. */ @@ -225,8 +237,19 @@ const createMongoOutbox = (options = {}) => { if (changeStream && typeof collection.watch !== 'function') { throw new TypeError('changeStream needs a collection with watch().'); } + const clockSkewToleranceMilliseconds = checkNonNegativeInteger( + options.clockSkewToleranceMilliseconds ?? DEFAULT_CLOCK_SKEW_MS, + 'clockSkewToleranceMilliseconds', + ); + // A lease shorter than the skew would be extended within the skew. + if ((options.leaseMilliseconds ?? DEFAULT_LEASE_MS) <= 3 * clockSkewToleranceMilliseconds) { + throw new TypeError( + 'leaseMilliseconds must be more than 3 times clockSkewToleranceMilliseconds.', + ); + } const config = { now: checkOptionalFunction(options.now, 'now') ?? Date.now, + clockSkewToleranceMilliseconds, changeStream, retainFinishedSeconds: checkPositiveInteger( options.retainFinishedSeconds ?? DEFAULT_RETAIN_SECONDS, diff --git a/tests/helpers/queue/source-contract.js b/tests/helpers/queue/source-contract.js index 8d9457b..cabf904 100644 --- a/tests/helpers/queue/source-contract.js +++ b/tests/helpers/queue/source-contract.js @@ -8,6 +8,8 @@ const assert = require('node:assert'); const START = Date.parse('2026-10-18T09:00:00Z'); const LEASE = 60_000; +// Past the end of a lease by more than the clocks of two workers may be apart. +const PAST_LEASE = LEASE + 10_000; const requestOf = (id, extra = {}) => ({ id, @@ -147,7 +149,7 @@ const sourceContract = (name, makeSource, options = {}) => { assert.strictEqual(lease.job.redeliveries, run); // The attempt is run again: only the count of lost runs grows. assert.strictEqual(lease.job.attempt, 1); - clock.advance(LEASE + 1); + clock.advance(PAST_LEASE); } const { lease } = await source.reserve(LEASE); assert.strictEqual(lease.job.redeliveries, 3); @@ -163,7 +165,7 @@ const sourceContract = (name, makeSource, options = {}) => { contract('a lease lost writes nothing over the next one', async (source, clock) => { await source.enqueue(requestOf('a')); const lost = (await source.reserve(LEASE)).lease; - clock.advance(LEASE + 1); + clock.advance(PAST_LEASE); const held = (await source.reserve(LEASE)).lease; assert.strictEqual(await source.extend(lost, LEASE), false); // Each settling write of a lost lease writes nothing, and says so. diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index ce8629d..7051cbe 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -41,6 +41,13 @@ const setup = async (t, options = {}) => { const client = new MongoClient(url); await client.connect(); const db = client.db(`mailer_test_${randomUUID().replaceAll('-', '')}`); + const workers = []; + // Before anything that may throw: a client left open holds the run. + t.after(async () => { + for (const worker of workers) await worker.close({ timeoutMilliseconds: 1000 }); + await db.dropDatabase(); + await client.close(); + }); const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), pollIntervalMilliseconds: 20, @@ -52,12 +59,6 @@ const setup = async (t, options = {}) => { await audit.ensureIndexes(); await store.ensureIndexes(); const queue = createMailQueue({ driver: outbox, audit }); - const workers = []; - t.after(async () => { - for (const worker of workers) await worker.close({ timeoutMilliseconds: 1000 }); - await db.dropDatabase(); - await client.close(); - }); const worker = (workerOptions) => { const consumer = queue.createWorker(workerOptions); workers.push(consumer); @@ -138,7 +139,11 @@ test( 'MongoDB (replica set): a job whose lease ran out goes to another worker, redelivered', { skip }, async (t) => { - const { outbox, queue } = await setup(t, { leaseMilliseconds: 100 }); + // One process, one clock: no skew to wait out. + const { outbox, queue } = await setup(t, { + leaseMilliseconds: 100, + clockSkewToleranceMilliseconds: 0, + }); const { id } = await queue.enqueue(job()); const { lease } = await outbox.source.reserve(100); assert.equal(lease.job.id, id); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index b15f240..ab0fac0 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -121,7 +121,8 @@ test('MongoDB: a lease that runs out gives the job to another, the late settle i await source.enqueue(requestOf('a')); const first = (await source.reserve(1000)).lease; assert.strictEqual(await source.extend(first, 1000), true); - advance(1001); + // Past its end by more than the skew of the clocks (5 s). + advance(6001); const second = (await source.reserve(1000)).lease; assert.deepStrictEqual([second.job.attempt, second.job.redelivered], [1, true]); // The first holder's writes do nothing now. @@ -404,3 +405,27 @@ test('MongoDB: IdempotencyTransport sends a key once over the shared store', asy assert.strictEqual(inner.calls.length, 1); assert.strictEqual(again.messageId, first.messageId); }); + +test('MongoDB: a lease is taken back only past the skew of the workers’ clocks', async () => { + const { source, advance } = outboxOf({ extra: { clockSkewToleranceMilliseconds: 2000 } }); + await source.enqueue(requestOf('a')); + await source.reserve(1000); + // Run out by this clock, but a worker whose clock is 2 s behind still holds it. + advance(2500); + assert.deepStrictEqual(await source.reserve(1000), {}); + advance(501); + assert.strictEqual((await source.reserve(1000)).lease.job.redelivered, true); + const strict = outboxOf({ extra: { clockSkewToleranceMilliseconds: 0 } }); + await strict.source.enqueue(requestOf('b')); + await strict.source.reserve(1000); + strict.advance(1000); + assert.strictEqual((await strict.source.reserve(1000)).lease.job.id, 'b'); + assert.throws( + () => outboxOf({ extra: { clockSkewToleranceMilliseconds: -1 } }), + /clockSkewToleranceMilliseconds must be a whole number/, + ); + assert.throws( + () => outboxOf({ extra: { leaseMilliseconds: 15_000 } }), + /more than 3 times clockSkewToleranceMilliseconds/, + ); +}); From 1ac61fc533f3cfcaf10190411c5b4170a55d89ce Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:21:50 +0300 Subject: [PATCH 16/83] feat(queue): drop the message of a job done with A finished job kept its data, the message with its addresses, context and attachments (a reset link among them), for seven days in the outbox and a day (a week if failed) in Redis, and in their backups. retainData, on the memory queue, the outbox and the BullMQ driver, drops it once the job is done with: 'failed' (the default) keeps the data of failed jobs only, for a retry by hand, 'all' keeps every one and 'none' none. The version and the hash stay, so an enqueue again is still told a duplicate. The outbox unsets data; the BullMQ processor writes { v, hash } through job.updateData, and a store that refuses it leaves the send acknowledged and warns (MAILER_QUEUE_RETAIN_FAILED). ./bullmq goes from 3.4 to 3.7 KB min+gzip: its budget goes from 3.5 to 4 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++- docs/guide/outbox.md | 5 ++- docs/guide/queue.md | 5 +++ docs/transports/bullmq.md | 4 ++- scripts/size-budgets.json | 2 +- src/bullmq/bullmq-driver.js | 14 +++++--- src/bullmq/index.d.ts | 5 +++ src/bullmq/processor.js | 22 +++++++++++- src/internal/options.js | 17 +++++++++ src/mongodb/index.d.ts | 2 ++ src/mongodb/outbox.js | 14 ++++++-- src/queue/index.d.ts | 12 ++++++- src/queue/memory-queue.js | 14 ++++++-- tests/helpers/bullmq/fake-bullmq.js | 4 +++ tests/helpers/mongodb/fake-collection.js | 5 ++- tests/types/queue.test-d.ts | 2 ++ tests/unit/bullmq.test.js | 44 ++++++++++++++++++++++++ tests/unit/mongodb.test.js | 24 +++++++++++++ tests/unit/queue-memory.test.js | 26 ++++++++++++++ 19 files changed, 209 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f549cb..7f20f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,10 @@ dependency, BullMQ, MongoDB and the template engines being passed in. in the transaction) and `createMongoIdempotencyStore` (each step atomic, for `IdempotencyTransport` across processes) go with it. `pnpm test:mongodb` runs them on a replica set. A lease is taken back `clockSkewToleranceMilliseconds` (5 s) after it ran out, so - that a worker whose clock is behind does not run a job beside its holder. + that a worker whose clock is behind does not run a job beside its holder. Every driver drops the + message of a job done with (`retainData`: `'failed'` keeps the failed ones, `'all'`, `'none'`), + keeping its version and hash, so a duplicate is still told and a reset link does not wait in the + store. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index cbe4b8c..ee891ee 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -94,7 +94,10 @@ is only the fallback. A finished job is kept `retainFinishedSeconds` (a week), and while it is kept, an enqueue under its key is a duplicate: the retention is the window of -deduplication. A key of the store goes when its time passes. +deduplication. A key of the store goes when its time passes. A completed job +keeps no `data`, only its hash (`retainData: 'failed'`, the default; `'all'` +or `'none'`), so a reset link in it does not wait a week in the collection and +its backups. ## A job as stored diff --git a/docs/guide/queue.md b/docs/guide/queue.md index cd1615d..105745f 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -80,6 +80,11 @@ not the worker: same message; - a job is 256 KiB at most. +Once a job is done with, the store drops its message (`retainData`, on the +driver): `'failed'` by default keeps the data of the failed jobs only, for a +retry by hand; `'all'` keeps every one, `'none'` none. The version and the hash +of a job stay, so an enqueue again is still told a duplicate. + A job is of version 1 (`v: 1`), and its fields are those of that version. A worker that meets a later version, or a field it does not know, gives the job back for a worker that reads it (`queue.unsupported_version`) instead of diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index fb8b56a..0f591c6 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -35,7 +35,9 @@ A duplicate is told by reading the job back (`detectDuplicates`, one read per enqueue; `'bulk'` for `enqueueMany` too). It holds while BullMQ keeps the job, so the retention is by time: one by count shrinks the window of duplicates as the queue gets busy. `deduplication` of BullMQ is not used, as -it holds only while the first job is not finished. +it holds only while the first job is not finished. A completed job keeps only +`{ v, hash }` of its data (`job.updateData`, by `retainData: 'failed'`, the +default; `'all'` or `'none'`), so its message does not wait a day in Redis. ## The workers it makes diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 89be9dd..58b5952 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -38,6 +38,6 @@ "./tee": 3.5, "./audit": 2, "./queue": 17, - "./bullmq": 3.5, + "./bullmq": 4, "./mongodb": 5.5 } diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index 138d50e..43fc9bd 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -5,7 +5,7 @@ // 1, as 0 in BullMQ runs before every prioritized job. const { createLog } = require('../internal/log.js'); -const { checkPositiveInteger } = require('../internal/options.js'); +const { checkPositiveInteger, checkRetainData } = require('../internal/options.js'); const { settledWithin } = require('../internal/settle.js'); const { MAILER_BACKOFF_TYPE, @@ -90,6 +90,7 @@ const createDriverConfig = (options) => { workerOptions: checkReserved(options.workerOptions, RESERVED_WORKER_OPTIONS, 'workerOptions'), priorities: checkPriorities(options.priorities), detectDuplicates, + retainData: checkRetainData(options.retainData), }; }; @@ -134,7 +135,11 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { ? queue.rateLimit(milliseconds) : holder.worker.rateLimit(milliseconds) : undefined; - const processor = createBullMQProcessor(handler, { rateLimit, signal: consumeOptions.signal }); + const processor = createBullMQProcessor(handler, { + rateLimit, + signal: consumeOptions.signal, + retainData: config.retainData, + }); const worker = new Worker(config.queueName, processor, { ...workerOptions, connection: config.connection, @@ -187,7 +192,8 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { * `connection`, `prefix`, `queueName` ('mail'); `jobOptions`, such as * `removeOnComplete`; `workerOptions`, such as `limiter` or * `lockDuration`; `priorities` (`{ high: 1, normal: 2, low: 3 }`); - * `detectDuplicates` (true); `logger`. + * `detectDuplicates` (true); `retainData` ('failed'), the finished jobs whose + * data is kept; `logger`. * @returns The driver, with `queue` and `createProcessor(handler)`. * @throws {TypeError} If an option is invalid. */ @@ -230,7 +236,7 @@ const createBullMQDriver = (options = {}) => { consume: (handler, consumeOptions = {}) => startWorker(config, queue, handler, consumeOptions, log), createProcessor: (handler, processorOptions) => - createBullMQProcessor(handler, processorOptions), + createBullMQProcessor(handler, { retainData: config.retainData, ...processorOptions }), async close() { if (config.owned) await queue.close(); }, diff --git a/src/bullmq/index.d.ts b/src/bullmq/index.d.ts index ece436d..19f8158 100644 --- a/src/bullmq/index.d.ts +++ b/src/bullmq/index.d.ts @@ -17,6 +17,7 @@ export interface BullMQJobLike { stalledCounter?: number; moveToDelayed?(timestamp: number, token?: string): Promise; moveToWait?(token?: string): Promise; + updateData?(data: any): Promise; } /** The members of a BullMQ `Queue` the driver calls. */ @@ -124,6 +125,8 @@ export interface BullMQDriverOptions Promise; readonly signal?: AbortSignal; + /** The finished jobs whose data is kept, through `job.updateData`. @default 'failed' */ + readonly retainData?: 'all' | 'failed' | 'none'; } /** A consumer of a BullMQ driver: its worker. */ diff --git a/src/bullmq/processor.js b/src/bullmq/processor.js index 573b30e..256abf2 100644 --- a/src/bullmq/processor.js +++ b/src/bullmq/processor.js @@ -4,6 +4,7 @@ // and a job moved back for a release, which spends no attempt. const { warnThrown } = require('../internal/callback.js'); +const { checkRetainData } = require('../internal/options.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { namedError, rateLimitError, retryError } = require('./backoff.js'); @@ -42,6 +43,21 @@ const release = async (job, token, outcome) => { }); }; +// The data of a job done with, by `retainData`: its version and its hash, +// by which an enqueue again is still told a duplicate, and no message. A +// store that fails it leaves the message there, and the send stands. +const dropData = async (job) => { + if (typeof job.updateData !== 'function') return; + const kept = {}; + if (job.data?.v !== undefined) kept.v = job.data.v; + if (typeof job.data?.hash === 'string') kept.hash = job.data.hash; + try { + await job.updateData(kept); + } catch (error) { + warnThrown('The data of a finished BullMQ job', error, 'MAILER_QUEUE_RETAIN_FAILED'); + } +}; + /** * Makes a BullMQ processor of a queue handler, for a `Worker` of your own. * Its worker needs `settings.backoffStrategy` from @@ -49,7 +65,8 @@ const release = async (job, token, outcome) => { * * @param handler The handler, such as `createMailJobHandler()`. * @param options `rateLimit(ms)`, the queue's `rateLimit`, for a retry of the - * whole queue (it needs the worker's `limiter`); `signal`. + * whole queue (it needs the worker's `limiter`); `signal`; `retainData` + * ('failed'), the finished jobs whose data is kept. * @returns The processor, with `shutdown(reason)`, which aborts the jobs it * runs. * @throws {TypeError} If the handler is not a function. @@ -60,6 +77,7 @@ const createBullMQProcessor = (handler, options = {}) => { if (rateLimit !== undefined && typeof rateLimit !== 'function') { throw new TypeError('rateLimit must be a function.'); } + const retainData = checkRetainData(options.retainData); const shutdown = new AbortController(); const signals = [shutdown.signal]; if (options.signal !== undefined) signals.push(options.signal); @@ -81,6 +99,7 @@ const createBullMQProcessor = (handler, options = {}) => { } switch (outcome?.type) { case 'ack': + if (retainData !== 'all') await dropData(job); return outcome.result ?? null; case 'retry': if (outcome.scope === 'queue' && rateLimit !== undefined) { @@ -89,6 +108,7 @@ const createBullMQProcessor = (handler, options = {}) => { } throw retryError(outcome); case 'fail': + if (retainData === 'none') await dropData(job); // Its reason goes into the dashboards of the queue. throw namedError( 'UnrecoverableError', diff --git a/src/internal/options.js b/src/internal/options.js index a11489d..8cd4871 100644 --- a/src/internal/options.js +++ b/src/internal/options.js @@ -49,8 +49,25 @@ const checkNonNegativeInteger = (value, name) => { return value; }; +// What a queue keeps of the data of a finished job: all of it, that of the +// failed ones (for a retry by hand), or none. +const RETAIN_DATA = new Set(['all', 'failed', 'none']); + +/** + * Checks `retainData` of a queue, `'failed'` by default. + * + * @internal + */ +const checkRetainData = (value = 'failed') => { + if (!RETAIN_DATA.has(value)) { + throw new TypeError(`retainData must be 'all', 'failed' or 'none': ${String(value)}`); + } + return value; +}; + module.exports = { checkNonNegativeInteger, + checkRetainData, checkOptionalFunction, checkPositiveInteger, checkPositiveNumber, diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index a06ffd1..738c297 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -60,6 +60,8 @@ export interface MongoOutboxOptions extends PollingDriverOptions { readonly clockSkewToleranceMilliseconds?: number; /** The time a finished job is kept, and so the window of its duplicates. @default 604800 */ readonly retainFinishedSeconds?: number; + /** The finished jobs whose data is kept: the others keep their version and hash. @default 'failed' */ + readonly retainData?: 'all' | 'failed' | 'none'; readonly logger?: Logger; } diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 802b038..86034fe 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -13,6 +13,7 @@ const { checkNonNegativeInteger, checkOptionalFunction, checkPositiveInteger, + checkRetainData, } = require('../internal/options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); const { checkCollection, documentOf, sessionOf } = require('./collection.js'); @@ -81,11 +82,14 @@ class MongoOutboxSource { if (config.changeStream) this.subscribe = (listener) => subscribe(collection, listener, config); } - async #settle(lease, set, inc) { + async #settle(lease, set, inc, dropData = false) { const update = { $set: { ...set, token: null, leaseUntil: null, updatedAt: new Date(this.#config.now()) }, }; if (inc !== undefined) update.$inc = inc; + // The message is not kept once it is done with: its addresses, context + // and a reset link among it. The hash stays, by which a duplicate is told. + if (dropData) update.$unset = { data: '' }; const result = await this.#collection.updateOne(leased(lease), update); // false: the lease was lost, and the job is another worker's. return (result?.matchedCount ?? 0) > 0; @@ -176,7 +180,8 @@ class MongoOutboxSource { ack(lease, result) { const finishedAt = new Date(this.#config.now()); - return this.#settle(lease, { state: 'completed', result: result ?? null, finishedAt }); + const set = { state: 'completed', result: result ?? null, finishedAt }; + return this.#settle(lease, set, undefined, this.#config.retainData !== 'all'); } retry(lease, availableAt, failure) { @@ -185,7 +190,8 @@ class MongoOutboxSource { fail(lease, failure) { const finishedAt = new Date(this.#config.now()); - return this.#settle(lease, { state: 'failed', failure, finishedAt }); + const set = { state: 'failed', failure, finishedAt }; + return this.#settle(lease, set, undefined, this.#config.retainData === 'none'); } release(lease, availableAt) { @@ -223,6 +229,7 @@ const subscribe = (collection, listener, config) => { * * @param options `collection`, `db.collection('mail_outbox')`; * `changeStream` (false), to wake the workers on a change (a replica set); + * `retainData` ('failed'), the finished jobs whose data is kept; * `retainFinishedSeconds` (7 days), the time a finished job is kept, and so * the window of its duplicates; `clockSkewToleranceMilliseconds` (5000), how * far the clocks of the workers may be apart; `now`, `logger`, and the @@ -249,6 +256,7 @@ const createMongoOutbox = (options = {}) => { } const config = { now: checkOptionalFunction(options.now, 'now') ?? Date.now, + retainData: checkRetainData(options.retainData), clockSkewToleranceMilliseconds, changeStream, retainFinishedSeconds: checkPositiveInteger( diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 05a93ac..366e0e8 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -461,7 +461,8 @@ export interface MemoryQueueJob { readonly attempt: number; readonly maxAttempts: number; readonly availableAt: number; - readonly data: JsonObject; + /** Absent once the job is done with, by `retainData`. */ + readonly data?: JsonObject; readonly failure?: { readonly code: string; readonly reason: string }; readonly result?: JsonObject; } @@ -472,8 +473,17 @@ export interface MemoryQueueOptions extends PollingDriverOptions { readonly maxJobs?: number; /** The jobs finished it keeps, and so the window of its duplicates. @default 10000 */ readonly retainFinished?: number; + /** See {@link MailQueueRetainData}. @default 'failed' */ + readonly retainData?: MailQueueRetainData; } +/** + * The finished jobs whose data, the message with its addresses and context, + * a store keeps: `'failed'` those that failed, for a retry by hand; `'all'`; + * or `'none'`. A job's version and hash stay, by which a duplicate is told. + */ +export type MailQueueRetainData = 'all' | 'failed' | 'none'; + /** The queue in memory. */ export interface MemoryQueue extends QueueDriver { /** The source the polling driver reads: what a store's outbox implements, in memory. */ diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 8cc9e20..45044d1 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -8,7 +8,11 @@ const { randomUUID } = require('node:crypto'); const { Heap } = require('../internal/heap.js'); const { createPollingDriver } = require('../internal/polling-driver.js'); -const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { + checkOptionalFunction, + checkPositiveInteger, + checkRetainData, +} = require('../internal/options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); // The data of a job as a store gives it back: a copy, never the object @@ -226,6 +230,7 @@ class MemorySource { const entry = this.#holds(lease); if (entry === undefined) return false; entry.result = result; + if (this.#config.retainData !== 'all') entry.data = undefined; this.#finish(entry, 'completed'); return true; } @@ -245,6 +250,7 @@ class MemorySource { const entry = this.#holds(lease); if (entry === undefined) return false; entry.failure = failure; + if (this.#config.retainData === 'none') entry.data = undefined; this.#finish(entry, 'failed'); return true; } @@ -303,8 +309,9 @@ const viewOf = (entry, now) => { attempt: entry.attempt, maxAttempts: entry.maxAttempts, availableAt: entry.availableAt, - data: stored(entry.data), }; + // Dropped once the job is done with, by `retainData`. + if (entry.data !== undefined) view.data = stored(entry.data); if (entry.failure !== undefined) view.failure = entry.failure; if (entry.result !== undefined) view.result = entry.result; return view; @@ -315,7 +322,7 @@ const viewOf = (entry, now) => { * * @param options `now`, `sleep`, `leaseMilliseconds` (60000), * `pollIntervalMilliseconds` (1000), `maxJobs` (100000), `retainFinished` - * (10000), `logger`. + * (10000), `retainData` ('failed'), `logger`. * @returns The driver, with `source`, `list()`, `get(id)`, `drain()` and * `wake()`. * @throws {TypeError} If an option is invalid. @@ -325,6 +332,7 @@ const createMemoryQueue = (options = {}) => { now: checkOptionalFunction(options.now, 'now') ?? Date.now, maxJobs: checkPositiveInteger(options.maxJobs ?? 100_000, 'maxJobs'), retainFinished: checkPositiveInteger(options.retainFinished ?? 10_000, 'retainFinished'), + retainData: checkRetainData(options.retainData), }; const source = new MemorySource(config); const driver = createPollingDriver(source, { ...options, now: config.now }); diff --git a/tests/helpers/bullmq/fake-bullmq.js b/tests/helpers/bullmq/fake-bullmq.js index a0ce9db..1b89230 100644 --- a/tests/helpers/bullmq/fake-bullmq.js +++ b/tests/helpers/bullmq/fake-bullmq.js @@ -80,6 +80,10 @@ const fakeJob = (extra = {}) => { moves, moveToDelayed: async (timestamp, token) => moves.push(['delayed', timestamp, token]), moveToWait: async (token) => moves.push(['wait', token]), + async updateData(data) { + moves.push(['data', data]); + this.data = data; + }, ...extra, }; }; diff --git a/tests/helpers/mongodb/fake-collection.js b/tests/helpers/mongodb/fake-collection.js index ade4bd4..a0232c6 100644 --- a/tests/helpers/mongodb/fake-collection.js +++ b/tests/helpers/mongodb/fake-collection.js @@ -1,6 +1,6 @@ // Not ported: a collection of MongoDB in memory for the tests of ./mongodb, // with what the stores call of it: filters of equality, `$lte` and `$or`; -// `$set`, `$setOnInsert`, `$inc` and `$max`; an upsert that meets an id held +// `$set`, `$setOnInsert`, `$unset`, `$inc` and `$max`; an upsert that meets an id held // fails as MongoDB does (E11000); sorts; and a change stream. const comparable = (value) => (value instanceof Date ? value.getTime() : value); @@ -57,6 +57,9 @@ const apply = (document, update, inserting) => { document[field] = clone(value); } } + for (const field of Object.keys(update.$unset ?? {})) { + delete document[field]; + } for (const [field, value] of Object.entries(update.$inc ?? {})) { document[field] = (document[field] ?? 0) + value; } diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 4d917ad..3abc444 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -32,6 +32,8 @@ declare const engine: TemplateEngine; const driver = createMemoryQueue({ maxJobs: 10, pollIntervalMilliseconds: 100 }); expectType(driver); expectType(driver.source); +createMemoryQueue({ retainData: 'none' }); +expectError(createMemoryQueue({ retainData: 'some' })); expectAssignable(driver); const queue = createMailQueue({ driver, diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index 5dd0cb6..6f6939a 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -402,3 +402,47 @@ test('BullMQ: the driver checks its options', () => { assert.throws(() => driver.consume(async () => {}, { concurrency: 0 }), /concurrency/); assert.strictEqual(typeof driver.createProcessor(async () => ({ type: 'ack' })), 'function'); }); + +test('BullMQ: a job done with keeps its version and hash, not its message, unless told', async (t) => { + const warnings = []; + const listener = (warning) => warnings.push(warning.code); + process.on('warning', listener); + t.after(() => process.off('warning', listener)); + const data = { v: 1, hash: 'h1', to: [{ address: 'ada@example.com' }] }; + const run = async (outcome, options) => { + const job = fakeJob({ data: { ...data } }); + let thrown; + const result = await createBullMQProcessor(async () => outcome, options)(job, 'tok').catch( + (error) => { + thrown = error; + }, + ); + return { job, result, thrown }; + }; + const acked = await run({ type: 'ack', result: { provider: 'p' } }); + assert.deepStrictEqual(acked.result, { provider: 'p' }); + assert.deepStrictEqual(acked.job.data, { v: 1, hash: 'h1' }); + const kept = await run({ type: 'ack' }, { retainData: 'all' }); + assert.deepStrictEqual(kept.job.data, data); + const failed = await run({ type: 'fail', code: 'x', reason: 'y' }); + assert.strictEqual(failed.thrown.name, 'UnrecoverableError'); + assert.deepStrictEqual(failed.job.data, data); + const none = await run({ type: 'fail', code: 'x', reason: 'y' }, { retainData: 'none' }); + assert.deepStrictEqual(none.job.data, { v: 1, hash: 'h1' }); + // A store that fails to drop it leaves the send acknowledged, and tells. + const job = fakeJob({ + data: { ...data }, + updateData: async () => { + throw new Error('READONLY'); + }, + }); + assert.strictEqual(await createBullMQProcessor(async () => ({ type: 'ack' }))(job, 'tok'), null); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(warnings.includes('MAILER_QUEUE_RETAIN_FAILED')); + assert.throws(() => createBullMQProcessor(async () => ({}), { retainData: true }), /retainData/); + const bullmq = fakeBullMQ(); + assert.throws( + () => createBullMQDriver({ bullmq: { Queue: bullmq.Queue }, retainData: 'x' }), + /retainData/, + ); +}); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index ab0fac0..5a11589 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -429,3 +429,27 @@ test('MongoDB: a lease is taken back only past the skew of the workers’ clocks /more than 3 times clockSkewToleranceMilliseconds/, ); }); + +test('MongoDB: a job done with keeps no data, unless told; its duplicates are still told', async () => { + const cases = [ + [undefined, [false, true]], + ['all', [true, true]], + ['none', [false, false]], + ]; + for (const [retainData, [completedKeeps, failedKeeps]] of cases) { + const { collection, source, outbox } = outboxOf({ extra: { retainData } }); + await source.enqueue(requestOf('done')); + await source.enqueue(requestOf('refused')); + for (let index = 0; index < 2; index++) { + const { lease } = await source.reserve(1000); + if (lease.job.id === 'done') await source.ack(lease, { provider: 'p' }); + else await source.fail(lease, { code: 'x', reason: 'y' }); + } + assert.strictEqual('data' in collection.documents.get('done'), completedKeeps); + assert.strictEqual('data' in collection.documents.get('refused'), failedKeeps); + assert.strictEqual((await outbox.get('done')).data !== undefined, completedKeeps); + const again = await source.enqueue(requestOf('done')); + assert.deepStrictEqual([again.duplicate, typeof again.existingHash], [true, 'string']); + } + assert.throws(() => outboxOf({ extra: { retainData: 1 } }), /retainData must be/); +}); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index d0432fb..177f188 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -508,3 +508,29 @@ test('Queue: a run whose lease cannot be extended stops before the lease runs ou assert.ok(records.includes('queue.lease.extend_failed')); assert.ok(records.includes('queue.lease.lost')); }); + +test('Queue: a job done with keeps no data, unless told; its duplicates are still told', async () => { + const cases = [ + [undefined, [false, true]], + ['all', [true, true]], + ['none', [false, false]], + ]; + for (const [retainData, [completedKeeps, failedKeeps]] of cases) { + const queue = createMemoryQueue({ retainData }); + await queue.enqueue(requestOf('done')); + await queue.enqueue(requestOf('refused')); + for (let index = 0; index < 2; index++) { + const { lease } = await queue.source.reserve(1000); + if (lease.job.id === 'done') await queue.source.ack(lease, null); + else await queue.source.fail(lease, { code: 'x', reason: 'y' }); + } + assert.strictEqual('data' in queue.get('done'), completedKeeps, `${retainData} done`); + assert.strictEqual('data' in queue.get('refused'), failedKeeps, `${retainData} refused`); + assert.deepStrictEqual(await queue.enqueue(requestOf('done')), { + id: 'done', + duplicate: true, + existingHash: 'h-done', + }); + } + assert.throws(() => createMemoryQueue({ retainData: 'some' }), /retainData must be/); +}); From 02a939a8cc3769423c8e9b53f8fa4d40e19d0fbf Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:26:39 +0300 Subject: [PATCH 17/83] feat(queue): one retention of finished jobs on every driver The window in which an enqueue again is a duplicate was a day on BullMQ, a week in the outbox and 10000 jobs in memory, so the promise of "once per key" changed with the driver. dedupeWindowSeconds (a day) keeps a completed job and its key, and retainFailedSeconds (a week) a failed one, with the same names and defaults on the memory queue, the outbox (retainFinishedSeconds is gone) and BullMQ (removeOnComplete and removeOnFail, which jobOptions may still set). The outbox drops a job by a TTL on expireAt rather than finishedAt; the memory queue keeps its count bound, the oldest finished dropped first. The checks the queue and its drivers share move to src/internal/queue-options.js, so that the entries with no queue (./webhooks, ./azure, ./audit) carry none of them. ./queue goes from 16.8 to 17.2 KB min+gzip and ./mongodb from 5.5 to 5.6: their budgets go from 17 to 17.5 and from 5.5 to 6 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/guide/outbox.md | 8 +-- docs/guide/queue.md | 6 ++- docs/transports/bullmq.md | 2 +- scripts/size-budgets.json | 4 +- src/bullmq/bullmq-driver.js | 18 +++---- src/bullmq/index.d.ts | 4 ++ src/bullmq/processor.js | 2 +- src/internal/options.js | 36 +------------- src/internal/queue-options.js | 58 ++++++++++++++++++++++ src/mongodb/index.d.ts | 8 +-- src/mongodb/outbox.js | 37 +++++++------- src/queue/handler.js | 7 +-- src/queue/index.d.ts | 6 ++- src/queue/memory-queue.js | 87 ++++++++++++++++++++++++++------- tests/types/mongodb.test-d.ts | 2 +- tests/unit/bullmq.test.js | 26 ++++++++++ tests/unit/mongodb.test.js | 34 +++++++++++-- tests/unit/queue-memory.test.js | 35 +++++++++++++ 19 files changed, 280 insertions(+), 103 deletions(-) create mode 100644 src/internal/queue-options.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f20f05..0823b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. that a worker whose clock is behind does not run a job beside its holder. Every driver drops the message of a job done with (`retainData`: `'failed'` keeps the failed ones, `'all'`, `'none'`), keeping its version and hash, so a duplicate is still told and a reset link does not wait in the - store. + store. Every driver keeps a completed job `dedupeWindowSeconds` (a day, its window of duplicates) + and a failed one `retainFailedSeconds` (a week); the outbox drops them by a TTL on `expireAt`. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index ee891ee..0afbc1e 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -88,13 +88,14 @@ is only the fallback. | Collection | Index | | --- | --- | -| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; a TTL on `finishedAt` | +| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; a TTL on `expireAt` | | audit | `{ correlationId, at, attempt, rank }`; `{ jobId }` | | idempotency | a TTL on `expiresAt` | -A finished job is kept `retainFinishedSeconds` (a week), and while it is kept, +A completed job is kept `dedupeWindowSeconds` (a day), and while it is kept, an enqueue under its key is a duplicate: the retention is the window of -deduplication. A key of the store goes when its time passes. A completed job +deduplication. A failed job is kept `retainFailedSeconds` (a week), for a retry +by hand. Both are the same options, with the same defaults, on every driver. A key of the store goes when its time passes. A completed job keeps no `data`, only its hash (`retainData: 'failed'`, the default; `'all'` or `'none'`), so a reset link in it does not wait a week in the collection and its backups. @@ -113,6 +114,7 @@ its backups. failure: { code, reason }, // of the last failed attempt result: { provider, providerMessageId }, finishedAt: ISODate(…), + expireAt: ISODate(…), // when the TTL drops it: dedupeWindowSeconds or retainFailedSeconds on } ``` diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 105745f..5e6839b 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -80,8 +80,10 @@ not the worker: same message; - a job is 256 KiB at most. -Once a job is done with, the store drops its message (`retainData`, on the -driver): `'failed'` by default keeps the data of the failed jobs only, for a +A completed job is kept `dedupeWindowSeconds` (a day), the window in which an +enqueue under its key is a duplicate, and a failed one `retainFailedSeconds` (a +week), on every driver. Once a job is done with, the store drops its message +(`retainData`, on the driver): `'failed'` by default keeps the data of the failed jobs only, for a retry by hand; `'all'` keeps every one, `'none'` none. The version and the hash of a job stay, so an enqueue again is still told a duplicate. diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index 0f591c6..fdd2d39 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -29,7 +29,7 @@ A process that only enqueues passes `{ Queue }`, or a queue of its own | `backoff` | `{ type: 'alexify-mailer' }`: the delay the queue chose | | `priority` | `high` 1, `normal` 2, `low` 3 (`priorities`); never 0, which BullMQ runs before every prioritized job | | `delay` | until `scheduledAt` | -| `removeOnComplete`, `removeOnFail` | a day and a week, by time (`jobOptions`) | +| `removeOnComplete`, `removeOnFail` | `dedupeWindowSeconds` (a day) and `retainFailedSeconds` (a week), by time, as on every driver (`jobOptions` wins) | A duplicate is told by reading the job back (`detectDuplicates`, one read per enqueue; `'bulk'` for `enqueueMany` too). It holds while BullMQ keeps the diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 58b5952..73cbfd1 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 17, + "./queue": 17.5, "./bullmq": 4, - "./mongodb": 5.5 + "./mongodb": 6 } diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index 43fc9bd..027e011 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -5,7 +5,8 @@ // 1, as 0 in BullMQ runs before every prioritized job. const { createLog } = require('../internal/log.js'); -const { checkPositiveInteger, checkRetainData } = require('../internal/options.js'); +const { checkPositiveInteger } = require('../internal/options.js'); +const { checkRetainData, retentionOf } = require('../internal/queue-options.js'); const { settledWithin } = require('../internal/settle.js'); const { MAILER_BACKOFF_TYPE, @@ -31,12 +32,6 @@ const RESERVED_WORKER_OPTIONS = new Set(['connection', 'prefix', 'concurrency', // BullMQ's highest priority number, 2^21 - 1. const MAX_PRIORITY = 2_097_151; const DEFAULT_PRIORITIES = Object.freeze({ high: 1, normal: 2, low: 3 }); -const DEFAULT_JOB_OPTIONS = Object.freeze({ - // Kept by time, not by count: the window of duplicates does not shrink - // under load. - removeOnComplete: { age: 86_400 }, - removeOnFail: { age: 604_800 }, -}); const checkReserved = (options, reserved, what) => { if (options === undefined) return {}; @@ -91,11 +86,15 @@ const createDriverConfig = (options) => { priorities: checkPriorities(options.priorities), detectDuplicates, retainData: checkRetainData(options.retainData), + // Kept by time, not by count: the window of duplicates does not shrink + // under load. + retention: retentionOf(options), }; }; const optionsOf = (config, request) => ({ - ...DEFAULT_JOB_OPTIONS, + removeOnComplete: { age: config.retention.dedupeWindowSeconds }, + removeOnFail: { age: config.retention.retainFailedSeconds }, ...config.jobOptions, jobId: request.id, timestamp: request.enqueuedAt, @@ -193,7 +192,8 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { * `removeOnComplete`; `workerOptions`, such as `limiter` or * `lockDuration`; `priorities` (`{ high: 1, normal: 2, low: 3 }`); * `detectDuplicates` (true); `retainData` ('failed'), the finished jobs whose - * data is kept; `logger`. + * data is kept; `dedupeWindowSeconds` (a day) and `retainFailedSeconds` (a + * week), BullMQ's `removeOnComplete` and `removeOnFail`; `logger`. * @returns The driver, with `queue` and `createProcessor(handler)`. * @throws {TypeError} If an option is invalid. */ diff --git a/src/bullmq/index.d.ts b/src/bullmq/index.d.ts index 19f8158..149b170 100644 --- a/src/bullmq/index.d.ts +++ b/src/bullmq/index.d.ts @@ -127,6 +127,10 @@ export interface BullMQDriverOptions { return value; }; -/** - * Checks a whole number of zero or more. - * - * @internal - */ -const checkNonNegativeInteger = (value, name) => { - if (!Number.isInteger(value) || value < 0) { - throw new TypeError(`${name} must be a whole number of zero or more: ${String(value)}`); - } - return value; -}; - -// What a queue keeps of the data of a finished job: all of it, that of the -// failed ones (for a retry by hand), or none. -const RETAIN_DATA = new Set(['all', 'failed', 'none']); - -/** - * Checks `retainData` of a queue, `'failed'` by default. - * - * @internal - */ -const checkRetainData = (value = 'failed') => { - if (!RETAIN_DATA.has(value)) { - throw new TypeError(`retainData must be 'all', 'failed' or 'none': ${String(value)}`); - } - return value; -}; - -module.exports = { - checkNonNegativeInteger, - checkRetainData, - checkOptionalFunction, - checkPositiveInteger, - checkPositiveNumber, -}; +module.exports = { checkOptionalFunction, checkPositiveInteger, checkPositiveNumber }; diff --git a/src/internal/queue-options.js b/src/internal/queue-options.js new file mode 100644 index 0000000..11a7ced --- /dev/null +++ b/src/internal/queue-options.js @@ -0,0 +1,58 @@ +// Not ported: the checks of the options the queue and its drivers share +// (./queue, ./bullmq, ./mongodb), apart from those of the decorators so that +// an entry that has no queue carries none of them. + +const { checkPositiveInteger } = require('./options.js'); + +/** + * Checks a whole number of zero or more. + * + * @internal + */ +const checkNonNegativeInteger = (value, name) => { + if (!Number.isInteger(value) || value < 0) { + throw new TypeError(`${name} must be a whole number of zero or more: ${String(value)}`); + } + return value; +}; + +// What a queue keeps of the data of a finished job: all of it, that of the +// failed ones (for a retry by hand), or none. +const RETAIN_DATA = new Set(['all', 'failed', 'none']); + +/** + * Checks `retainData` of a queue, `'failed'` by default. + * + * @internal + */ +const checkRetainData = (value = 'failed') => { + if (!RETAIN_DATA.has(value)) { + throw new TypeError(`retainData must be 'all', 'failed' or 'none': ${String(value)}`); + } + return value; +}; + +// How long a store keeps a job done with: a completed one is its key's +// memory, the window in which an enqueue again is a duplicate; a failed one +// waits for a retry by hand. The same for every driver. +const DEFAULT_DEDUPE_WINDOW_SECONDS = 24 * 60 * 60; +const DEFAULT_RETAIN_FAILED_SECONDS = 7 * 24 * 60 * 60; + +/** + * Checks the retention of a queue: `dedupeWindowSeconds` (a day) and + * `retainFailedSeconds` (a week). + * + * @internal + */ +const retentionOf = (options) => ({ + dedupeWindowSeconds: checkPositiveInteger( + options.dedupeWindowSeconds ?? DEFAULT_DEDUPE_WINDOW_SECONDS, + 'dedupeWindowSeconds', + ), + retainFailedSeconds: checkPositiveInteger( + options.retainFailedSeconds ?? DEFAULT_RETAIN_FAILED_SECONDS, + 'retainFailedSeconds', + ), +}); + +module.exports = { checkNonNegativeInteger, checkRetainData, retentionOf }; diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index 738c297..1ebf6ac 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -58,8 +58,10 @@ export interface MongoOutboxOptions extends PollingDriverOptions { * longer. @default 5000 */ readonly clockSkewToleranceMilliseconds?: number; - /** The time a finished job is kept, and so the window of its duplicates. @default 604800 */ - readonly retainFinishedSeconds?: number; + /** The time a completed job is kept, and so the window of its duplicates. @default 86400 */ + readonly dedupeWindowSeconds?: number; + /** The time a failed job is kept, for a retry by hand. @default 604800 */ + readonly retainFailedSeconds?: number; /** The finished jobs whose data is kept: the others keep their version and hash. @default 'failed' */ readonly retainData?: 'all' | 'failed' | 'none'; readonly logger?: Logger; @@ -83,7 +85,7 @@ export interface MongoOutboxJob { /** The mail queue in a collection: a driver, its source, and what it holds. */ export interface MongoOutbox extends QueueDriver { readonly source: QueueSource; - /** Makes the indexes of the outbox: the ready jobs, the leases, and the expiry of finished ones. */ + /** Makes the indexes of the outbox: the ready jobs, the leases, and the expiry of finished ones (`expireAt`). */ ensureIndexes(): Promise; get(id: string): Promise; } diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 86034fe..45c8b17 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -9,18 +9,17 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { createLog } = require('../internal/log.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { checkNonNegativeInteger, - checkOptionalFunction, - checkPositiveInteger, checkRetainData, -} = require('../internal/options.js'); + retentionOf, +} = require('../internal/queue-options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); const { checkCollection, documentOf, sessionOf } = require('./collection.js'); // The document of a pause of the whole queue: no job id has a colon. const PAUSE_ID = 'mailer:pause'; -const DEFAULT_RETAIN_SECONDS = 7 * 24 * 60 * 60; // How far the clocks of two workers may be apart: a lease is taken back only // that long after it ran out by the clock of the worker that takes it. const DEFAULT_CLOCK_SKEW_MS = 5000; @@ -179,8 +178,11 @@ class MongoOutboxSource { } ack(lease, result) { - const finishedAt = new Date(this.#config.now()); - const set = { state: 'completed', result: result ?? null, finishedAt }; + const now = this.#config.now(); + const finishedAt = new Date(now); + // Kept, and its key remembered, for the window of duplicates. + const expireAt = new Date(now + this.#config.dedupeWindowSeconds * 1000); + const set = { state: 'completed', result: result ?? null, finishedAt, expireAt }; return this.#settle(lease, set, undefined, this.#config.retainData !== 'all'); } @@ -189,8 +191,10 @@ class MongoOutboxSource { } fail(lease, failure) { - const finishedAt = new Date(this.#config.now()); - const set = { state: 'failed', failure, finishedAt }; + const now = this.#config.now(); + const finishedAt = new Date(now); + const expireAt = new Date(now + this.#config.retainFailedSeconds * 1000); + const set = { state: 'failed', failure, finishedAt, expireAt }; return this.#settle(lease, set, undefined, this.#config.retainData === 'none'); } @@ -230,8 +234,9 @@ const subscribe = (collection, listener, config) => { * @param options `collection`, `db.collection('mail_outbox')`; * `changeStream` (false), to wake the workers on a change (a replica set); * `retainData` ('failed'), the finished jobs whose data is kept; - * `retainFinishedSeconds` (7 days), the time a finished job is kept, and so - * the window of its duplicates; `clockSkewToleranceMilliseconds` (5000), how + * `dedupeWindowSeconds` (a day), the time a completed job is kept, and so + * the window of its duplicates; `retainFailedSeconds` (a week), that of a + * failed one; `clockSkewToleranceMilliseconds` (5000), how * far the clocks of the workers may be apart; `now`, `logger`, and the * options of the polling driver. * @returns The driver, with `source`, `ensureIndexes()` and `get(id)`. @@ -259,10 +264,7 @@ const createMongoOutbox = (options = {}) => { retainData: checkRetainData(options.retainData), clockSkewToleranceMilliseconds, changeStream, - retainFinishedSeconds: checkPositiveInteger( - options.retainFinishedSeconds ?? DEFAULT_RETAIN_SECONDS, - 'retainFinishedSeconds', - ), + ...retentionOf(options), log: createLog(options.logger, { component: 'queue', queue: 'mongodb' }), }; const source = new MongoOutboxSource(collection, config); @@ -276,10 +278,11 @@ const createMongoOutbox = (options = {}) => { { name: 'mailer_ready' }, ); await collection.createIndex({ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }); - // A finished job goes after its time, and with it the memory of its key. + // A finished job goes at its time, a completed one and with it the + // memory of its key after the window of duplicates, a failed one later. await collection.createIndex( - { finishedAt: 1 }, - { name: 'mailer_finished', expireAfterSeconds: config.retainFinishedSeconds }, + { expireAt: 1 }, + { name: 'mailer_expire', expireAfterSeconds: 0 }, ); }, async get(id) { diff --git a/src/queue/handler.js b/src/queue/handler.js index 2fff148..55b8f92 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -14,11 +14,8 @@ const { messageOf } = require('../internal/decorator.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { fingerprintOf } = require('../internal/fingerprint.js'); const { createLog } = require('../internal/log.js'); -const { - checkNonNegativeInteger, - checkOptionalFunction, - checkPositiveInteger, -} = require('../internal/options.js'); +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { checkNonNegativeInteger } = require('../internal/queue-options.js'); const { isCallerAbort } = require('../internal/sleep.js'); const { isProgrammerError, receiptOfThrown } = require('../internal/thrown.js'); const { toUuid } = require('../internal/uuid.js'); diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 366e0e8..d94592f 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -471,8 +471,12 @@ export interface MemoryQueueJob { export interface MemoryQueueOptions extends PollingDriverOptions { /** @default 100000 */ readonly maxJobs?: number; - /** The jobs finished it keeps, and so the window of its duplicates. @default 10000 */ + /** The most jobs finished it keeps, the oldest dropped first. @default 10000 */ readonly retainFinished?: number; + /** The time a completed job is kept, and so the window of its duplicates. @default 86400 */ + readonly dedupeWindowSeconds?: number; + /** The time a failed job is kept. @default 604800 */ + readonly retainFailedSeconds?: number; /** See {@link MailQueueRetainData}. @default 'failed' */ readonly retainData?: MailQueueRetainData; } diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 45044d1..53792dc 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -8,11 +8,8 @@ const { randomUUID } = require('node:crypto'); const { Heap } = require('../internal/heap.js'); const { createPollingDriver } = require('../internal/polling-driver.js'); -const { - checkOptionalFunction, - checkPositiveInteger, - checkRetainData, -} = require('../internal/options.js'); +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +const { checkRetainData, retentionOf } = require('../internal/queue-options.js'); const { isJobRequest, rankOf } = require('../internal/priority.js'); // The data of a job as a store gives it back: a copy, never the object @@ -44,6 +41,20 @@ const nodeOf = (entry) => ({ const isCurrent = (node, state) => node.version === node.entry.version && node.entry.state === state; +// A list of the ids of finished jobs, the oldest at its head, taken from the +// head without shifting the array. +const finishedList = () => ({ ids: [], head: 0 }); +const sizeOf = (list) => list.ids.length - list.head; +const headOf = (list) => (list.head < list.ids.length ? list.ids[list.head] : undefined); +const dropHead = (list) => { + list.ids[list.head] = undefined; + list.head += 1; + if (list.head > 1024 && list.head * 2 > list.ids.length) { + list.ids = list.ids.slice(list.head); + list.head = 0; + } +}; + class MemorySource { id = 'memory'; jobs = new Map(); @@ -54,12 +65,14 @@ class MemorySource { #leases = new Heap(leaseBefore); #open = 0; #active = 0; - // The ids of the jobs finished, the oldest at #finishedHead. - #finished = []; - #finishedHead = 0; + // The jobs finished, kept for different times: a completed one for the + // window of its duplicates, a failed one for a retry by hand. + #completed = finishedList(); + #failed = finishedList(); #listeners = new Set(); #waiters = new Set(); #seq = 0; + #finishedSeq = 0; #pausedUntil = 0; #config; @@ -81,22 +94,54 @@ class MemorySource { } #finish(entry, state) { + const now = this.#config.now(); entry.state = state; entry.token = undefined; entry.version += 1; this.#active -= 1; this.#open -= 1; - this.#finished.push(entry.id); - while (this.#finished.length - this.#finishedHead > this.#config.retainFinished) { - this.jobs.delete(this.#finished[this.#finishedHead]); - this.#finished[this.#finishedHead] = undefined; - this.#finishedHead += 1; + entry.finishedSeq = this.#finishedSeq++; + const completed = state === 'completed'; + const seconds = completed ? this.#config.dedupeWindowSeconds : this.#config.retainFailedSeconds; + entry.expireAt = now + seconds * 1000; + (completed ? this.#completed : this.#failed).ids.push(entry.id); + this.#sweep(now); + this.#notify(); + } + + // Drops the finished jobs whose time is up, then the oldest beyond the + // count kept, the memory of their keys with them. + #sweep(now) { + this.#expireList(this.#completed, now); + this.#expireList(this.#failed, now); + while (sizeOf(this.#completed) + sizeOf(this.#failed) > this.#config.retainFinished) { + const completed = this.jobs.get(headOf(this.#completed)); + const failed = this.jobs.get(headOf(this.#failed)); + const older = + failed === undefined || + (completed !== undefined && completed.finishedSeq < failed.finishedSeq) + ? this.#completed + : this.#failed; + this.jobs.delete(headOf(older)); + dropHead(older); } - if (this.#finishedHead > 1024 && this.#finishedHead * 2 > this.#finished.length) { - this.#finished = this.#finished.slice(this.#finishedHead); - this.#finishedHead = 0; + } + + // Drops the finished jobs whose time is up: what the queue holds is read + // only after. + prune() { + this.#sweep(this.#config.now()); + } + + #expireList(list, now) { + for (;;) { + const id = headOf(list); + if (id === undefined) return; + const entry = this.jobs.get(id); + if (entry !== undefined && entry.expireAt > now) return; + this.jobs.delete(id); + dropHead(list); } - this.#notify(); } #holds(lease) { @@ -152,6 +197,7 @@ class MemorySource { async enqueue(request) { if (!isJobRequest(request)) throw new TypeError('The memory queue takes a job request.'); + this.#sweep(this.#config.now()); const held = this.jobs.get(request.id); if (held !== undefined) return { id: request.id, duplicate: true, existingHash: held.hash }; if (this.#open >= this.#config.maxJobs) { @@ -185,6 +231,7 @@ class MemorySource { async reserve(leaseMilliseconds) { const now = this.#config.now(); + this.#sweep(now); this.#expire(now); if (this.#pausedUntil > now) return { nextAvailableAt: this.#pausedUntil }; this.#promote(now); @@ -322,7 +369,8 @@ const viewOf = (entry, now) => { * * @param options `now`, `sleep`, `leaseMilliseconds` (60000), * `pollIntervalMilliseconds` (1000), `maxJobs` (100000), `retainFinished` - * (10000), `retainData` ('failed'), `logger`. + * (10000), `dedupeWindowSeconds` (a day), `retainFailedSeconds` (a week), + * `retainData` ('failed'), `logger`. * @returns The driver, with `source`, `list()`, `get(id)`, `drain()` and * `wake()`. * @throws {TypeError} If an option is invalid. @@ -333,6 +381,7 @@ const createMemoryQueue = (options = {}) => { maxJobs: checkPositiveInteger(options.maxJobs ?? 100_000, 'maxJobs'), retainFinished: checkPositiveInteger(options.retainFinished ?? 10_000, 'retainFinished'), retainData: checkRetainData(options.retainData), + ...retentionOf(options), }; const source = new MemorySource(config); const driver = createPollingDriver(source, { ...options, now: config.now }); @@ -340,12 +389,14 @@ const createMemoryQueue = (options = {}) => { ...driver, source, list() { + source.prune(); const views = []; const now = config.now(); for (const entry of source.jobs.values()) views.push(viewOf(entry, now)); return views; }, get(id) { + source.prune(); const entry = source.jobs.get(id); return entry === undefined ? undefined : viewOf(entry, config.now()); }, diff --git a/tests/types/mongodb.test-d.ts b/tests/types/mongodb.test-d.ts index 513c366..8763895 100644 --- a/tests/types/mongodb.test-d.ts +++ b/tests/types/mongodb.test-d.ts @@ -24,7 +24,7 @@ expectAssignable(typed); const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), changeStream: true, - retainFinishedSeconds: 3600, + dedupeWindowSeconds: 3600, leaseMilliseconds: 30_000, pollIntervalMilliseconds: 500, }); diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index 6f6939a..ac2ef97 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -446,3 +446,29 @@ test('BullMQ: a job done with keeps its version and hash, not its message, unles /retainData/, ); }); + +test('BullMQ: the retention of jobs is the queue’s, as on every driver', async () => { + const bullmq = fakeBullMQ(); + const driver = createBullMQDriver({ bullmq }); + await driver.enqueue(requestOf()); + const [queue] = bullmq.queues; + assert.deepStrictEqual( + [queue.added[0].opts.removeOnComplete, queue.added[0].opts.removeOnFail], + [{ age: 86_400 }, { age: 604_800 }], + ); + const short = fakeBullMQ(); + await createBullMQDriver({ + bullmq: short, + dedupeWindowSeconds: 3600, + retainFailedSeconds: 7200, + }).enqueue(requestOf()); + const { opts } = short.queues[0].added[0]; + assert.deepStrictEqual( + [opts.removeOnComplete, opts.removeOnFail], + [{ age: 3600 }, { age: 7200 }], + ); + assert.throws( + () => createBullMQDriver({ bullmq, dedupeWindowSeconds: -1 }), + /dedupeWindowSeconds/, + ); +}); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 5a11589..17041b0 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -177,7 +177,6 @@ test('MongoDB: the outbox makes its indexes, and wakes its workers on a change s const { collection, outbox } = outboxOf({ extra: { changeStream: true, - retainFinishedSeconds: 3600, logger: { warn: (record, message) => records.push([message, record.error]) }, }, }); @@ -185,7 +184,7 @@ test('MongoDB: the outbox makes its indexes, and wakes its workers on a change s assert.deepStrictEqual(collection.indexes, [ [{ state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, { name: 'mailer_ready' }], [{ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }], - [{ finishedAt: 1 }, { name: 'mailer_finished', expireAfterSeconds: 3600 }], + [{ expireAt: 1 }, { name: 'mailer_expire', expireAfterSeconds: 0 }], ]); let woken = 0; const unsubscribe = outbox.source.subscribe(() => woken++); @@ -225,8 +224,12 @@ test('MongoDB: the outbox checks its options', () => { /changeStream needs a collection with watch/, ); assert.throws( - () => createMongoOutbox({ collection, retainFinishedSeconds: 0 }), - /retainFinishedSeconds/, + () => createMongoOutbox({ collection, dedupeWindowSeconds: 0 }), + /dedupeWindowSeconds/, + ); + assert.throws( + () => createMongoOutbox({ collection, retainFailedSeconds: 1.5 }), + /retainFailedSeconds/, ); assert.throws(() => createMongoOutbox({ collection, now: 1 }), /now/); // Without a change stream, there is nothing to subscribe to. @@ -453,3 +456,26 @@ test('MongoDB: a job done with keeps no data, unless told; its duplicates are st } assert.throws(() => outboxOf({ extra: { retainData: 1 } }), /retainData must be/); }); + +test('MongoDB: a completed job goes after the window of duplicates, a failed one later', async () => { + const { collection, source, now } = outboxOf({ + extra: { dedupeWindowSeconds: 3600, retainFailedSeconds: 7200 }, + }); + await source.enqueue(requestOf('done')); + await source.enqueue(requestOf('refused')); + for (let index = 0; index < 2; index++) { + const { lease } = await source.reserve(1000); + if (lease.job.id === 'done') await source.ack(lease, null); + else await source.fail(lease, { code: 'x', reason: 'y' }); + } + assert.strictEqual(collection.documents.get('done').expireAt.getTime(), now() + 3_600_000); + assert.strictEqual(collection.documents.get('refused').expireAt.getTime(), now() + 7_200_000); + // By default, a day and a week, whatever the driver. + const defaults = outboxOf(); + await defaults.source.enqueue(requestOf('a')); + await defaults.source.ack((await defaults.source.reserve(1000)).lease, null); + assert.strictEqual( + defaults.collection.documents.get('a').expireAt.getTime(), + defaults.now() + 86_400_000, + ); +}); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 177f188..1c48ef9 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -534,3 +534,38 @@ test('Queue: a job done with keeps no data, unless told; its duplicates are stil } assert.throws(() => createMemoryQueue({ retainData: 'some' }), /retainData must be/); }); + +test('Queue: the memory queue keeps a completed job for the window of duplicates, a failed one longer', async () => { + const clock = manualClock(0); + const queue = createMemoryQueue({ + now: clock.now, + dedupeWindowSeconds: 60, + retainFailedSeconds: 120, + }); + await queue.enqueue(requestOf('done')); + await queue.enqueue(requestOf('refused')); + for (let index = 0; index < 2; index++) { + const { lease } = await queue.source.reserve(1000); + if (lease.job.id === 'done') await queue.source.ack(lease, null); + else await queue.source.fail(lease, { code: 'x', reason: 'y' }); + } + clock.advance(59_999); + assert.strictEqual((await queue.enqueue(requestOf('done'))).duplicate, true); + clock.advance(1); + // Past its window, the key is forgotten: the same id is a new job. + assert.strictEqual((await queue.enqueue(requestOf('done'))).duplicate, false); + assert.strictEqual(queue.get('refused').state, 'failed'); + clock.advance(60_000); + assert.strictEqual(queue.get('refused'), undefined); + // The count bound drops the oldest finished first, of either kind. + const small = createMemoryQueue({ now: clock.now, retainFinished: 1 }); + await small.enqueue(requestOf('a')); + await small.enqueue(requestOf('b')); + await small.source.fail((await small.source.reserve(1000)).lease, { code: 'x', reason: 'y' }); + await small.source.ack((await small.source.reserve(1000)).lease, null); + assert.deepStrictEqual( + small.list().map((job) => job.id), + ['b'], + ); + assert.throws(() => createMemoryQueue({ dedupeWindowSeconds: 0 }), /dedupeWindowSeconds/); +}); From 31acd2728aa563f5c9659a67f42c73a8061a2822 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:28:19 +0300 Subject: [PATCH 18/83] fix(queue): refuse a reused key of enqueueMany once the batch is told enqueueMany threw queue.key_reused at the first job whose key was used before for another message, after the driver had enqueued every job, so the queued events of the jobs after it were never written and the caller could not tell which were enqueued. It tells every other job to the sink first, then refuses with the jobIds refused and the answers of the others; a single enqueue's refusal carries its jobId too. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++- docs/guide/queue.md | 5 ++++- src/queue/index.d.ts | 4 ++++ src/queue/mail-queue.js | 29 ++++++++++++++++++++++++----- tests/unit/queue.test.js | 23 +++++++++++++++++++++++ 5 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0823b74..ebc9696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. checks them, its context refused with the path to what JSON would change, attachments as bytes (bounded) or references the worker resolves. Its id is a UUID of the namespace, tenant and `idempotencyKey`, so the same message enqueued again is one job and another under the key is - refused (`queue.key_reused`); its Message-ID and Content-IDs are fixed for every attempt. The + refused (`queue.key_reused`, with the `jobIds` refused and, from `enqueueMany`, the `answers` of + the other jobs, all of them enqueued and audited first); its Message-ID and Content-IDs are fixed + for every attempt. The worker fails a permanent failure, a missing template or a bug at once, retries a temporary one never sooner than `Retry-After`, an operator's failure (`auth`, `configuration`) after five minutes, and never sends again a message of unknown delivery. `scheduledAt` is a delay of the diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 5e6839b..ffa2a92 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -156,7 +156,10 @@ can seal can still forge, so seal with `signingKeys` too. The id of a job is a UUID of its namespace, tenant and `idempotencyKey`: the same message enqueued again is the same job (`duplicate: true`), and another -message under the key is refused with `MailQueueError` `queue.key_reused`. +message under the key is refused with `MailQueueError` `queue.key_reused`, its +`jobIds` the jobs refused. `enqueueMany` refuses them only once every other +job of the call is enqueued and told to the audit, and gives what those gave as +the error's `answers`. The worker gives the message that id as its `idempotencyKey`, so [`IdempotencyTransport`](/transports/idempotency), with a store the workers share, and the providers that take a key (Resend, Lettermint, Brevo, Azure) diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index d94592f..1ce8b0d 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -267,6 +267,10 @@ export type MailQueueErrorCode = /** A failure of the queue, by its code. */ export declare class MailQueueError extends Error { readonly code: MailQueueErrorCode; + /** `queue.key_reused`: the jobs whose keys were used before for other messages. */ + readonly jobIds?: readonly string[]; + /** `queue.key_reused` of `enqueueMany`: what the other jobs of the call gave. */ + readonly answers?: readonly MailEnqueueResult[]; constructor(code: MailQueueErrorCode, message: string, options?: ErrorOptions); } diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 1b7254e..38eab28 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -38,6 +38,19 @@ const queuedEventOf = (data, request) => { return event; }; +// The refusal of keys used before for other messages, with what the other +// jobs of the call gave. +const keyReused = (jobIds, answers) => + Object.assign( + new MailQueueError( + 'queue.key_reused', + jobIds.length === 1 + ? `The key of job ${jobIds[0]} was used before for another message.` + : `The keys of ${jobIds.length} jobs were used before for other messages: ${jobIds.join(', ')}.`, + ), + { jobIds, answers }, + ); + // A first enqueue is told to the sink, in the transaction of the store when // there is one; a job enqueued again is not, and one enqueued again for // another message is refused. @@ -45,10 +58,7 @@ const afterEnqueue = async (queue, { data, request }, result, enqueueOptions) => const answer = { id: request.id, correlationId: data.correlationId, messageId: data.messageId }; if (result?.duplicate === true) { if (result.existingHash !== undefined && result.existingHash !== request.hash) { - throw new MailQueueError( - 'queue.key_reused', - `The key of job ${request.id} was used before for another message.`, - ); + throw keyReused([request.id], []); } queue.log.write('debug', () => ({ jobId: request.id }), 'queue.duplicate'); return { ...answer, duplicate: true }; @@ -151,10 +161,19 @@ const createMailQueue = (options = {}) => { results = []; for (const job of prepared) results.push(await driver.enqueue(job.request, enqueueOptions)); } + // Every job the driver took is told to the sink, a key used before for + // another message refused once all are. const answers = []; + const reused = []; for (let index = 0; index < prepared.length; index++) { - answers.push(await afterEnqueue(queue, prepared[index], results[index], enqueueOptions)); + try { + answers.push(await afterEnqueue(queue, prepared[index], results[index], enqueueOptions)); + } catch (error) { + if (error?.code !== 'queue.key_reused') throw error; + reused.push(prepared[index].request.id); + } } + if (reused.length > 0) throw keyReused(reused, answers); return answers; }, createWorker(workerOptions = {}) { diff --git a/tests/unit/queue.test.js b/tests/unit/queue.test.js index e5cdd9f..6800dd8 100644 --- a/tests/unit/queue.test.js +++ b/tests/unit/queue.test.js @@ -268,3 +268,26 @@ test('Queue: the queue logs its enqueues; it checks its options', async () => { const { id } = await scoped.enqueue(resetJob()); assert.strictEqual(driver.get(id).maxAttempts, 2); }); + +test('Queue: enqueueMany refuses a key used before only once every other job is told', async () => { + const { driver, audit, queue } = setup(); + await queue.enqueue(resetJob({ idempotencyKey: 'taken' })); + const batch = [ + resetJob({ idempotencyKey: 'first' }), + resetJob({ idempotencyKey: 'taken', context: { url: 'https://example.com/other' } }), + resetJob({ idempotencyKey: 'last' }), + ]; + const error = await queue.enqueueMany(batch).catch((thrown) => thrown); + assert.ok(error instanceof MailQueueError); + assert.strictEqual(error.code, 'queue.key_reused'); + assert.strictEqual(error.jobIds.length, 1); + assert.deepStrictEqual( + error.answers.map((answer) => answer.duplicate), + [false, false], + ); + // The jobs after the refused one were enqueued, and told to the sink. + for (const answer of error.answers) { + assert.strictEqual(driver.get(answer.id).state, 'waiting'); + assert.strictEqual(audit.stateOf(answer.correlationId), 'queued'); + } +}); From c2b8a69b521dc915f79bfda9610be095cfac5742 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:28:20 +0300 Subject: [PATCH 19/83] fix(templates): render a Handlebars partial that names itself A partial was loaded with the partials it names, and the promise of its load was the one returned for its name: a partial that named itself, or one that named it back, waited for itself, and the render never settled, the pending promise kept until clear(). A partial is now registered as soon as it is compiled, as Handlebars finds partials when it renders, and a template walks the partials it names, and theirs, each once. Co-Authored-By: Claude Opus 5.5 --- src/templates/handlebars.js | 48 ++++++++++++++++++---------- tests/unit/templates-engines.test.js | 30 +++++++++++++++++ 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/templates/handlebars.js b/src/templates/handlebars.js index ca801a0..1f66ec7 100644 --- a/src/templates/handlebars.js +++ b/src/templates/handlebars.js @@ -42,6 +42,15 @@ const visit = (node, named, inline) => { } }; +// The partials a template names but does not define inline. +const namesOf = (ast) => { + const named = new Set(); + const inline = new Set(); + visit(ast, named, inline); + for (const name of inline) named.delete(name); + return named; +}; + /** * Makes the Handlebars engine. * @@ -72,33 +81,40 @@ const createHandlebarsEngine = (options = {}) => { const env = handlebars.create(); for (const name of Object.keys(helpers)) env.registerHelper(name, helpers[name]); configure?.(env); - environment = { env, loading: new Map() }; + environment = { env, partials: new Map() }; environments.set(includes, environment); } return environment; }; - // Reads, compiles and registers a partial and the partials it names, once - // per environment however many templates name it at once. - const loadPartial = (environment, includes, name, from) => { - let loading = environment.loading.get(name); - if (loading === undefined) { - loading = (async () => { + // Reads, compiles and registers a partial once per environment however many + // templates name it at once, and gives the partials it names, which it does + // not wait for: a partial that names itself, or one that names it back, is + // registered all the same, as Handlebars finds partials when it renders. + const registerPartial = (environment, includes, name, from) => { + let entry = environment.partials.get(name); + if (entry === undefined) { + entry = (async () => { const found = await includes.lookup(name, from, extensions); const ast = environment.env.parse(found.source); - await loadPartials(environment, includes, ast, found.name); environment.env.registerPartial(name, environment.env.compile(ast, given)); + return { from: found.name, names: namesOf(ast) }; })(); - environment.loading.set(name, loading); - loading.catch(() => environment.loading.delete(name)); + environment.partials.set(name, entry); + entry.catch(() => environment.partials.delete(name)); } - return loading; + return entry; }; + // Registers the partials a template names, and those they name, each once. const loadPartials = async (environment, includes, ast, from) => { - const named = new Set(); - const inline = new Set(); - visit(ast, named, inline); - for (const name of named) { - if (!inline.has(name)) await loadPartial(environment, includes, name, from); + const pending = [{ from, names: namesOf(ast) }]; + const seen = new Set(); + for (let index = 0; index < pending.length; index++) { + const named = pending[index]; + for (const name of named.names) { + if (seen.has(name)) continue; + seen.add(name); + pending.push(await registerPartial(environment, includes, name, named.from)); + } } }; return { diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 22c7e17..6eba6c5 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -192,6 +192,36 @@ test('Templates: Handlebars fails on a missing or dynamic partial and on bad syn assert.throws(() => createHandlebarsEngine({ handlebars, configure: 1 }), /configure/); }); +test( + 'Templates: a Handlebars partial that names itself, or one that names it back, renders', + { timeout: 10_000 }, + async () => { + // Before: loading a partial waited for the partials it names, so one that + // named itself waited for itself, and the render never settled. + const engine = createHandlebarsEngine({ handlebars }); + const tree = { + 'tree.hbs': '
    {{#each items}}{{> node}}{{/each}}
', + 'node.hbs': + '
  • {{name}}{{#if children}}
      {{#each children}}{{> node}}{{/each}}
    {{/if}}
    • ', + }; + const items = [{ name: 'a', children: [{ name: 'b', children: [{ name: 'c' }] }] }]; + assert.strictEqual( + (await renderWith(engine, tree, { template: 'tree', context: { items } })).html, + '
    • a
      • b
        • c
    ', + ); + const mutual = { + 'count.hbs': '{{> even n=3}}', + 'even.hbs': 'e{{n}}{{#if n}}{{> odd n=(dec n)}}{{/if}}', + 'odd.hbs': 'o{{n}}{{#if n}}{{> even n=(dec n)}}{{/if}}', + }; + const counting = createHandlebarsEngine({ handlebars, helpers: { dec: (n) => n - 1 } }); + assert.strictEqual( + (await renderWith(counting, mutual, { template: 'count' })).html, + 'e3o2e1o0', + ); + }, +); + test('Templates: a partial that fails to load is read again on the next render', async () => { let fail = true; const loader = { From c8dd64869b5b40f78039c237e469318a21f73137 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:29:32 +0300 Subject: [PATCH 20/83] fix(templates): key a Handlebars partial by the template it resolves to Partials were registered by the name as written, so `./footer` written in a/x and in b/y was one partial, and b/y rendered the footer of a/x when a/x was compiled first. Each name a template writes is looked up from it, registered under the name of the template it resolves to, and renamed to it in the AST of the template that writes it. Co-Authored-By: Claude Opus 5.5 --- src/templates/handlebars.js | 65 +++++++++++++++++----------- tests/unit/templates-engines.test.js | 17 ++++++++ 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/templates/handlebars.js b/src/templates/handlebars.js index 1f66ec7..b1c0dc2 100644 --- a/src/templates/handlebars.js +++ b/src/templates/handlebars.js @@ -22,7 +22,8 @@ const partialNameOf = (name) => { return undefined; }; -// The partials a template names, and those it defines inline, from its AST. +// The statements of a template that name a partial, and the partials it +// defines inline, from its AST. const visit = (node, named, inline) => { if (Array.isArray(node)) { for (const child of node) visit(child, named, inline); @@ -31,7 +32,7 @@ const visit = (node, named, inline) => { if (typeof node !== 'object' || node === null) return; if (PARTIALS.has(node.type)) { const name = partialNameOf(node.name); - if (name !== undefined && name[0] !== '@') named.add(name); + if (name !== undefined && name[0] !== '@') named.push(node); } if (node.type === 'DecoratorBlock' && node.path?.original === 'inline') { const name = node.params?.[0]?.value; @@ -42,13 +43,27 @@ const visit = (node, named, inline) => { } }; -// The partials a template names but does not define inline. -const namesOf = (ast) => { - const named = new Set(); +// Looks up the partials a template names from the template `from`, and names +// each in its AST by the template it resolves to, so that `./footer` in two +// directories is two partials. +const resolvePartials = async (includes, ast, from, extensions) => { + const named = []; const inline = new Set(); visit(ast, named, inline); - for (const name of inline) named.delete(name); - return named; + const found = new Map(); + for (const node of named) { + const name = partialNameOf(node.name); + if (inline.has(name)) continue; + const source = await includes.lookup(name, from, extensions); + node.name = { + type: 'StringLiteral', + value: source.name, + original: source.name, + loc: node.name.loc, + }; + found.set(source.name, source); + } + return found; }; /** @@ -86,34 +101,34 @@ const createHandlebarsEngine = (options = {}) => { } return environment; }; - // Reads, compiles and registers a partial once per environment however many - // templates name it at once, and gives the partials it names, which it does - // not wait for: a partial that names itself, or one that names it back, is - // registered all the same, as Handlebars finds partials when it renders. - const registerPartial = (environment, includes, name, from) => { - let entry = environment.partials.get(name); + // Compiles and registers a partial once per environment however many + // templates name it at once, under the name of its template, and gives the + // partials it names, which it does not wait for: a partial that names + // itself, or one that names it back, is registered all the same, as + // Handlebars finds partials when it renders. + const registerPartial = (environment, includes, source) => { + let entry = environment.partials.get(source.name); if (entry === undefined) { entry = (async () => { - const found = await includes.lookup(name, from, extensions); - const ast = environment.env.parse(found.source); - environment.env.registerPartial(name, environment.env.compile(ast, given)); - return { from: found.name, names: namesOf(ast) }; + const ast = environment.env.parse(source.source); + const named = await resolvePartials(includes, ast, source.name, extensions); + environment.env.registerPartial(source.name, environment.env.compile(ast, given)); + return named; })(); - environment.partials.set(name, entry); - entry.catch(() => environment.partials.delete(name)); + environment.partials.set(source.name, entry); + entry.catch(() => environment.partials.delete(source.name)); } return entry; }; // Registers the partials a template names, and those they name, each once. const loadPartials = async (environment, includes, ast, from) => { - const pending = [{ from, names: namesOf(ast) }]; + const pending = [await resolvePartials(includes, ast, from, extensions)]; const seen = new Set(); for (let index = 0; index < pending.length; index++) { - const named = pending[index]; - for (const name of named.names) { - if (seen.has(name)) continue; - seen.add(name); - pending.push(await registerPartial(environment, includes, name, named.from)); + for (const source of pending[index].values()) { + if (seen.has(source.name)) continue; + seen.add(source.name); + pending.push(await registerPartial(environment, includes, source)); } } }; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 6eba6c5..88954ee 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -222,6 +222,23 @@ test( }, ); +test('Templates: a Handlebars partial is the template it resolves to, not the name written', async () => { + // Before: partials were registered by the name written, so `./footer` of + // b/y rendered the footer of a/x, whichever was compiled first. + const engine = createHandlebarsEngine({ handlebars }); + const renderer = createTemplateRenderer({ + engine, + loader: createMemoryLoader({ + 'a/x.hbs': '{{> ./footer}}', + 'a/footer.hbs': 'A', + 'b/y.hbs': '{{> ./footer}}{{> "./footer"}}', + 'b/footer.hbs': 'B', + }), + }); + assert.strictEqual((await renderer.render({ template: 'a/x' })).html, 'A'); + assert.strictEqual((await renderer.render({ template: 'b/y' })).html, 'BB'); +}); + test('Templates: a partial that fails to load is read again on the next render', async () => { let fail = true; const loader = { From a94ad6b308ff926ce07ed1b3500f0b584a7abf7e Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:29:55 +0300 Subject: [PATCH 21/83] fix(queue): tell the queued event before a job enqueued without a transaction With a driver that writes no transaction (BullMQ, the memory queue), the queued event was told after the job was enqueued, and a strict sink that failed made the enqueue reject though the job was in the queue: a caller that retried without a key enqueued the message twice. Without a transaction the event is told first (written once by its id), so a strict sink that fails leaves nothing enqueued. A driver declares transactional (the outbox does), and one that does not refuses a transaction with a TypeError rather than ignore it, which let the job go without the application's writes it was meant to go with. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- docs/guide/queue.md | 6 +++- index.d.ts | 6 ++++ src/mongodb/index.d.ts | 1 + src/mongodb/outbox.js | 2 ++ src/queue/mail-queue.js | 57 +++++++++++++++++++++++++---------- tests/types/mongodb.test-d.ts | 2 ++ tests/unit/queue.test.js | 35 +++++++++++++++++++++ 8 files changed, 95 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc9696..247e4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs O(log n) however many wait, its `source` held to the same contract as the outbox. A source's settling writes give `false` when the token of the lease no longer holds the job, and the driver - logs `queue.settle_fenced`; a run whose lease the store does not extend is aborted before the + logs `queue.settle_fenced`. Without a transaction, the `queued` event is told before the job is + enqueued, so a strict sink that fails refuses the enqueue with nothing enqueued; a driver that + writes no transaction refuses one (`transactional` drivers, the outbox, take it). A run whose lease the store does not extend is aborted before the lease runs out, not after another worker took the job. A source counts the runs of a job that end unsettled (`redeliveries`, never counted down, BullMQ's `stalledCounter`), and the worker gives up a job past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does diff --git a/docs/guide/queue.md b/docs/guide/queue.md index ffa2a92..e9d6845 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -215,7 +215,11 @@ enqueue (in the transaction of an outbox), `rendered`, `submitted` just before the send, then `accepted`, `failed` or `delivery_unknown`, and `queued` again for a retry. A sink that fails is a warning (`MAILER_AUDIT_FAILED`); `auditStrict: true` makes it retry the job before the send, so that no message -goes unrecorded. +goes unrecorded. Without a transaction, `queued` is told before the job is +enqueued, so a strict sink that fails refuses the enqueue and nothing is +enqueued, rather than a job enqueued and reported as failed. A driver that +writes no transaction (BullMQ, the memory queue) refuses `{ transaction }` with +a `TypeError` rather than ignore it; the outbox is `transactional`. `logger`, pino-like, hears what the queue does, never an address: the enqueues and the starts at `debug`, the sends at `info`, the retries, diff --git a/index.d.ts b/index.d.ts index 0806982..2d1bced 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1560,6 +1560,12 @@ export interface QueueConsumer { /** A queue: BullMQ, a store's outbox, the memory one of the tests. */ export interface QueueDriver { readonly id: string; + /** + * true: `enqueue` writes in the `transaction` of its options, as an outbox + * does. A driver without it refuses a transaction, which would not hold + * the job with the application's writes. + */ + readonly transactional?: boolean; enqueue(request: QueueJobRequest, options?: QueueEnqueueOptions): Promise; enqueueMany?( requests: readonly QueueJobRequest[], diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index 1ebf6ac..b1c6b33 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -84,6 +84,7 @@ export interface MongoOutboxJob { /** The mail queue in a collection: a driver, its source, and what it holds. */ export interface MongoOutbox extends QueueDriver { + readonly transactional: true; readonly source: QueueSource; /** Makes the indexes of the outbox: the ready jobs, the leases, and the expiry of finished ones (`expireAt`). */ ensureIndexes(): Promise; diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 45c8b17..a4db4e6 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -271,6 +271,8 @@ const createMongoOutbox = (options = {}) => { const driver = createPollingDriver(source, { ...options, now: config.now }); return { ...driver, + // An enqueue writes in the transaction it is given. + transactional: true, source, async ensureIndexes() { await collection.createIndex( diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 38eab28..05a792b 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -51,10 +51,38 @@ const keyReused = (jobIds, answers) => { jobIds, answers }, ); -// A first enqueue is told to the sink, in the transaction of the store when -// there is one; a job enqueued again is not, and one enqueued again for -// another message is refused. -const afterEnqueue = async (queue, { data, request }, result, enqueueOptions) => { +// Tells the sink of a first enqueue. Without a transaction it is told before +// the job is enqueued, so that a strict sink that fails refuses the enqueue +// rather than reports a job enqueued as failed, enqueued again on a retry; +// the event is written once by its id, a duplicate's with it. +const appendQueued = async (queue, { data, request }, transaction) => { + if (queue.audit === undefined) return; + try { + await queue.audit.append(queuedEventOf(data, request), { transaction }); + } catch (error) { + if (queue.auditStrict) throw error; + warnThrown('The audit sink of the mail queue', error, 'MAILER_AUDIT_FAILED'); + } +}; + +// The transaction of an enqueue, refused by a driver that writes none: the +// job would not go with the application's writes, as the caller believes. +const transactionOf = (driver, enqueueOptions) => { + const { transaction } = enqueueOptions; + if (transaction === undefined || transaction === null) return undefined; + if (driver.transactional !== true) { + throw new TypeError( + `The ${driver.id} driver writes no transaction: the job would not go with yours.`, + ); + } + return transaction; +}; + +// What an enqueue gives. A first one is told to the sink in the transaction +// of the store when there is one; a job enqueued again is not, and one +// enqueued again for another message is refused. +const afterEnqueue = async (queue, prepared, result, transaction) => { + const { data, request } = prepared; const answer = { id: request.id, correlationId: data.correlationId, messageId: data.messageId }; if (result?.duplicate === true) { if (result.existingHash !== undefined && result.existingHash !== request.hash) { @@ -63,16 +91,7 @@ const afterEnqueue = async (queue, { data, request }, result, enqueueOptions) => queue.log.write('debug', () => ({ jobId: request.id }), 'queue.duplicate'); return { ...answer, duplicate: true }; } - if (queue.audit !== undefined) { - try { - await queue.audit.append(queuedEventOf(data, request), { - transaction: enqueueOptions.transaction, - }); - } catch (error) { - if (queue.auditStrict) throw error; - warnThrown('The audit sink of the mail queue', error, 'MAILER_AUDIT_FAILED'); - } - } + if (transaction !== undefined) await appendQueued(queue, prepared, transaction); queue.log.write( 'debug', () => ({ jobId: request.id, priority: request.priority, template: data.template }), @@ -144,14 +163,20 @@ const createMailQueue = (options = {}) => { driver, async enqueue(input, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); + const transaction = transactionOf(driver, enqueueOptions); const prepared = await prepare(input); + if (transaction === undefined) await appendQueued(queue, prepared, undefined); const result = await driver.enqueue(prepared.request, enqueueOptions); - return afterEnqueue(queue, prepared, result, enqueueOptions); + return afterEnqueue(queue, prepared, result, transaction); }, async enqueueMany(inputs, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); + const transaction = transactionOf(driver, enqueueOptions); const prepared = []; for (const input of inputs) prepared.push(await prepare(input)); + if (transaction === undefined) { + for (const job of prepared) await appendQueued(queue, job, undefined); + } let results; if (typeof driver.enqueueMany === 'function') { const requests = []; @@ -167,7 +192,7 @@ const createMailQueue = (options = {}) => { const reused = []; for (let index = 0; index < prepared.length; index++) { try { - answers.push(await afterEnqueue(queue, prepared[index], results[index], enqueueOptions)); + answers.push(await afterEnqueue(queue, prepared[index], results[index], transaction)); } catch (error) { if (error?.code !== 'queue.key_reused') throw error; reused.push(prepared[index].request.id); diff --git a/tests/types/mongodb.test-d.ts b/tests/types/mongodb.test-d.ts index 8763895..5704d25 100644 --- a/tests/types/mongodb.test-d.ts +++ b/tests/types/mongodb.test-d.ts @@ -21,6 +21,8 @@ expectAssignable(db.collection('mail_outbox')); declare const typed: Collection<{ _id: string; state: string }>; expectAssignable(typed); +const typedOutbox = createMongoOutbox({ collection: db.collection('x') }); +expectType(typedOutbox.transactional); const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), changeStream: true, diff --git a/tests/unit/queue.test.js b/tests/unit/queue.test.js index 6800dd8..6a2a6ff 100644 --- a/tests/unit/queue.test.js +++ b/tests/unit/queue.test.js @@ -220,6 +220,7 @@ test('Queue: the queued event goes in the transaction of the enqueue; a strict s const sink = { append: (event, context) => contexts.push([event.state, context?.transaction]) }; const transactional = { id: 'outbox', + transactional: true, enqueue: async (request, options) => ({ id: request.id, duplicate: false, @@ -291,3 +292,37 @@ test('Queue: enqueueMany refuses a key used before only once every other job is assert.strictEqual(audit.stateOf(answer.correlationId), 'queued'); } }); + +test('Queue: without a transaction, a strict sink that fails refuses the enqueue before the job', async () => { + const broken = { append: () => Promise.reject(new Error('audit down')) }; + const { driver, queue } = setup({ audit: broken, auditStrict: true }); + await assert.rejects(queue.enqueue(resetJob()), /audit down/); + await assert.rejects(queue.enqueueMany([resetJob({ idempotencyKey: 'b' })]), /audit down/); + // Nothing was enqueued: a retry of the caller is not a second message. + assert.strictEqual(driver.list().length, 0); + // A driver that writes no transaction refuses one rather than ignore it. + await assert.rejects(queue.enqueue(resetJob(), { transaction: {} }), { + name: 'TypeError', + message: /memory driver writes no transaction/, + }); + await assert.rejects(queue.enqueueMany([resetJob()], { transaction: {} }), TypeError); + // The event of a first enqueue is written before it, a duplicate's once. + const order = []; + const sink = createMemoryAuditSink(); + const recording = { + append: (event) => { + order.push(`audit:${event.state}`); + return sink.append(event); + }, + }; + const tracked = setup({ audit: recording }); + const enqueue = tracked.driver.enqueue; + tracked.driver.enqueue = (request, options) => { + order.push('enqueue'); + return enqueue(request, options); + }; + const first = await tracked.queue.enqueue(resetJob()); + await tracked.queue.enqueue(resetJob()); + assert.deepStrictEqual(order, ['audit:queued', 'enqueue', 'audit:queued', 'enqueue']); + assert.strictEqual(sink.eventsOf(first.correlationId).length, 1); +}); From bb05f9330fe533f59fcddf7a13c578f32733a1c7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:31:30 +0300 Subject: [PATCH 22/83] fix(queue): give back a job this worker cannot read, its attempts kept A job of a later version, or sealed for a codec the worker lacks, was retried after five minutes, each try an attempt, so during a rolling deploy longer than its attempts the job failed for good before a worker that reads it came up. It is released instead (its attempts not spent) until it expires or has waited a week; a codec that fails, which may be an outage of its key, is still retried within the attempts. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +-- docs/guide/queue.md | 10 ++-- src/queue/handler.js | 79 ++++++++++++++++++------------- tests/unit/queue-handler.test.js | 18 +++++-- tests/unit/queue-security.test.js | 12 +++-- 5 files changed, 77 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 247e4c6..f4dd436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,14 +43,15 @@ dependency, BullMQ, MongoDB and the template engines being passed in. past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does not take every worker down in turn for ever. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads - it (`queue.unsupported_version`) rather than fail its hash during a rolling deploy. + it (`queue.unsupported_version`, given back with its attempts not spent, for a week or until it + expires) rather than fail its hash during a rolling deploy. `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends no job they did not sign (`queue.signature_invalid`): the SHA-256 of a job tells a change, but whoever writes to the store can recompute it. The first key signs and each verifies, for rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the application's key) seals the fields of each job in its store, its addresses, context and attachments, leaving the version, - hash, signature and trace context in clear; a worker that cannot open a sealed job gives it - back (`queue.codec_missing`, `queue.codec_failed`). `idSecret` derives the id of a job from its + hash, signature and trace context in clear; a worker without the codec gives a sealed job back + (`queue.codec_missing`), and one whose codec fails retries it (`queue.codec_failed`). `idSecret` derives the id of a job from its key with an HMAC, so an id that goes into the audit and the logs cannot be traced back to a key that names a person. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ diff --git a/docs/guide/queue.md b/docs/guide/queue.md index e9d6845..03377b7 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -146,10 +146,10 @@ const queue = createMailQueue({ driver, codec, signingKeys }); A sealed job keeps in clear only `v`, `hash` (a duplicate is still told by it), `enqueuedAt`, `kid`, `mac` and `trace`; the rest is `enc`. Its workers -take the queue's codec. A worker without it, or whose codec throws (a key out -of reach), gives the job back for a worker that can open it -(`queue.codec_missing`, `queue.codec_failed`); a codec that throws an error -with `retryable: false` fails it. Encryption is not a signature: a writer who +take the queue's codec. A worker without it gives the job back, its attempts +not spent, for a worker that can open it (`queue.codec_missing`); one whose +codec throws (a key out of reach) retries it (`queue.codec_failed`), and a +codec that throws an error with `retryable: false` fails it. Encryption is not a signature: a writer who can seal can still forge, so seal with `signingKeys` too. ## Once per key @@ -191,7 +191,7 @@ rotated, as another secret makes other ids and the same message another job. | an outage of the template loader or of `resolveAttachment` | retried | `queued` | | a bug: a `TypeError` thrown by the transport | failed at once (`queue.thrown`), logged with its stack | `failed` | | a job past `expiresAt`, or whose retry would come after it | failed (`queue.expired`) | `failed` | -| a job written by a later version of the worker | retried, for a worker of that version | `queued` | +| a job written by a later version of the worker, or sealed for a codec it lacks | given back, its attempts not spent, for a worker that reads it; failed once it expires or has waited a week | — | | the worker stops before the send | given back, its attempt not spent | — | | a job whose runs ended unsettled more than `maxRedeliveries` times (3): it kills or blocks its worker | failed before it renders (`queue.redelivery_exhausted`) | `delivery_unknown` | diff --git a/src/queue/handler.js b/src/queue/handler.js index 55b8f92..2000190 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -35,14 +35,13 @@ const DEFAULT_OPERATOR_DELAY_MS = 5 * 60_000; const DEFAULT_MAX_REDELIVERIES = 3; // The errors of a template that fails again the same way. const TEMPLATE_CODES = new Set(['MAILER_TEMPLATE_NOT_FOUND', 'MAILER_TEMPLATE_RENDER']); -// The jobs this worker cannot read yet: of a later version, or sealed for a -// codec it lacks or that fails. They wait for a worker that can. -const WAITING_CODES = new Set([ - 'queue.unsupported_version', - 'queue.codec_missing', - 'queue.codec_failed', -]); - +// The jobs this worker cannot read yet, written by a later version or sealed +// for a codec it lacks: they are given back, their attempts not spent, for a +// worker that can, as during a rolling deploy. +const LATER_CODES = new Set(['queue.unsupported_version', 'queue.codec_missing']); +// How long such a job waits for its worker before it is given up: a week, +// the time a failed job is kept. +const MAX_WAIT_FOR_WORKER_MS = 7 * 24 * 60 * 60 * 1000; const checkBackoff = (given) => { const backoff = { ...DEFAULT_BACKOFF, ...given }; for (const name of ['baseDelayMilliseconds', 'maxDelayMilliseconds']) { @@ -443,37 +442,51 @@ const attempt = async (run) => { return sendStep(run, message); }; -// A codec that says its failure is for good (a ciphertext that does not -// authenticate) fails the job; any other failure may be an outage of its key. -const isWaiting = (error) => - WAITING_CODES.has(error?.code) && - !(error.code === 'queue.codec_failed' && error.cause?.retryable === false); +// Whether a job waiting for a worker that can read it has waited enough: past +// its `expiresAt`, when that can be read, or a week after it was enqueued. +const isOverdue = (config, job) => { + const now = config.now(); + const expiresAt = typeof job.data?.expiresAt === 'string' ? Date.parse(job.data.expiresAt) : NaN; + if (Number.isFinite(expiresAt) && now > expiresAt) return true; + return job.enqueuedAt !== undefined && now - job.enqueuedAt > MAX_WAIT_FOR_WORKER_MS; +}; + +// What the queue does with a job this worker cannot read. +const unreadOutcome = (config, job, error) => { + const code = error?.code ?? 'queue.invalid_job'; + const reason = reasonOf(messageOf(error)); + if (LATER_CODES.has(code) && !isOverdue(config, job)) { + config.log.write( + 'warn', + () => ({ jobId: job.id, attempt: job.attempt, code }), + 'queue.job.released', + ); + return { type: 'release', delayMilliseconds: config.operatorDelayMilliseconds }; + } + // A codec that fails may be an outage of its key: retried, its attempts + // spent, unless it says the failure is for good (a ciphertext that does not + // authenticate). + const retryable = + code === 'queue.codec_failed' && + error.cause?.retryable !== false && + job.attempt < job.maxAttempts; + config.log.write( + retryable ? 'warn' : 'error', + () => ({ jobId: job.id, attempt: job.attempt, code }), + retryable ? 'queue.job.retrying' : 'queue.job.failed', + ); + if (retryable) { + return { type: 'retry', delayMilliseconds: config.operatorDelayMilliseconds, code, reason }; + } + return { type: 'fail', code, reason }; +}; const readJob = async (config, job) => { try { const data = await openMailJob(job.data, config.codec); return { data: parseMailJob(data, { signingKeys: config.signingKeys }) }; } catch (error) { - const code = error?.code ?? 'queue.invalid_job'; - const later = isWaiting(error) && job.attempt < job.maxAttempts; - config.log.write( - later ? 'warn' : 'error', - () => ({ jobId: job.id, attempt: job.attempt, code }), - later ? 'queue.job.retrying' : 'queue.job.failed', - ); - // A job a later version wrote waits for a worker of that version, and a - // sealed one for a worker with its codec. - if (later) { - return { - outcome: { - type: 'retry', - delayMilliseconds: config.operatorDelayMilliseconds, - code, - reason: reasonOf(error.message), - }, - }; - } - return { outcome: { type: 'fail', code, reason: reasonOf(messageOf(error)) } }; + return { outcome: unreadOutcome(config, job, error) }; } }; diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index c944f9e..dfd3fa7 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -334,11 +334,21 @@ test('Queue: a job that is not one, or of a later version, or expired, is not se const tampered = jobOf(); tampered.data.to = [{ address: 'evil@example.com' }]; assert.strictEqual((await run(handler, tampered)).code, 'queue.hash_mismatch'); + // A job of a later version is given back for a worker of it, its attempts + // not spent, until it expires or has waited a week. const later = { ...jobOf(), data: { ...jobOf().data, v: 2 } }; - assert.deepStrictEqual( - [(await run(handler, later)).type, (await run(handler, { ...later, attempt: 5 })).type], - ['retry', 'fail'], - ); + assert.deepStrictEqual(await run(handler, { ...later, attempt: 5 }), { + type: 'release', + delayMilliseconds: 300_000, + }); + const week = 7 * 24 * 60 * 60 * 1000; + const stale = await run(handler, { ...later, enqueuedAt: NOW - week - 1 }); + assert.deepStrictEqual([stale.type, stale.code], ['fail', 'queue.unsupported_version']); + const lapsed = await run(handler, { + ...later, + data: { ...later.data, expiresAt: new Date(NOW - 1).toISOString() }, + }); + assert.strictEqual(lapsed.type, 'fail'); const expired = jobOf({ expiresAt: new Date(NOW - 1) }); assert.strictEqual((await run(handler, expired)).code, 'queue.expired'); // A retry that would come after the job expires fails it now. diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index 264cf0e..b176eda 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -206,10 +206,14 @@ test('Queue: a sealed job waits for a worker with its codec, and fails one that const sealed = queue.driver.list()[0].data; const transport = fakeTransport(); const plain = createMailJobHandler({ transport, now: () => NOW }); - const waiting = await plain(runOf(sealed), new AbortController().signal); - assert.deepStrictEqual([waiting.type, waiting.code], ['retry', 'queue.codec_missing']); - const last = await plain(runOf(sealed, { attempt: 5 }), new AbortController().signal); - assert.deepStrictEqual([last.type, last.code], ['fail', 'queue.codec_missing']); + // Given back for a worker with the codec, its attempts not spent, for a week. + const waiting = await plain(runOf(sealed, { attempt: 5 }), new AbortController().signal); + assert.strictEqual(waiting.type, 'release'); + const stale = await plain( + runOf(sealed, { enqueuedAt: Date.now() - 8 * 24 * 60 * 60 * 1000 }), + new AbortController().signal, + ); + assert.deepStrictEqual([stale.type, stale.code], ['fail', 'queue.codec_missing']); // Another key: the ciphertext does not authenticate, which may be an outage of the key... const other = createMailJobHandler({ transport, codec: aesCodec(Buffer.alloc(32, 1)) }); const outage = await other(runOf(sealed), new AbortController().signal); From ba8955dcd5c8d486aaa2c7dac32762b418a5fa70 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:31:33 +0300 Subject: [PATCH 23/83] fix(templates): give each Nunjucks render its own missing include The failed lookups of an environment were one list, cut to the window of each render, so of two concurrent renders one took the other's missing include and the other lost its own; and a missing include was a failure of the loader, which `ignore missing` does not skip. A lookup that finds nothing now tells Nunjucks there is no such template, which it fails on itself unless `ignore missing`, and the TemplateNotFoundError is made again from the name its error gives, with the names tried kept by name. Co-Authored-By: Claude Opus 5.5 --- src/templates/nunjucks.js | 58 ++++++++++++++++++---------- tests/unit/templates-engines.test.js | 24 ++++++++++++ 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/src/templates/nunjucks.js b/src/templates/nunjucks.js index 55737bf..6bd1734 100644 --- a/src/templates/nunjucks.js +++ b/src/templates/nunjucks.js @@ -3,6 +3,7 @@ // Nunjucks's file system, for `extends`, `include` and `import`; output is // escaped, as Nunjucks does by default. +const { createCache } = require('./cache.js'); const { TemplateNotFoundError } = require('./errors.js'); const { checkEngineExtensions, @@ -14,6 +15,31 @@ const { checkOptionalFunction } = require('../internal/options.js'); const isRelative = (name) => name.startsWith('./') || name.startsWith('../'); +// What Nunjucks says of a template no loader has. +const NOT_FOUND = 'template not found: '; +// The failed lookups an environment keeps, to tell which names were tried. +const MAX_FAILURES = 1000; + +// The template Nunjucks found missing, by its error, whose message ends a +// line with `template not found: `: the error is made anew, without +// the cause, unless `dev` is on. +const missingOf = (error, failures) => { + let current = error; + for (let depth = 0; depth < 8 && typeof current === 'object' && current !== null; depth++) { + const { message } = current; + const at = typeof message === 'string' ? message.lastIndexOf(NOT_FOUND) : -1; + if (at !== -1) { + const start = at + NOT_FOUND.length; + const end = message.indexOf('\n', start); + const name = message.slice(start, end === -1 ? message.length : end); + const failure = failures.get(name); + return new TemplateNotFoundError(name, failure?.tried ?? [name], { cause: error }); + } + current = current.cause; + } + return undefined; +}; + /** * Makes the Nunjucks engine. * @@ -39,9 +65,10 @@ const createNunjucksEngine = (options = {}) => { const environmentOf = (includes) => { let state = environments.get(includes); if (state !== undefined) return state; - // What a lookup failed with: Nunjucks rewraps the error of a loader into - // a TemplateError of its text, which loses its class. - const failures = []; + // The lookups that found nothing, by name: Nunjucks is told there is no + // such template, which it fails on itself unless `ignore missing`, and + // its error names the template, so each render finds its own. + const failures = createCache(MAX_FAILURES); const loader = { async: true, isRelative, @@ -57,8 +84,12 @@ const createNunjucksEngine = (options = {}) => { noCache: !includes.cache, }), (error) => { - failures.push(error); - process.nextTick(callback, error); + if (!(error instanceof TemplateNotFoundError)) { + process.nextTick(callback, error); + return; + } + failures.set(name, error); + process.nextTick(callback, null, null); }, ); }, @@ -78,22 +109,9 @@ const createNunjucksEngine = (options = {}) => { const template = new nunjucks.Template(source, state.env, context.name, true); return (scope) => new Promise((resolve, reject) => { - const before = state.failures.length; template.render(scope, (error, output) => { - if (error == null) { - resolve(output ?? ''); - return; - } - // The failure of a lookup during this render, under the error - // Nunjucks made of it. - let failure; - for (let index = before; index < state.failures.length; index++) { - if (state.failures[index] instanceof TemplateNotFoundError) { - failure = state.failures[index]; - } - } - state.failures.length = before; - reject(failure === undefined ? error : Object.assign(error, { cause: failure })); + if (error == null) resolve(output ?? ''); + else reject(missingOf(error, state.failures) ?? error); }); }); }, diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 88954ee..ce4d919 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -574,6 +574,30 @@ test('Templates: Nunjucks autoescapes, extends and includes through the loader', ); }); +test('Templates: concurrent Nunjucks renders each fail with their own missing include', async () => { + // Before: the failures of an environment were one list, cut to the window + // of each render, so a render took the failure of another one. + const engine = createNunjucksEngine({ nunjucks }); + const renderer = createTemplateRenderer({ + engine, + loader: createMemoryLoader({ + 'a.njk': '{% include "missing-a.njk" %}', + 'b.njk': '{% include "missing-b.njk" %}', + 'c.njk': '

    {% include "nope.njk" ignore missing %}

    ', + }), + }); + const [a, b] = await Promise.all([ + failureOf(renderer.render({ template: 'a' })), + failureOf(renderer.render({ template: 'b' })), + ]); + assert.ok(a instanceof TemplateNotFoundError); + assert.ok(b instanceof TemplateNotFoundError); + assert.deepStrictEqual([a.template, b.template], ['missing-a.njk', 'missing-b.njk']); + assert.deepStrictEqual(a.tried, ['missing-a.njk']); + // Before: a missing include failed even with `ignore missing`. + assert.strictEqual((await renderer.render({ template: 'c' })).html, '

    '); +}); + test('Templates: Eta includes and lays out through the loader, relative to the file', async () => { const engine = createEtaEngine({ Eta }); const templates = { From 177d6cba501f038f8e0bdbda53ea108e38159eac Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:33:40 +0300 Subject: [PATCH 24/83] feat(queue): a run again after a submit may be of unknown delivery A job whose run died was lent again and sent again, the redelivery only logged: behind IdempotencyTransport that is deduped, but a transport with no idempotency key (SES, SMTP) sent the message twice when the run before died after its send. redelivered: 'delivery_unknown' makes such a run of unknown delivery (queue.redelivered), unless the audit sink can tell that the run before died before it submitted: AuditSink.has(id), on the memory, MongoDB and combined sinks, asked for the submitted event of the attempt, whose id is deterministic. The default stays 'send'. ./queue goes from 17.2 to 17.6 KB min+gzip: its budget goes from 17.5 to 18 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++++- docs/guide/audit.md | 5 ++++- docs/guide/queue.md | 7 ++++++ index.d.ts | 5 +++++ scripts/size-budgets.json | 2 +- src/audit/combine.js | 17 ++++++++++++-- src/audit/index.d.ts | 1 + src/audit/memory-sink.js | 3 ++- src/mongodb/audit-sink.js | 7 +++++- src/mongodb/index.d.ts | 1 + src/queue/handler.js | 38 ++++++++++++++++++++++++++++++-- src/queue/index.d.ts | 9 ++++++++ tests/types/queue.test-d.ts | 3 ++- tests/unit/audit.test.js | 10 +++++++++ tests/unit/mongodb.test.js | 19 ++++++++++++++++ tests/unit/queue-handler.test.js | 30 +++++++++++++++++++++++++ 16 files changed, 152 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4dd436..33061db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,10 @@ dependency, BullMQ, MongoDB and the template engines being passed in. lease runs out, not after another worker took the job. A source counts the runs of a job that end unsettled (`redeliveries`, never counted down, BullMQ's `stalledCounter`), and the worker gives up a job past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does - not take every worker down in turn for ever. The fields of a job of version 1 are fixed: + not take every worker down in turn for ever. `redelivered: 'delivery_unknown'` makes a run again + of an attempt whose run before died `delivery_unknown` rather than a second send, unless the + audit sink tells (`has`, new on the memory, MongoDB and combined sinks) that it died before + its submit. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`, given back with its attempts not spent, for a week or until it expires) rather than fail its hash during a rolling deploy. diff --git a/docs/guide/audit.md b/docs/guide/audit.md index 4b73c54..5ae6d54 100644 --- a/docs/guide/audit.md +++ b/docs/guide/audit.md @@ -40,7 +40,10 @@ comes after a `delivered`. `isFinalState` is true for `delivered`, `bounced`, } ``` -An event holds no recipient: the message is found by its `correlationId`. +An event holds no recipient: the message is found by its `correlationId`. Its +`id` is the same however often it is told, and a sink that can tell whether it +holds one (`has(id)`: the memory and MongoDB sinks, and combined sinks when one +of them can) lets a worker know a run that submitted before it died. ## Sinks diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 03377b7..beddd04 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -169,6 +169,13 @@ attempt again is of unknown delivery (`delivery_unknown`), not a second send. The rendering must then be the same for the same context: no `Date.now()` in a template, or an attempt again is refused as another message. +With a transport that takes no idempotency key (SES, SMTP) and no +`IdempotencyTransport`, give the worker `redelivered: 'delivery_unknown'`: a +run again of an attempt whose run before died is then of unknown delivery +(`queue.redelivered`), not a second send, unless the audit sink can tell +(`has(id)`, as the memory and MongoDB sinks can) that the run before died +before it submitted the message. + The id of a job is a hash of its key, and it goes where addresses do not: into the audit, the logs, the provider's idempotency key and the Message-ID. A key is an id of what the message tells of (`reset-password:${resetRequest.id}`), diff --git a/index.d.ts b/index.d.ts index 2d1bced..c4bf075 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1465,6 +1465,11 @@ export interface AuditContext { /** Where audit events go: append-only, an event written once by its `id`. */ export interface AuditSink { append(event: AuditEvent, context?: AuditContext): void | Promise; + /** + * Whether an event of this id was written: lets a worker tell a run that + * submitted its message before it died (the ids are deterministic). + */ + has?(id: string): boolean | Promise; } /** A value of JSON. */ diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 73cbfd1..445a425 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 17.5, + "./queue": 18, "./bullmq": 4, "./mongodb": 6 } diff --git a/src/audit/combine.js b/src/audit/combine.js index fbd909a..dcc2655 100644 --- a/src/audit/combine.js +++ b/src/audit/combine.js @@ -6,14 +6,18 @@ * * @param sinks The sinks. * @returns A sink that appends to each; it rejects with what one threw, or an - * `AggregateError` of what several threw, once all were told. + * `AggregateError` of what several threw, once all were told. It has `has` + * when one of them has: true when any holds the event. * @throws {TypeError} If one is not a sink. */ const combineAuditSinks = (...sinks) => { for (const sink of sinks) { if (typeof sink?.append !== 'function') throw new TypeError('An audit sink has append().'); } - return { + // `has`, when a sink can tell: an event any of them holds was written. + let readable = false; + for (const sink of sinks) readable ||= typeof sink.has === 'function'; + const combined = { async append(event, context) { const errors = []; for (const sink of sinks) { @@ -27,6 +31,15 @@ const combineAuditSinks = (...sinks) => { if (errors.length > 1) throw new AggregateError(errors, 'Audit sinks failed.'); }, }; + if (readable) { + combined.has = async (id) => { + for (const sink of sinks) { + if (typeof sink.has === 'function' && (await sink.has(id))) return true; + } + return false; + }; + } + return combined; }; module.exports = { combineAuditSinks }; diff --git a/src/audit/index.d.ts b/src/audit/index.d.ts index 8935e21..85d3327 100644 --- a/src/audit/index.d.ts +++ b/src/audit/index.d.ts @@ -6,6 +6,7 @@ import type { AuditEvent, AuditSink, DeliveryEvent, Logger, MessageState } from /** An audit sink in memory, with what it holds. */ export interface MemoryAuditSink extends AuditSink { append(event: AuditEvent): void; + has(id: string): boolean; /** Its events, in the order they came, one per id. */ readonly events: readonly AuditEvent[]; /** The events of one message. */ diff --git a/src/audit/memory-sink.js b/src/audit/memory-sink.js index 08f2b4e..d16fd32 100644 --- a/src/audit/memory-sink.js +++ b/src/audit/memory-sink.js @@ -9,7 +9,7 @@ const { stateOf } = require('./states.js'); * Makes an audit sink that holds its events in memory. * * @param options `maxEvents` (10000): past it, the oldest go first. - * @returns The sink, with `events`, `eventsOf(correlationId)`, + * @returns The sink, with `has(id)`, `events`, `eventsOf(correlationId)`, * `stateOf(correlationId)` and `clear()`. * @throws {TypeError} If `maxEvents` is not a positive integer. */ @@ -31,6 +31,7 @@ const createMemoryAuditSink = (options = {}) => { events.set(event.id, Object.freeze({ ...event })); if (events.size > maxEvents) events.delete(events.keys().next().value); }, + has: (id) => events.has(id), get events() { return [...events.values()]; }, diff --git a/src/mongodb/audit-sink.js b/src/mongodb/audit-sink.js index 09c6a17..23ab203 100644 --- a/src/mongodb/audit-sink.js +++ b/src/mongodb/audit-sink.js @@ -21,13 +21,14 @@ const eventOfDocument = (document) => { * Makes an audit sink in a collection. * * @param options `collection`, `db.collection('mail_audit')`. - * @returns The sink, with `eventsOf(correlationId)`, + * @returns The sink, with `has(id)`, `eventsOf(correlationId)`, * `stateOf(correlationId)` and `ensureIndexes()`. * @throws {TypeError} If it is given no collection. */ const createMongoAuditSink = (options = {}) => { const collection = checkCollection(options.collection, 'createMongoAuditSink', [ 'updateOne', + 'findOne', 'find', 'createIndex', ]); @@ -42,6 +43,10 @@ const createMongoAuditSink = (options = {}) => { return events; }; return { + async has(id) { + const found = await collection.findOne({ _id: id }, { projection: { _id: 1 } }); + return found !== null && found !== undefined; + }, async append(event, context) { if (typeof event?.id !== 'string') throw new TypeError('An audit event has an id.'); await collection.updateOne( diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index b1c6b33..9654ded 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -119,6 +119,7 @@ export interface MongoAuditSinkOptions { /** An audit sink in a collection. */ export interface MongoAuditSink extends AuditSink { append(event: AuditEvent, context?: { readonly transaction?: unknown }): Promise; + has(id: string): Promise; eventsOf(correlationId: string): Promise; stateOf(correlationId: string): Promise; /** Makes the indexes of the events: by correlation id and time, by job. */ diff --git a/src/queue/handler.js b/src/queue/handler.js index 2000190..40c4b2a 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -73,6 +73,10 @@ const createHandlerConfig = (options = {}) => { if (options.auditStrict !== undefined && typeof options.auditStrict !== 'boolean') { throw new TypeError('auditStrict must be a boolean.'); } + const redelivered = options.redelivered ?? 'send'; + if (redelivered !== 'send' && redelivered !== 'delivery_unknown') { + throw new TypeError("redelivered must be 'send' or 'delivery_unknown'."); + } // Checked now, so that a bad key fails the worker, not each job. keyringOf(options.signingKeys); return { @@ -91,6 +95,7 @@ const createHandlerConfig = (options = {}) => { 'operatorDelayMilliseconds', ), rateLimitScope, + redelivered, maxRedeliveries: checkNonNegativeInteger( options.maxRedeliveries ?? DEFAULT_MAX_REDELIVERIES, 'maxRedeliveries', @@ -120,6 +125,10 @@ const baseAuditOf = (job, data) => { return base; }; +// The id of the audit event of a state of a run: the same however often it +// is told, so that a run again of an attempt can ask whether it was told. +const eventIdOf = (job, attempt, state) => toUuid(`${job.id}\u0000${attempt}\u0000${state}`); + // Tells the audit sink a state. A sink that fails is a warning; with // `auditStrict`, before the message is submitted, it makes the job retried, // as an attempt is not made unrecorded. @@ -127,7 +136,7 @@ const emit = async (run, state, extra = {}, attempt = run.job.attempt) => { const { config, job } = run; if (config.audit === undefined) return undefined; const event = { - id: toUuid(`${job.id}\u0000${attempt}\u0000${state}`), + id: eventIdOf(job, attempt, state), state, at: new Date(config.now()).toISOString(), ...run.base, @@ -399,6 +408,27 @@ const sendStep = async (run, message) => { return settleFailure(run, receipt); }; +// A run again of an attempt whose run before may have sent the message: of +// unknown delivery, rather than sent blindly again, by a transport with no +// idempotency key (SES, SMTP). A sink that tells whether that run submitted +// lets one that died before it be sent; without one, it is not sent. +const sentBefore = async (run) => { + const { audit } = run.config; + if (typeof audit?.has === 'function') { + let submitted; + try { + submitted = await audit.has(eventIdOf(run.job, run.job.attempt, 'submitted')); + } catch { + submitted = true; + } + if (!submitted) return undefined; + } + return fail(run, 'queue.redelivered', 'A run before of this attempt may have sent it.', { + state: 'delivery_unknown', + category: 'unknown', + }); +}; + const isExpired = (run) => run.data.expiresAt !== undefined && run.config.now() > Date.parse(run.data.expiresAt); @@ -415,6 +445,10 @@ const attempt = async (run) => { { state: 'delivery_unknown', category: 'unknown' }, ); } + if (job.redelivered && config.redelivered === 'delivery_unknown') { + const unknown = await sentBefore(run); + if (unknown !== undefined) return unknown; + } if (config.checkJob !== undefined) { try { await config.checkJob(data, job); @@ -499,7 +533,7 @@ const readJob = async (config, job) => { * `resolveAttachment(ref, { job, attachment, signal })`; * `checkJob(data, job)`; `classify({ receipt, job })`; `backoff`, * `operatorDelayMilliseconds` (5 minutes), `rateLimitScope` ('job'), - * `maxRedeliveries` (3); + * `maxRedeliveries` (3), `redelivered` ('send'); * `logger`, `random`, `now`. * @returns The handler: `(job, signal) => Promise`, which does * not throw. diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 1ce8b0d..2c71c78 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -346,6 +346,15 @@ export interface MailWorkerOptions { * in turn for ever. @default 3 */ readonly maxRedeliveries?: number; + /** + * What a run again of an attempt does, its run before having died: + * `send` (for `IdempotencyTransport` or a provider's key to dedupe), or + * `delivery_unknown`, which sends it only when the audit sink can tell + * (`has`) that the run before died before it submitted. For a transport + * with no idempotency key (SES, SMTP) without `IdempotencyTransport`. + * @default 'send' + */ + readonly redelivered?: 'send' | 'delivery_unknown'; readonly logger?: Logger; readonly random?: () => number; readonly now?: () => number; diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 3abc444..39685f4 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -155,7 +155,8 @@ prepareMailJob( expectError(createMailQueue({ driver, idSecret: 42 })); const codec = { encode: (text: string) => text, decode: async (text: string) => text }; createMailQueue({ driver, codec }); -createMailJobHandler({ transport, codec, maxRedeliveries: 0 }); +createMailJobHandler({ transport, codec, maxRedeliveries: 0, redelivered: 'delivery_unknown' }); +expectError(createMailJobHandler({ transport, redelivered: 'never' })); expectError(createMailQueue({ driver, codec: { encode: (text: string) => text } })); createMailJobHandler({ transport, signingKeys: keys }); expectError(createMailJobHandler({ transport, signingKeys: [{ id: 'k', secret: 1 }] })); diff --git a/tests/unit/audit.test.js b/tests/unit/audit.test.js index 1612813..059b183 100644 --- a/tests/unit/audit.test.js +++ b/tests/unit/audit.test.js @@ -231,3 +231,13 @@ test('Audit: combined sinks are each told, whatever another throws', async () => assert.strictEqual(twice.errors.length, 2); assert.throws(() => combineAuditSinks(ok, {}), /has append/); }); + +test('Audit: a sink tells whether it holds an event, combined ones if any can', async () => { + const memory = createMemoryAuditSink(); + memory.append({ id: 'e1', state: 'submitted', correlationId: 'c' }); + assert.deepStrictEqual([memory.has('e1'), memory.has('e2')], [true, false]); + const blind = { append: () => {} }; + assert.strictEqual(combineAuditSinks(blind, blind).has, undefined); + const combined = combineAuditSinks(blind, memory); + assert.deepStrictEqual([await combined.has('e1'), await combined.has('e2')], [true, false]); +}); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 17041b0..ce9159b 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -479,3 +479,22 @@ test('MongoDB: a completed job goes after the window of duplicates, a failed one defaults.now() + 86_400_000, ); }); + +test('MongoDB: the audit sink tells whether it holds an event', async () => { + const collection = fakeCollection(); + const sink = createMongoAuditSink({ collection }); + await sink.append({ + id: 'e1', + state: 'submitted', + at: '2026-10-18T09:00:00.000Z', + source: 'worker', + correlationId: 'c1', + attempt: 1, + }); + assert.deepStrictEqual([await sink.has('e1'), await sink.has('e2')], [true, false]); + assert.deepStrictEqual(collection.calls.at(-1), [ + 'findOne', + { _id: 'e2' }, + { projection: { _id: 1 } }, + ]); +}); diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index dfd3fa7..eec7a65 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -564,3 +564,33 @@ test('Queue: a job whose runs keep ending unsettled is given up, unsent', async assert.throws(() => handlerOf({ maxRedeliveries }), /maxRedeliveries must be a whole number/); } }); + +test('Queue: under redelivered: delivery_unknown, a run again after a submit is not sent', async () => { + const { toUuid } = require('../../src/internal/uuid.js'); + const job = jobOf({}, { redelivered: true }); + const submittedId = toUuid(`${job.id}\u00001\u0000submitted`); + // The run before died after it told the sink it submitted. + const audit = createMemoryAuditSink(); + audit.append({ id: submittedId, state: 'submitted', correlationId: 'c', attempt: 1 }); + const { handler, transport } = handlerOf({ audit, redelivered: 'delivery_unknown' }); + const unknown = await run(handler, job); + assert.deepStrictEqual([unknown.type, unknown.code], ['fail', 'queue.redelivered']); + assert.strictEqual(audit.events.at(-1).state, 'delivery_unknown'); + assert.strictEqual(transport.calls.length, 0); + // One that died before its submit is sent. + const { handler: fresh, transport: sending } = handlerOf({ redelivered: 'delivery_unknown' }); + assert.strictEqual((await run(fresh, job)).type, 'ack'); + assert.strictEqual(sending.calls.length, 1); + // A sink that cannot tell, or none, sends nothing again. + const blind = { append: () => {} }; + const { handler: unsure } = handlerOf({ audit: blind, redelivered: 'delivery_unknown' }); + assert.strictEqual((await run(unsure, job)).code, 'queue.redelivered'); + const failing = { append: () => {}, has: () => Promise.reject(new Error('down')) }; + const { handler: down } = handlerOf({ audit: failing, redelivered: 'delivery_unknown' }); + assert.strictEqual((await run(down, job)).code, 'queue.redelivered'); + // By default a run again is sent, for IdempotencyTransport to dedupe. + const { handler: plain, transport: again } = handlerOf({ audit }); + assert.strictEqual((await run(plain, job)).type, 'ack'); + assert.strictEqual(again.calls.length, 1); + assert.throws(() => handlerOf({ redelivered: 'never' }), /redelivered must be/); +}); From c93ca720a86e00219156a0c59eb9bdbbacad892c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:35:11 +0300 Subject: [PATCH 25/83] fix(templates): name the template a loader failed to read A loader's failure came back as it was, naming no template, and under Nunjucks, which drops the error it wraps, and Liquid, which keeps it as `originalError`, it became a TemplateRenderError, which a queue fails at once; Liquid's missing include was liquidjs's ENOENT, a TemplateRenderError too. What a loader throws is now a TemplateLoadError (MAILER_TEMPLATE_LOAD) naming the template, its error the cause, retryable unless that error is a bug or says it is not, when `retryable` is left unset; Nunjucks keeps its causes (`dev: true`), Liquid's become causes, and liquidjs's last lookup, `fallback`, tells a missing template by the name written. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 8 +- src/templates/errors.js | 69 ++++++++++++-- src/templates/includes.js | 45 ++++------ src/templates/index.d.ts | 23 +++++ src/templates/index.js | 3 +- src/templates/liquid.js | 125 ++++++++++++++++++-------- src/templates/nunjucks.js | 3 +- src/templates/renderer.js | 15 ++-- tests/types/templates.test-d.ts | 4 + tests/unit/templates-engines.test.js | 37 +++++++- tests/unit/templates-renderer.test.js | 45 ++++++++-- 11 files changed, 285 insertions(+), 92 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index ac224b3..a837d75 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -216,11 +216,13 @@ reads and compiles on every render, for development. | --- | --- | --- | | `TemplateNotFoundError` (`MAILER_TEMPLATE_NOT_FOUND`) | no template, include, partial or layout; `tried` lists the names looked up | `false` | | `TemplateRenderError` (`MAILER_TEMPLATE_RENDER`) | syntax, a helper that threw, MJML under `strict`, the inliner; `phase`, `engine`, `cause` | `false` | +| `TemplateLoadError` (`MAILER_TEMPLATE_LOAD`) | the loader failed (`EMFILE`, a database timeout); `template` names what it read, `cause` is its error | `true`; unset for a bug of the loader (`TypeError`…) | | `TypeError`, `RangeError` | a name that is not one, a bad locale or context | — | -| the loader's own error | the loader failed (`EMFILE`, a database timeout) | as classified | -The queue fails a job at once on the first three, and retries an outage of the -loader. +The queue fails a job at once on the first two and the last, and retries an +outage of the loader. An engine that wraps what a loader threw in its own error +(Nunjucks, Liquid) does not hide it: the error of ./templates is found among its +causes and thrown as it is. ## Migrating from @nestjs-modules/mailer diff --git a/src/templates/errors.js b/src/templates/errors.js index c6a5ca2..48712f5 100644 --- a/src/templates/errors.js +++ b/src/templates/errors.js @@ -1,7 +1,8 @@ -// Not ported: the errors of ./templates. Each has a stable `code` and -// `retryable: false`, so that a queue fails a job whose template is missing -// or does not render at once, rather than retrying what fails again the same -// way; a queue reads the code, which holds across two copies of the package. +// Not ported: the errors of ./templates. Each has a stable `code`, which a +// queue reads, as it holds across two copies of the package. A template that +// is missing or does not render has `retryable: false`, so that a queue fails +// its job at once rather than retrying what fails again the same way; a +// loader that fails may pass on another attempt. // The names a message lists, of all those tried. const MAX_NAMES = 8; @@ -61,4 +62,62 @@ class TemplateRenderError extends Error { } } -module.exports = { TemplateNotFoundError, TemplateRenderError }; +// The errors a loader throws for a bug of its own, or of its caller. +const BUGS = [TypeError, RangeError, ReferenceError, SyntaxError]; + +const isBug = (error) => { + for (const type of BUGS) { + if (error instanceof type) return true; + } + return false; +}; + +/** + * A loader that failed to read a template: a file system that refused it, a + * database that timed out. The loader's error is its cause. It is + * retryable, as it may pass on another attempt, unless the loader's error is + * a bug or says it is not, when `retryable` is left unset for a queue to + * classify by the cause. + */ +class TemplateLoadError extends Error { + /** + * @param template The name the loader was asked for. + * @param options `from`, the template that included it, and `cause`, the + * loader's error. + */ + constructor(template, options = {}) { + const { cause } = options; + const reason = cause instanceof Error ? cause.message : String(cause); + super(`The template ${JSON.stringify(template)} failed to load: ${reason}`, options); + this.name = 'TemplateLoadError'; + this.code = 'MAILER_TEMPLATE_LOAD'; + if (cause?.retryable !== false && !isBug(cause)) this.retryable = true; + this.template = template; + if (options.from !== undefined) this.from = options.from; + } +} + +/** + * The missing template or the failed load an engine wrapped in its own + * error: the error itself or one of its causes. + * + * @returns The error of ./templates, or `undefined` when there is none. + * @internal + */ +const templateErrorOf = (error) => { + let current = error; + for (let depth = 0; depth < 8 && typeof current === 'object' && current !== null; depth++) { + if (current instanceof TemplateNotFoundError || current instanceof TemplateLoadError) { + return current; + } + current = current.cause; + } + return undefined; +}; + +module.exports = { + TemplateLoadError, + TemplateNotFoundError, + TemplateRenderError, + templateErrorOf, +}; diff --git a/src/templates/includes.js b/src/templates/includes.js index 9d3c297..e335311 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -2,36 +2,10 @@ // template: through the loader of its renderer, so that they are checked and // kept as its templates are, and never read from disk by the engine itself. -const { TemplateNotFoundError } = require('./errors.js'); +const { TemplateLoadError, TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); const { isTemplateName } = require('./loader.js'); const { createCache, memoize } = require('./cache.js'); -// The errors a loader threw, such as EMFILE or a database timeout: an engine -// that wraps them in its own does not make them a failure of the template, as -// they may pass on another attempt. -const loaderErrors = new WeakSet(); - -const markLoaderError = (error) => { - if (typeof error === 'object' && error !== null) loaderErrors.add(error); - return error; -}; - -/** - * The error a loader threw, found on what an engine made of it: the error - * itself or one of its causes. - * - * @returns The loader's error, or `undefined` when there is none. - * @internal - */ -const loaderErrorOf = (error) => { - let current = error; - for (let depth = 0; depth < 8 && typeof current === 'object' && current !== null; depth++) { - if (loaderErrors.has(current)) return current; - current = current.cause; - } - return undefined; -}; - // The segments of a name, `.` dropped and `..` taking the one before; undefined // when `..` leads out of the root. const normalize = (segments) => { @@ -104,6 +78,17 @@ const candidatesOf = (include, from, extensions, partials) => { return names; }; +// What a loader threw, as an error that names the template: one of +// ./templates as it is, and any other, such as EMFILE or a database timeout, +// as a TemplateLoadError, which an engine that wraps it does not make a +// failure of the template, as it may pass on another attempt. +const loadFailure = (name, error) => + error instanceof TemplateNotFoundError || + error instanceof TemplateRenderError || + error instanceof TemplateLoadError + ? error + : new TemplateLoadError(name, { cause: error }); + /** * Makes the includes of a renderer: the reads of its loader, kept while its * cache is on, and the lookup of a name an engine includes. @@ -123,7 +108,7 @@ const createIncludes = (loader, options) => { try { return (await loader.load(name)) ?? undefined; } catch (error) { - throw markLoaderError(error); + throw loadFailure(name, error); } }); const readSync = (name) => { @@ -138,7 +123,7 @@ const createIncludes = (loader, options) => { try { source = loader.loadSync(name) ?? undefined; } catch (error) { - throw markLoaderError(error); + throw loadFailure(name, error); } syncSources.set(name, { source }); return source; @@ -168,4 +153,4 @@ const createIncludes = (loader, options) => { }); }; -module.exports = { createIncludes, joinName, loaderErrorOf }; +module.exports = { createIncludes, joinName }; diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 17f1728..6899c59 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -246,6 +246,7 @@ export interface TemplateRenderer< /** * @throws {TemplateNotFoundError} If there is no such template. * @throws {TemplateRenderError} If it does not render. + * @throws {TemplateLoadError} If the loader failed to read one. * @throws {TypeError} If the input is invalid. */ render(input: TInput): Promise; @@ -292,6 +293,28 @@ export declare class TemplateNotFoundError extends Error { constructor(template: string, tried: readonly string[], options?: TemplateNotFoundErrorOptions); } +/** The options of {@link TemplateLoadError}. */ +export interface TemplateLoadErrorOptions extends ErrorOptions { + /** The template that included it, for an include, a partial or a layout. */ + readonly from?: string; +} + +/** + * A loader that failed to read a template: a file system that refused it, a + * database that timed out. The loader's error is its `cause`. Retryable, as + * it may pass on another attempt, unless the loader's error is a bug + * (`TypeError`, `RangeError`…) or says `retryable: false`: then `retryable` + * is unset, for a queue to classify by the cause. + */ +export declare class TemplateLoadError extends Error { + readonly code: 'MAILER_TEMPLATE_LOAD'; + readonly retryable?: true; + /** The name the loader was asked for. */ + readonly template: string; + readonly from?: string; + constructor(template: string, options?: TemplateLoadErrorOptions); +} + /** Where a template failed. */ export type TemplateRenderPhase = | 'front-matter' diff --git a/src/templates/index.js b/src/templates/index.js index fa1f105..0b2cd2b 100644 --- a/src/templates/index.js +++ b/src/templates/index.js @@ -3,7 +3,7 @@ const { createCssInliner } = require('./css.js'); const { createEjsEngine } = require('./ejs.js'); -const { TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); +const { TemplateLoadError, TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); const { createEtaEngine } = require('./eta.js'); const { createHandlebarsEngine } = require('./handlebars.js'); const { createLiquidEngine } = require('./liquid.js'); @@ -29,6 +29,7 @@ module.exports = { createPugEngine, createReactEmailEngine, createTemplateRenderer, + TemplateLoadError, TemplateNotFoundError, TemplateRenderError, }; diff --git a/src/templates/liquid.js b/src/templates/liquid.js index 872eb48..579dd04 100644 --- a/src/templates/liquid.js +++ b/src/templates/liquid.js @@ -23,9 +23,81 @@ const RESERVED = new Set([ 'cache', ]); -// A name no loader has: where a reference leads out of the root. +// A name no loader has: where a reference leads out of the root, and, before +// the name written, where liquidjs looks last for a template it did not find. const NOWHERE = '\u0000'; +const isMissing = (name) => name.length > 1 && name[0] === NOWHERE; + +// liquidjs keeps the error it wraps as `originalError`: it is made the cause, +// so that the renderer finds a missing include or a loader's failure in it. +const withCauses = (error) => { + let current = error; + for (let depth = 0; depth < 8 && typeof current === 'object' && current !== null; depth++) { + if (current.cause === undefined && current.originalError !== undefined) { + Object.defineProperty(current, 'cause', { + value: current.originalError, + configurable: true, + writable: true, + }); + } + current = current.cause; + } + return error; +}; + +// The file system of liquidjs over the includes of a renderer. +const fileSystemOf = (includes, roots, extname) => { + const resolve = (dir, file, extension) => { + const relative = dir === '' ? file : `${dir}/${file}`; + const name = includes.join(undefined, `/${relative}`); + if (name === undefined) return NOWHERE; + return name.lastIndexOf('.') > name.lastIndexOf('/') ? name : name + extension; + }; + // The template liquidjs did not find, by the names it looked up. + const missing = (name) => { + const file = name.slice(1); + const tried = []; + for (const root of roots) { + const candidate = resolve(root, file, extname); + if (candidate !== NOWHERE) tried.push(candidate); + } + return new TemplateNotFoundError(file, tried); + }; + return { + sep: '/', + async exists(name) { + if (name[0] === NOWHERE) return isMissing(name); + return (await includes.read(name)) !== undefined; + }, + existsSync(name) { + if (name[0] === NOWHERE) return isMissing(name); + return includes.readSync(name) !== undefined; + }, + async readFile(name) { + if (isMissing(name)) throw missing(name); + const found = await includes.read(name); + if (found === undefined) throw new TemplateNotFoundError(name, [name]); + return found.source; + }, + readFileSync(name) { + if (isMissing(name)) throw missing(name); + const found = includes.readSync(name); + if (found === undefined) throw new TemplateNotFoundError(name, [name]); + return found.source; + }, + resolve, + // Tried after every directory, so that a template not found is told by + // the name written, rather than by liquidjs's ENOENT. + fallback: (file) => NOWHERE + file, + dirname(name) { + const slash = name.lastIndexOf('/'); + return slash === -1 ? '' : name.slice(0, slash); + }, + contains: () => true, + }; +}; + /** * Makes the Liquid engine. * @@ -50,45 +122,17 @@ const createLiquidEngine = (options = {}) => { const instanceOf = (includes) => { let liquid = instances.get(includes); if (liquid !== undefined) return liquid; - const source = async (name) => (await includes.read(name))?.source; - const fs = { - sep: '/', - async exists(name) { - return name !== NOWHERE && (await includes.read(name)) !== undefined; - }, - existsSync: (name) => name !== NOWHERE && includes.readSync(name) !== undefined, - async readFile(name) { - const text = await source(name); - if (text === undefined) throw new TemplateNotFoundError(name, [name]); - return text; - }, - readFileSync(name) { - const found = includes.readSync(name); - if (found === undefined) throw new TemplateNotFoundError(name, [name]); - return found.source; - }, - resolve(dir, file, extname) { - const relative = dir === '' ? file : `${dir}/${file}`; - const name = includes.join(undefined, `/${relative}`); - if (name === undefined) return NOWHERE; - return name.lastIndexOf('.') > name.lastIndexOf('/') ? name : name + extname; - }, - dirname(name) { - const slash = name.lastIndexOf('/'); - return slash === -1 ? '' : name.slice(0, slash); - }, - contains: () => true, - }; + const roots = ['', ...includes.partials]; liquid = new Liquid({ outputEscape: 'escape', ownPropertyOnly: true, ...given, root: [''], - partials: ['', ...includes.partials], - layouts: ['', ...includes.partials], + partials: roots, + layouts: roots, extname: extensions[0], relativeReference: true, - fs, + fs: fileSystemOf(includes, roots, extensions[0]), cache: includes.cache, }); for (const name of Object.keys(filters)) liquid.registerFilter(name, filters[name]); @@ -101,8 +145,19 @@ const createLiquidEngine = (options = {}) => { extensions, compile(source, context) { const liquid = instanceOf(context.includes); - const template = liquid.parse(source, context.name); - return async (scope) => String(await liquid.render(template, scope)); + let template; + try { + template = liquid.parse(source, context.name); + } catch (error) { + throw withCauses(error); + } + return async (scope) => { + try { + return String(await liquid.render(template, scope)); + } catch (error) { + throw withCauses(error); + } + }; }, }; }; diff --git a/src/templates/nunjucks.js b/src/templates/nunjucks.js index 6bd1734..8fa2d04 100644 --- a/src/templates/nunjucks.js +++ b/src/templates/nunjucks.js @@ -94,7 +94,8 @@ const createNunjucksEngine = (options = {}) => { ); }, }; - const env = new nunjucks.Environment([loader], { autoescape: true, ...given }); + // `dev` keeps the error Nunjucks wraps as its cause, such as a loader's. + const env = new nunjucks.Environment([loader], { autoescape: true, dev: true, ...given }); for (const name of Object.keys(filters)) env.addFilter(name, filters[name]); configure?.(env); state = { env, failures }; diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 19e275e..00dddff 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -10,9 +10,9 @@ const { decodeEntities } = require('../render/entities.js'); const { messageOf } = require('../internal/decorator.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { createCache, memoize } = require('./cache.js'); -const { TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); +const { TemplateNotFoundError, TemplateRenderError, templateErrorOf } = require('./errors.js'); const { parseFrontMatter } = require('./front-matter.js'); -const { createIncludes, loaderErrorOf } = require('./includes.js'); +const { createIncludes } = require('./includes.js'); const { checkTemplateName } = require('./loader.js'); const DEFAULT_MAX_ENTRIES = 1000; @@ -149,16 +149,11 @@ const checkInput = (input) => { }; const wrapError = (error, template, phase, engine) => { - if (error instanceof TemplateNotFoundError || error instanceof TemplateRenderError) return error; + if (error instanceof TemplateRenderError) return error; // An engine may wrap what its include threw: a missing include stays one, // and a loader's failure stays the loader's, which may pass on another try. - let cause = error; - for (let depth = 0; depth < 8 && typeof cause === 'object' && cause !== null; depth++) { - if (cause instanceof TemplateNotFoundError) return cause; - cause = cause.cause; - } - const failure = loaderErrorOf(error); - if (failure !== undefined) return failure; + const found = templateErrorOf(error); + if (found !== undefined) return found; return new TemplateRenderError(template, phase, messageOf(error), { cause: error, engine: engine?.name, diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index dc98437..7fe62d9 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -20,6 +20,7 @@ import { createPugEngine, createReactEmailEngine, createTemplateRenderer, + TemplateLoadError, TemplateNotFoundError, TemplateRenderError, type TemplateEngine, @@ -123,3 +124,6 @@ expectType(missing.retryable); expectType(missing.tried); const broken = new TemplateRenderError('welcome', 'compile', 'x', { engine: 'handlebars' }); expectType<'MAILER_TEMPLATE_RENDER'>(broken.code); +const failed = new TemplateLoadError('welcome.hbs', { from: 'layout', cause: new Error('EMFILE') }); +expectType<'MAILER_TEMPLATE_LOAD'>(failed.code); +expectType(failed.retryable); diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index ce4d919..9d51a9b 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -27,6 +27,7 @@ const { createPugEngine, createReactEmailEngine, createTemplateRenderer, + TemplateLoadError, TemplateNotFoundError, TemplateRenderError, } = require('../../src/templates/index.js'); @@ -470,11 +471,14 @@ test('Templates: Liquid escapes by default, its includes and layouts through the const missing = await failureOf( renderWith(engine, { 'a.liquid': '{% render "nope" %}' }, { template: 'a' }), ); - assert.ok(missing instanceof TemplateRenderError || missing instanceof TemplateNotFoundError); + // Before: liquidjs's own ENOENT, a TemplateRenderError. + assert.ok(missing instanceof TemplateNotFoundError); + assert.deepStrictEqual([missing.template, missing.tried], ['nope', ['nope.liquid']]); const out = await failureOf( renderWith(engine, { 'a.liquid': '{% include "../../x" %}' }, { template: 'a' }), ); - assert.ok(out instanceof Error); + assert.ok(out instanceof TemplateNotFoundError); + assert.deepStrictEqual(out.tried, []); // Own properties only: a prototype's are not read. const proto = Object.create({ secret: 's' }); assert.strictEqual( @@ -631,6 +635,35 @@ test('Templates: Eta includes and lays out through the loader, relative to the f assert.throws(() => createEtaEngine({ Eta, options: { views: '/' } }), /views/); }); +test('Templates: a loader that fails under an engine fails as a load, naming the template', async () => { + // Before: the loader's own error, without the template, and under + // Nunjucks and Liquid a TemplateRenderError, which a queue does not retry. + const outage = Object.assign(new Error('EMFILE: too many open files'), { code: 'EMFILE' }); + const cases = [ + [createHandlebarsEngine({ handlebars }), 'a.hbs', '{{> b}}', 'b.hbs'], + [createNunjucksEngine({ nunjucks }), 'a.njk', '{% include "b.njk" %}', 'b.njk'], + [createLiquidEngine({ Liquid }), 'a.liquid', '{% render "b" %}', 'b.liquid'], + [createEjsEngine({ ejs }), 'a.ejs', '<%- include("b") %>', 'b.ejs'], + [createEtaEngine({ Eta }), 'a.eta', '<%~ include("b") %>', 'b.eta'], + [createMustacheEngine({ mustache }), 'a.mustache', '{{> b}}', 'b.mustache'], + [createPugEngine({ pug }), 'a.pug', 'include b', 'b.pug'], + ]; + for (const [engine, name, source, include] of cases) { + const read = (asked) => { + if (asked === include) throw outage; + return asked === name ? { name, source } : undefined; + }; + const renderer = createTemplateRenderer({ engine, loader: { load: read, loadSync: read } }); + const error = await failureOf(renderer.render({ template: 'a' })); + assert.ok(error instanceof TemplateLoadError, engine.name); + assert.deepStrictEqual( + [error.code, error.template, error.retryable, error.cause], + ['MAILER_TEMPLATE_LOAD', include, true, outage], + engine.name, + ); + } +}); + test('Templates: an engine that reads synchronously needs a loader with loadSync', async () => { const loader = { load: (name) => (name === 'a.ejs' ? { name, source: '<%- include("b") %>' } : undefined), diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index 6b2375b..2fbd34c 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -7,6 +7,7 @@ const { createCssInliner, createMemoryLoader, createTemplateRenderer, + TemplateLoadError, TemplateNotFoundError, TemplateRenderError, } = require('../../src/templates/index.js'); @@ -305,21 +306,55 @@ test('Templates: failures are classified by phase, the engine error their cause' } }); -test('Templates: a loader that fails fails with its own error, which may pass on a retry', async () => { +test('Templates: a loader that fails fails as a load of its template, which may pass on a retry', async () => { + // Before: the loader's error came back as it was, naming no template. const outage = Object.assign(new Error('EMFILE: too many open files'), { code: 'EMFILE' }); const loader = { load: (name) => { - if (name === 'a.tpl') return { name, source: '{{> b}}' }; - throw outage; + if (name === 'b.tpl') throw outage; + return name === 'a.tpl' ? { name, source: '{{> b}}' } : undefined; }, }; const renderer = createTemplateRenderer({ engine: fakeEngine(), loader }); - assert.strictEqual(await renderer.render({ template: 'a' }).catch((e) => e), outage); + const include = await renderer.render({ template: 'a' }).catch((e) => e); + assert.ok(include instanceof TemplateLoadError); + assert.deepStrictEqual( + [include.code, include.template, include.retryable, include.cause], + ['MAILER_TEMPLATE_LOAD', 'b.tpl', true, outage], + ); + assert.match(include.message, /The template "b.tpl" failed to load: EMFILE/); const failing = createTemplateRenderer({ engine: fakeEngine(), loader: { load: () => Promise.reject(outage) }, }); - assert.strictEqual(await failing.render({ template: 'x' }).catch((e) => e), outage); + const main = await failing.render({ template: 'x' }).catch((e) => e); + assert.deepStrictEqual([main.template, main.cause], ['x.tpl', outage]); + // A bug of the loader, or an error that says so, is left for the queue to + // classify by its cause, never marked retryable. + for (const cause of [new TypeError('x is undefined'), { retryable: false, message: 'gone' }]) { + const buggy = createTemplateRenderer({ + engine: fakeEngine(), + loader: { + load: () => { + throw cause; + }, + }, + }); + const error = await buggy.render({ template: 'x' }).catch((e) => e); + assert.ok(error instanceof TemplateLoadError); + assert.ok(!('retryable' in error)); + } + // A loader that throws an error of ./templates throws it as it is. + const refusal = new TemplateNotFoundError('x', []); + const own = createTemplateRenderer({ + engine: fakeEngine(), + loader: { + load: () => { + throw refusal; + }, + }, + }); + assert.strictEqual(await own.render({ template: 'x' }).catch((e) => e), refusal); }); test('Templates: finish makes the HTML of the outermost template, then the CSS is inlined', async () => { From ba1d0fd1415dff9f3a6c2dc0f63edef854b899b2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:35:39 +0300 Subject: [PATCH 26/83] fix(bullmq): tell the audit of a job BullMQ gives up itself A job BullMQ failed itself, stalled more than maxStalledCount, was only logged (queue.job.lost), so its audit trail ended at submitted though its run may have sent it. A driver's consume() takes the sink to tell (QueueConsumeOptions.audit, which a mail queue's worker passes), and the BullMQ driver appends delivery_unknown (queue.lost) for such a job, by its correlationId, which a sealed job now keeps in clear for it. The audit guide shows how to find the messages to reconcile. ./bullmq goes from 3.8 to 4.2 KB min+gzip: its budget goes from 4 to 4.5 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++-- docs/guide/audit.md | 25 +++++++++++++++++++++++ docs/guide/queue.md | 2 +- docs/transports/bullmq.md | 5 +++++ index.d.ts | 6 ++++++ scripts/size-budgets.json | 2 +- src/bullmq/bullmq-driver.js | 33 +++++++++++++++++++++++++++++++ src/queue/envelope.js | 5 +++-- src/queue/mail-queue.js | 1 + tests/unit/bullmq.test.js | 23 +++++++++++++++++++++ tests/unit/queue-security.test.js | 5 +++-- 11 files changed, 105 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33061db..0ca455e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,14 +54,16 @@ dependency, BullMQ, MongoDB and the template engines being passed in. rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the application's key) seals the fields of each job in its store, its addresses, context and attachments, leaving the version, hash, signature and trace context in clear; a worker without the codec gives a sealed job back - (`queue.codec_missing`), and one whose codec fails retries it (`queue.codec_failed`). `idSecret` derives the id of a job from its + (`queue.codec_missing`), and one whose codec fails retries it (`queue.codec_failed`); its + `correlationId` stays in clear too. `idSecret` derives the id of a job from its key with an HMAC, so an id that goes into the audit and the logs cannot be traced back to a key that names a person. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases that spend no attempt. `createBullMQProcessor` and `createBullMQBackoffStrategy` serve a Worker - of your own. `pnpm test:bullmq` runs it on BullMQ and Valkey. + of your own. A job BullMQ gives up itself (stalled too often) is told to the queue's audit sink + as `delivery_unknown` (`queue.lost`), so its trail does not end at `submitted`. `pnpm test:bullmq` runs it on BullMQ and Valkey. - **`./mongodb`**: the transactional outbox, `createMongoOutbox({ collection })`, a `QueueSource` in a collection of the MongoDB driver passed in: a job enqueued with `{ transaction: session }` is written in that transaction (by an upsert, so a duplicate does diff --git a/docs/guide/audit.md b/docs/guide/audit.md index 5ae6d54..e1aec85 100644 --- a/docs/guide/audit.md +++ b/docs/guide/audit.md @@ -86,3 +86,28 @@ await commit(); `failed`; `deferred` keeps a message accepted; an open, a click and an unsubscribe are engagement, not states. The queue writes `correlationId` into the metadata of each message, which the providers give back with its events. + +## Messages to reconcile + +A message whose last state is `submitted` long after its send, or +`delivery_unknown`, may have gone or not: its worker died after the request, +a timeout came after the provider took it, or BullMQ gave the job up after it +stalled too often (the driver tells the queue's sink `delivery_unknown`, code +`queue.lost`). Find them, then settle each from the provider's webhooks or its +dashboard rather than send it again. On MongoDB: + +```js +const stale = await db + .collection('mail_audit') + .aggregate([ + { $sort: { correlationId: 1, at: 1, attempt: 1, rank: 1 } }, + { $group: { _id: '$correlationId', state: { $last: '$state' }, at: { $last: '$at' } } }, + { + $match: { + state: { $in: ['submitted', 'delivery_unknown'] }, + at: { $lt: new Date(Date.now() - 15 * 60_000) }, + }, + }, + ]) + .toArray(); +``` diff --git a/docs/guide/queue.md b/docs/guide/queue.md index beddd04..99d65f3 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -145,7 +145,7 @@ const queue = createMailQueue({ driver, codec, signingKeys }); ``` A sealed job keeps in clear only `v`, `hash` (a duplicate is still told by -it), `enqueuedAt`, `kid`, `mac` and `trace`; the rest is `enc`. Its workers +it), `enqueuedAt`, `correlationId`, `kid`, `mac` and `trace`; the rest is `enc`. Its workers take the queue's codec. A worker without it gives the job back, its attempts not spent, for a worker that can open it (`queue.codec_missing`); one whose codec throws (a key out of reach) retries it (`queue.codec_failed`), and a diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index fdd2d39..46f9cb5 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -39,6 +39,11 @@ it holds only while the first job is not finished. A completed job keeps only `{ v, hash }` of its data (`job.updateData`, by `retainData: 'failed'`, the default; `'all'` or `'none'`), so its message does not wait a day in Redis. +A job BullMQ fails itself, which the handler never settled (stalled more than +`maxStalledCount`), is logged as `queue.job.lost` and told to the queue's audit +sink as `delivery_unknown` (code `queue.lost`): its run may have sent it before +it stalled. + ## The workers it makes `queue.createWorker()` makes a BullMQ `Worker` with your `workerOptions` diff --git a/index.d.ts b/index.d.ts index c4bf075..db64739 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1552,6 +1552,12 @@ export interface QueueConsumeOptions { /** The jobs run at once. @default 1 */ readonly concurrency?: number; readonly signal?: AbortSignal; + /** + * The sink a driver tells of a job it gave up itself, which the handler + * never settled (BullMQ's job stalled too often): `delivery_unknown`. A + * mail queue's worker passes the queue's. + */ + readonly audit?: AuditSink; } /** The options of closing a consumer. */ export interface QueueCloseOptions { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 445a425..3b30f76 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -38,6 +38,6 @@ "./tee": 3.5, "./audit": 2, "./queue": 18, - "./bullmq": 4, + "./bullmq": 4.5, "./mongodb": 6 } diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index 027e011..4188481 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -4,7 +4,10 @@ // it; its attempts and backoff are the queue's; its priority is ranked from // 1, as 0 in BullMQ runs before every prioritized job. +const { warnThrown } = require('../internal/callback.js'); const { createLog } = require('../internal/log.js'); +const { redactAddresses } = require('../internal/redact-addresses.js'); +const { toUuid } = require('../internal/uuid.js'); const { checkPositiveInteger } = require('../internal/options.js'); const { checkRetainData, retentionOf } = require('../internal/queue-options.js'); const { settledWithin } = require('../internal/settle.js'); @@ -116,6 +119,33 @@ const duplicateOf = async (queue, request) => { return answer; }; +// Tells the audit of a job BullMQ gave up itself, which the handler never +// settled: it may have been sent, by a run that stalled after its send. +const auditLost = async (audit, job, error) => { + const attempt = Math.max(1, job?.attemptsMade ?? 1); + const event = { + id: toUuid(`${job?.id}\u0000${attempt}\u0000delivery_unknown`), + state: 'delivery_unknown', + at: new Date().toISOString(), + source: 'worker', + correlationId: + typeof job?.data?.correlationId === 'string' ? job.data.correlationId : String(job?.id), + jobId: String(job?.id), + attempt, + error: { + category: 'unknown', + code: 'queue.lost', + message: redactAddresses(String(error?.message ?? 'lost')).slice(0, 500), + retryable: false, + }, + }; + try { + await audit.append(event); + } catch (thrown) { + warnThrown('The audit sink of a BullMQ queue', thrown, 'MAILER_AUDIT_FAILED'); + } +}; + const startWorker = (config, queue, handler, consumeOptions, log) => { const { Worker } = config.bullmq; if (typeof Worker !== 'function') { @@ -158,6 +188,9 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { // handler never saw. if (!isSettledError(error)) { log.write('error', () => ({ jobId: job?.id, reason: error?.message }), 'queue.job.lost'); + if (typeof consumeOptions.audit?.append === 'function') { + auditLost(consumeOptions.audit, job, error); + } } }); worker.on('stalled', (jobId) => log.write('warn', () => ({ jobId }), 'queue.job.stalled')); diff --git a/src/queue/envelope.js b/src/queue/envelope.js index 13e6c0f..3324cbe 100644 --- a/src/queue/envelope.js +++ b/src/queue/envelope.js @@ -8,8 +8,9 @@ const { messageOf } = require('../internal/decorator.js'); const { MailQueueError } = require('./errors.js'); -// The fields a sealed job keeps in clear, beside `enc`. -const ENVELOPE = new Set(['v', 'hash', 'enqueuedAt', 'mac', 'kid', 'trace']); +// The fields a sealed job keeps in clear, beside `enc`: an id of its message +// among them, by which a driver that gives a job up tells the audit. +const ENVELOPE = new Set(['v', 'hash', 'enqueuedAt', 'correlationId', 'mac', 'kid', 'trace']); /** * Checks a codec: `encode(text)` and `decode(text)`, sync or async, each diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 05a792b..c9dc67e 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -214,6 +214,7 @@ const createMailQueue = (options = {}) => { const consumer = driver.consume(handler, { concurrency: workerOptions.concurrency, signal: workerOptions.signal, + audit: workerOptions.audit ?? queue.audit, }); return { close: (closeOptions) => consumer.close(closeOptions), diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index ac2ef97..a7f8306 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -472,3 +472,26 @@ test('BullMQ: the retention of jobs is the queue’s, as on every driver', async /dedupeWindowSeconds/, ); }); + +test('BullMQ: a job BullMQ gives up itself is told to the audit as of unknown delivery', async () => { + const { createMemoryAuditSink } = require('../../src/audit/index.js'); + const bullmq = fakeBullMQ(); + const audit = createMemoryAuditSink(); + const driver = createBullMQDriver({ bullmq }); + const consumer = driver.consume(async () => ({ type: 'ack' }), { audit }); + const [worker] = bullmq.workers; + const job = fakeJob({ data: { v: 1, correlationId: 'order-42' }, attemptsMade: 1 }); + worker.emit('failed', job, new Error('job stalled more than allowable limit')); + await new Promise(setImmediate); + const [event] = audit.events; + assert.deepStrictEqual( + [event.state, event.correlationId, event.jobId, event.attempt, event.error.code], + ['delivery_unknown', 'order-42', 'j1', 1, 'queue.lost'], + ); + // What the processor threw itself is the handler's, told already. + const { retryError } = require('../../src/bullmq/backoff.js'); + worker.emit('failed', job, retryError({ code: 'x', reason: 'y', delayMilliseconds: 1 })); + await new Promise(setImmediate); + assert.strictEqual(audit.events.length, 1); + await consumer.close(); +}); diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index b176eda..2036a0f 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -176,6 +176,7 @@ test('Queue: with a codec the store holds no address, and the worker opens the j const enqueued = await queue.enqueue(withFile); const [stored] = driver.list(); assert.deepStrictEqual(Object.keys(stored.data).toSorted(), [ + 'correlationId', 'enc', 'enqueuedAt', 'hash', @@ -257,8 +258,8 @@ test('Queue: what a codec opens is checked, and a field in clear is refused', as const opened = { ...JSON.parse(signed.enc), ...signed }; delete opened.enc; const forged = forge(opened); - const { v, hash, enqueuedAt, kid, mac, ...fields } = forged; - const resealed = { v, hash, enqueuedAt, kid, mac, enc: JSON.stringify(fields) }; + const { v, hash, enqueuedAt, correlationId, kid, mac, ...fields } = forged; + const resealed = { v, hash, enqueuedAt, correlationId, kid, mac, enc: JSON.stringify(fields) }; const verifying = createMailJobHandler({ transport: fakeTransport(), codec: identity, From f10c135960e702b43c02919630231f167735b5ec Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:36:47 +0300 Subject: [PATCH 27/83] fix(templates): refuse an engine option that turns off escaping Handlebars `noEscape`, Nunjucks `autoescape: false`, a Liquid `outputEscape` other than 'escape' (`undefined` included, which replaced the default), Eta `autoEscape: false` or `escapeFunction`, and an EJS `escape` went through to the engine, which then wrote every value of a context raw into the HTML. Each throws a TypeError unless the engine is given `allowUnescaped: true`. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 7 ++++ src/templates/ejs.js | 13 ++++++- src/templates/eta.js | 13 ++++++- src/templates/handlebars.js | 4 ++ src/templates/index.d.ts | 30 +++++++++++++++ src/templates/liquid.js | 5 +++ src/templates/nunjucks.js | 6 +++ src/templates/options.js | 37 +++++++++++++++++- tests/types/templates.test-d.ts | 2 + tests/unit/templates-engines.test.js | 57 ++++++++++++++++++++++++++++ 10 files changed, 171 insertions(+), 3 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index a837d75..3be8b9c 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -148,6 +148,13 @@ option rather than accepted and ignored. Several engines go into one renderer with `engines: [...]`, chosen by extension, the longest first. +Output is escaped, and no option of an engine turns that off unless the engine +is given `allowUnescaped: true`: Handlebars `noEscape`, Nunjucks `autoescape: +false`, a Liquid `outputEscape` other than `'escape'`, Eta `autoEscape: false` +or an `escapeFunction`, and an EJS `escape` throw a `TypeError` without it. +The raw tags of each engine (`{{{ }}}`, `<%- %>`, `| raw`, `| safe`) stay +yours to use where a value is HTML you trust. + Some engines behave a little differently here than they do on their own: - **Handlebars** gets an environment per renderer (`handlebars.create()`), so diff --git a/src/templates/ejs.js b/src/templates/ejs.js index f0853ff..bdccebe 100644 --- a/src/templates/ejs.js +++ b/src/templates/ejs.js @@ -4,9 +4,19 @@ // its options. @nestjs-modules/mailer let an async template's promise reject // unheard, which hung the send. -const { checkEngineExtensions, checkEngineOptions, checkModule } = require('./options.js'); +const { + checkEngineExtensions, + checkEngineOptions, + checkEscaping, + checkModule, +} = require('./options.js'); const RESERVED = new Set(['filename', 'root', 'views', 'includer', 'cache', 'client']); +// The options of ejs.compile that replace its escaping. +const UNESCAPED = new Map([ + ['escape', () => true], + ['escapeFunction', () => true], +]); /** * Makes the EJS engine. @@ -23,6 +33,7 @@ const createEjsEngine = (options = {}) => { 'createEjsEngine needs ejs: the module of the ejs package.', ); const given = checkEngineOptions(options.options, RESERVED, 'ejs'); + checkEscaping(options, given, UNESCAPED, 'ejs'); const extensions = checkEngineExtensions(options.extensions, ['.ejs']); return { name: 'ejs', diff --git a/src/templates/eta.js b/src/templates/eta.js index 7f8b414..8b3dcf1 100644 --- a/src/templates/eta.js +++ b/src/templates/eta.js @@ -4,9 +4,19 @@ // template it is written in; output is escaped, as Eta does by default. const { TemplateNotFoundError } = require('./errors.js'); -const { checkEngineExtensions, checkEngineOptions, checkModule } = require('./options.js'); +const { + checkEngineExtensions, + checkEngineOptions, + checkEscaping, + checkModule, +} = require('./options.js'); const RESERVED = new Set(['views', 'cache', 'cacheFilepaths', 'defaultExtension']); +// The options of Eta that turn off its escaping, or replace it. +const UNESCAPED = new Map([ + ['autoEscape', (value) => !value], + ['escapeFunction', () => true], +]); /** * Makes the Eta engine. @@ -24,6 +34,7 @@ const createEtaEngine = (options = {}) => { 'createEtaEngine needs Eta: the Eta of the eta package.', ); const given = checkEngineOptions(options.options, RESERVED, 'eta'); + checkEscaping(options, given, UNESCAPED, 'eta'); const async = options.async ?? false; if (typeof async !== 'boolean') throw new TypeError('async must be a boolean.'); const extensions = checkEngineExtensions(options.extensions, ['.eta']); diff --git a/src/templates/handlebars.js b/src/templates/handlebars.js index b1c0dc2..4b76c84 100644 --- a/src/templates/handlebars.js +++ b/src/templates/handlebars.js @@ -8,12 +8,15 @@ const { checkEngineExtensions, checkEngineOptions, + checkEscaping, checkFunctions, checkModule, } = require('./options.js'); const { checkOptionalFunction } = require('../internal/options.js'); const PARTIALS = new Set(['PartialStatement', 'PartialBlockStatement']); +// The options of compile that turn off its escaping. +const UNESCAPED = new Map([['noEscape', Boolean]]); const partialNameOf = (name) => { if (name?.type === 'PathExpression') return name.original; @@ -85,6 +88,7 @@ const createHandlebarsEngine = (options = {}) => { ); const helpers = checkFunctions(options.helpers, 'helpers'); const given = checkEngineOptions(options.options, new Set(), 'handlebars'); + checkEscaping(options, given, UNESCAPED, 'handlebars'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.hbs', '.handlebars']); // The environment of each renderer, keyed by its includes: a new one after diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 6899c59..e023bc4 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -364,6 +364,12 @@ export interface EjsEngineOptions { * are the engine's, and throw. */ readonly options?: Readonly>; + /** + * Lets `escape` and `escapeFunction`, which replace it, turn off the escaping of output. Without it, they throw: the + * values of a context, which a user may have written, would go into the + * HTML raw. @default false + */ + readonly allowUnescaped?: boolean; /** @default ['.ejs'] */ readonly extensions?: readonly string[]; } @@ -393,6 +399,12 @@ export interface HandlebarsEngineOptions { readonly helpers?: Readonly unknown>>; /** Of compile: `strict`, `knownHelpers`, `knownHelpersOnly`, `preventIndent`, `noEscape`. */ readonly options?: Readonly>; + /** + * Lets `noEscape: true` turn off the escaping of output. Without it, they throw: the + * values of a context, which a user may have written, would go into the + * HTML raw. @default false + */ + readonly allowUnescaped?: boolean; /** Called with each environment the engine makes: decorators, more helpers. */ readonly configure?: (environment: any) => void; /** @default ['.hbs', '.handlebars'] */ @@ -483,6 +495,12 @@ export interface LiquidEngineOptions { readonly options?: Readonly>; readonly filters?: Readonly unknown>>; readonly configure?: (liquid: any) => void; + /** + * Lets an `outputEscape` other than `'escape'` turn off the escaping of output. Without it, they throw: the + * values of a context, which a user may have written, would go into the + * HTML raw. @default false + */ + readonly allowUnescaped?: boolean; /** @default ['.liquid'] */ readonly extensions?: readonly string[]; } @@ -511,6 +529,12 @@ export interface NunjucksEngineOptions { readonly nunjucks: NunjucksModule; /** `autoescape` (true), `throwOnUndefined`, `trimBlocks`, `lstripBlocks`, `tags`. */ readonly options?: Readonly>; + /** + * Lets a falsy `autoescape` turn off the escaping of output. Without it, they throw: the + * values of a context, which a user may have written, would go into the + * HTML raw. @default false + */ + readonly allowUnescaped?: boolean; readonly filters?: Readonly unknown>>; readonly configure?: (environment: any) => void; /** @default ['.njk', '.nunjucks'] */ @@ -529,6 +553,12 @@ export interface EtaEngineOptions { * `defaultExtension` are the engine's, and throw. */ readonly options?: Readonly>; + /** + * Lets `autoEscape: false` and an `escapeFunction` of your own turn off the escaping of output. Without it, they throw: the + * values of a context, which a user may have written, would go into the + * HTML raw. @default false + */ + readonly allowUnescaped?: boolean; /** Compiles to async functions, for `await` and `includeAsync`. @default false */ readonly async?: boolean; /** @default ['.eta'] */ diff --git a/src/templates/liquid.js b/src/templates/liquid.js index 579dd04..f02bfaa 100644 --- a/src/templates/liquid.js +++ b/src/templates/liquid.js @@ -7,6 +7,7 @@ const { TemplateNotFoundError } = require('./errors.js'); const { checkEngineExtensions, checkEngineOptions, + checkEscaping, checkFunctions, checkModule, } = require('./options.js'); @@ -23,6 +24,9 @@ const RESERVED = new Set([ 'cache', ]); +// The options of liquidjs that turn off its escaping, or replace it. +const UNESCAPED = new Map([['outputEscape', (value) => value !== 'escape']]); + // A name no loader has: where a reference leads out of the root, and, before // the name written, where liquidjs looks last for a template it did not find. const NOWHERE = '\u0000'; @@ -115,6 +119,7 @@ const createLiquidEngine = (options = {}) => { 'createLiquidEngine needs Liquid: the Liquid of the liquidjs package.', ); const given = checkEngineOptions(options.options, RESERVED, 'liquid'); + checkEscaping(options, given, UNESCAPED, 'liquid'); const filters = checkFunctions(options.filters, 'filters'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.liquid']); diff --git a/src/templates/nunjucks.js b/src/templates/nunjucks.js index 8fa2d04..b73e056 100644 --- a/src/templates/nunjucks.js +++ b/src/templates/nunjucks.js @@ -8,6 +8,7 @@ const { TemplateNotFoundError } = require('./errors.js'); const { checkEngineExtensions, checkEngineOptions, + checkEscaping, checkFunctions, checkModule, } = require('./options.js'); @@ -15,6 +16,10 @@ const { checkOptionalFunction } = require('../internal/options.js'); const isRelative = (name) => name.startsWith('./') || name.startsWith('../'); +// The options of the Environment that turn off its escaping: a falsy +// `autoescape`, null and undefined keeping the default. +const UNESCAPED = new Map([['autoescape', (value) => value != null && !value]]); + // What Nunjucks says of a template no loader has. const NOT_FOUND = 'template not found: '; // The failed lookups an environment keeps, to tell which names were tried. @@ -58,6 +63,7 @@ const createNunjucksEngine = (options = {}) => { ); checkModule(nunjucks.Template, undefined, 'nunjucks has no Template.'); const given = checkEngineOptions(options.options, new Set(), 'nunjucks'); + checkEscaping(options, given, UNESCAPED, 'nunjucks'); const filters = checkFunctions(options.filters, 'filters'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.njk', '.nunjucks']); diff --git a/src/templates/options.js b/src/templates/options.js index 3155531..37a2713 100644 --- a/src/templates/options.js +++ b/src/templates/options.js @@ -28,6 +28,35 @@ const checkEngineOptions = (options, reserved, engine) => { return { ...options }; }; +/** + * Refuses the options that turn off the escaping of an engine's output, or + * put an escaper of the application's in its place, unless the engine is + * given `allowUnescaped: true`: a template whose context a user wrote would + * otherwise put that user's HTML into the message. + * + * @param options The options of the engine. + * @param given The options passed on to the engine. + * @param switches By the name of each such option, whether a value turns the + * escaping off. + * @param engine The name of the engine, for the message. + * @throws {TypeError} If one is given without `allowUnescaped`. + * @internal + */ +const checkEscaping = (options, given, switches, engine) => { + const allowed = options.allowUnescaped ?? false; + if (typeof allowed !== 'boolean') throw new TypeError('allowUnescaped must be a boolean.'); + if (allowed) return; + for (const name of Object.keys(given)) { + const turnsOff = switches.get(name); + if (turnsOff !== undefined && turnsOff(given[name])) { + throw new TypeError( + `${name} turns off the escaping of the ${engine} engine: pass it allowUnescaped: true ` + + 'to allow that.', + ); + } + } +}; + /** * Checks the extensions of an engine. * @@ -76,4 +105,10 @@ const checkModule = (value, member, message) => { return value; }; -module.exports = { checkEngineExtensions, checkEngineOptions, checkFunctions, checkModule }; +module.exports = { + checkEngineExtensions, + checkEngineOptions, + checkEscaping, + checkFunctions, + checkModule, +}; diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index 7fe62d9..dc15822 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -62,6 +62,8 @@ createMustacheEngine({ mustache, tags: ['<%', '%>'] }); createNunjucksEngine({ nunjucks, filters: { upper: (s: string) => s.toUpperCase() } }); createMjmlEngine({ mjml2html, engine: hbs, includes: false, onErrors: (errors) => errors.length }); expectError(createEjsEngine({})); +createHandlebarsEngine({ handlebars, options: { noEscape: true }, allowUnescaped: true }); +expectError(createEtaEngine({ Eta, allowUnescaped: 'yes' })); expectError(createMustacheEngine({ mustache, tags: ['<%'] })); declare const render: (element: { type: string }) => Promise; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 9d51a9b..5820c1d 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -117,6 +117,63 @@ test('Templates: an async EJS template that throws rejects, rather than hanging assert.strictEqual(error.phase, 'render'); }); +test('Templates: no engine option turns off escaping unless allowUnescaped', async () => { + // Before: noEscape, autoescape: false, an outputEscape of one's own and + // autoEscape: false went through, and every value was written raw. + const refused = [ + () => createHandlebarsEngine({ handlebars, options: { noEscape: true } }), + () => createNunjucksEngine({ nunjucks, options: { autoescape: false } }), + () => createLiquidEngine({ Liquid, options: { outputEscape: undefined } }), + () => createLiquidEngine({ Liquid, options: { outputEscape: (value) => value } }), + () => createEtaEngine({ Eta, options: { autoEscape: false } }), + () => createEtaEngine({ Eta, options: { escapeFunction: String } }), + () => createEjsEngine({ ejs, options: { escape: String } }), + ]; + for (const make of refused) assert.throws(make, /turns off the escaping .* allowUnescaped: true/); + assert.throws( + () => createHandlebarsEngine({ handlebars, allowUnescaped: 'yes' }), + /allowUnescaped must be a boolean/, + ); + // What keeps escaping on, or keeps the default, passes. + createHandlebarsEngine({ handlebars, options: { noEscape: false } }); + createNunjucksEngine({ nunjucks, options: { autoescape: true } }); + createNunjucksEngine({ nunjucks, options: { autoescape: null } }); + createLiquidEngine({ Liquid, options: { outputEscape: 'escape' } }); + createEtaEngine({ Eta, options: { autoEscape: true } }); + const context = { name: '' }; + const allowed = [ + [ + createHandlebarsEngine({ handlebars, options: { noEscape: true }, allowUnescaped: true }), + 'a.hbs', + '{{name}}', + ], + [ + createNunjucksEngine({ nunjucks, options: { autoescape: false }, allowUnescaped: true }), + 'a.njk', + '{{ name }}', + ], + [ + createLiquidEngine({ Liquid, options: { outputEscape: undefined }, allowUnescaped: true }), + 'a.liquid', + '{{ name }}', + ], + [ + createEtaEngine({ Eta, options: { autoEscape: false }, allowUnescaped: true }), + 'a.eta', + '<%= it.name %>', + ], + [ + createEjsEngine({ ejs, options: { escape: String }, allowUnescaped: true }), + 'a.ejs', + '<%= name %>', + ], + ]; + for (const [engine, name, source] of allowed) { + const output = await renderWith(engine, { [name]: source }, { template: 'a', context }); + assert.strictEqual(output.html, '', engine.name); + } +}); + test('Templates: the EJS engine refuses what it sets itself', () => { assert.throws(() => createEjsEngine({}), /needs ejs/); for (const name of ['filename', 'root', 'views', 'includer', 'cache', 'client']) { From 31bbf053db3e4d47e3ae84e1dde665146cec235a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:37:10 +0300 Subject: [PATCH 28/83] fix(queue): keep the text of a failed render out of the store and the audit A template that failed to render put its error message, which may hold what a helper threw, an excerpt of the template or a value of the context, into the failure of the job in the store and into the audit event. Only the template, the phase and the engine go there now; the full text goes to the log at debug, its addresses redacted. A template not found keeps its message, which names only templates. The renderer is given the job's signal (MailRenderInput.signal), so a worker that stops can stop a render that hears it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- docs/guide/queue.md | 2 +- src/queue/handler.js | 26 +++++++++++++++- src/queue/index.d.ts | 2 ++ tests/unit/queue-handler.test.js | 52 +++++++++++++++++++++++++++++++- 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ca455e..4114c30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. for every attempt. The worker fails a permanent failure, a missing template or a bug at once, retries a temporary one never sooner than `Retry-After`, an operator's failure (`auth`, `configuration`) after five - minutes, and never sends again a message of unknown delivery. `scheduledAt` is a delay of the + minutes, and never sends again a message of unknown delivery. The renderer is given the job's + `signal`; a template that fails to render leaves only its name, phase and engine in the store and + the audit, its text in the log at `debug`. `scheduledAt` is a delay of the queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 99d65f3..1522137 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -194,7 +194,7 @@ rotated, as another secret makes other ids and the same message another job. | a retryable failure, the last attempt | failed (`queue.attempts_exhausted`) | `failed` | | a refused message (`validation`, `rejected`, `unsupported`) | failed at once | `failed` | | a failure of the operator (`auth`, `configuration`) | retried after `operatorDelayMilliseconds` (5 minutes) | `queued` | -| a template missing or broken, no subject, no renderer | failed at once (`render.*`) | `failed` | +| a template missing or broken, no subject, no renderer | failed at once (`render.*`); a broken one's reason names the template and where it failed, its text (a helper's error, an excerpt, a value of the context) going to the log at `debug` only | `failed` | | an outage of the template loader or of `resolveAttachment` | retried | `queued` | | a bug: a `TypeError` thrown by the transport | failed at once (`queue.thrown`), logged with its stack | `failed` | | a job past `expiresAt`, or whose retry would come after it | failed (`queue.expired`) | `failed` | diff --git a/src/queue/handler.js b/src/queue/handler.js index 40c4b2a..a42edb2 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -248,6 +248,28 @@ const settleThrown = (run, error, code) => { return settleFailure(run, receipt, code); }; +// A template that fails to render tells what a helper or filter threw, an excerpt of +// its source, or a value of the context: none of that goes into the store of +// the queue or the audit, only which template failed and where, the full +// text to the log at debug. +const templateFailureOf = (run, error) => { + // A template not found names only templates. + if (error?.code !== 'MAILER_TEMPLATE_RENDER') return error; + run.config.log.write( + 'debug', + () => ({ jobId: run.job.id, code: error.code, reason: reasonOf(messageOf(error)) }), + 'queue.render.failed', + ); + const where = error.phase === undefined ? '' : ` (${error.phase})`; + const engine = error.engine === undefined ? '' : ` with ${error.engine}`; + return Object.assign( + new Error( + `The template ${JSON.stringify(error.template ?? run.data.template)} failed${where}${engine}.`, + ), + { code: error.code, retryable: error.retryable, name: error.name }, + ); +}; + const renderStep = async (run) => { const { config, data } = run; if (data.kind === 'content') return { fields: { subject: data.subject, content: data.content } }; @@ -265,10 +287,12 @@ const renderStep = async (run) => { context: data.context, locale: data.locale, tenantId: data.tenantId, + // A worker that stops stops its render; a renderer may hear it. + signal: run.signal, }); } catch (error) { if (run.signal.aborted) return { outcome: release(run, 'aborted while rendering') }; - return { outcome: await settleThrown(run, error, 'render.failed') }; + return { outcome: await settleThrown(run, templateFailureOf(run, error), 'render.failed') }; } const subject = data.subject ?? fields.subject; if (typeof subject !== 'string' || subject === '') { diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 2c71c78..d4c7395 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -280,6 +280,8 @@ export interface MailRenderInput { readonly context: JsonObject; readonly locale?: string; readonly tenantId?: string; + /** Aborts when the worker stops the job: a renderer may stop with it. */ + readonly signal?: AbortSignal; } /** What `classify` may decide of a failed receipt. */ diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index eec7a65..508cb8f 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -144,8 +144,15 @@ test('Queue: a content job is sent as it is; the subject of the job wins over th const { handler: spied } = handlerOf({ renderer: spy }); await run(spied, jobOf({ locale: 'uk', tenantId: 't1' })); assert.deepStrictEqual(inputs, [ - { template: 'welcome', context: { name: 'Ada' }, locale: 'uk', tenantId: 't1' }, + { + template: 'welcome', + context: { name: 'Ada' }, + locale: 'uk', + tenantId: 't1', + signal: inputs[0].signal, + }, ]); + assert.ok(inputs[0].signal instanceof AbortSignal); }); test('Queue: a retryable failure is retried after at least its Retry-After', async () => { @@ -594,3 +601,46 @@ test('Queue: under redelivered: delivery_unknown, a run again after a submit is assert.strictEqual(again.calls.length, 1); assert.throws(() => handlerOf({ redelivered: 'never' }), /redelivered must be/); }); + +test('Queue: a template that fails leaves its text in the log, not the store or the audit', async () => { + const records = []; + const logger = { + debug: (record, message) => records.push([message, record.reason]), + error: () => {}, + }; + const inputs = []; + const renderer = { + render(input) { + inputs.push(input); + throw new TemplateNotFoundError('welcome', ['welcome.hbs']); + }, + }; + const leaky = { + render() { + throw Object.assign(new Error('helper failed on {"card":"4242 4242 4242 4242"}'), { + code: 'MAILER_TEMPLATE_RENDER', + template: 'welcome.uk.hbs', + phase: 'render', + engine: 'handlebars', + retryable: false, + }); + }, + }; + const { handler: missing } = handlerOf({ renderer, logger }); + const signal = new AbortController().signal; + await run(missing, jobOf(), signal); + assert.strictEqual(inputs[0].signal, signal); + const { handler, audit } = handlerOf({ renderer: leaky, logger }); + const outcome = await run(handler, jobOf()); + assert.deepStrictEqual([outcome.type, outcome.code], ['fail', 'render.failed']); + assert.strictEqual( + outcome.reason, + 'The template "welcome.uk.hbs" failed (render) with handlebars.', + ); + const event = audit.events.at(-1); + assert.strictEqual(event.state, 'failed'); + assert.doesNotMatch(JSON.stringify(event), /4242/); + assert.ok( + records.some(([message, reason]) => message === 'queue.render.failed' && /4242/.test(reason)), + ); +}); From 660018138f924b215e1525d8712ab049473dbb5a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:38:21 +0300 Subject: [PATCH 29/83] feat(templates): give Liquid limits by default, and say who may write a template The guide and the types named Liquid, with its limits, the engine for templates a tenant writes, and the engine set none, liquidjs having none of its own: a template could loop or allocate until the process fell over. parseLimit (1 MiB), renderLimit (1000 ms) and memoryLimit (1e8) are on by default, each overridden by the options of the engine. The guide has a matrix of who may write a template in each engine, the front-matter subject included. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 30 +++++++++++++++++++++++---- src/templates/index.d.ts | 11 ++++++---- src/templates/liquid.js | 15 +++++++++++--- tests/unit/templates-engines.test.js | 31 ++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 3be8b9c..f6bfd95 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -185,12 +185,34 @@ Some engines behave a little differently here than they do on their own: const templates = createTemplateRenderer({ engine }); ``` -Templates are code: EJS, Eta, Pug and Nunjucks run JavaScript or close to it. -For templates your tenants write, use Liquid (with `parseLimit`, `renderLimit` -and `memoryLimit`), Handlebars or Mustache. The context is data. - ::: +## Who may write a template + +A template is code that runs in your process, so who writes it decides the +engine. The context is data in every engine, escaped on output; what differs +is what a template itself can do. + +| Engine | A template | Templates a tenant writes | +| --- | --- | --- | +| EJS, Eta | compiles to JavaScript: `<% %>` runs any code, with the rights of the process | no | +| Pug | compiles to JavaScript: `- code` lines and `#{}` run any code | no | +| Nunjucks | compiles to JavaScript and is no sandbox: its own guide calls user-defined templates unsafe, and known escapes reach `Function` through the values a template sees | no | +| React Email | is a component, code of yours | no | +| MJML | is the engine inside it; `mj-include` reads files, so it stays off | as that engine | +| Handlebars | is logic-less: it calls the helpers you register and reads own properties only | yes, with helpers you trust and `knownHelpersOnly` | +| Mustache | is logic-less | yes | +| Liquid | was made for templates users write; parse, render and memory limits are on by default here (`parseLimit` 1 MiB, `renderLimit` 1000 ms, `memoryLimit` 1e8) | yes, the one to choose | + +The subject of a front matter is compiled by the same engine as the template, +under the same rules: a tenant who writes the subject writes a template. + +For templates a tenant writes, keep the bounds of the renderer on: the +includes a render reads, the size of its output, and its timeout (see +[Bounds](#bounds)). A loop of a template that never yields can be stopped by +none of them, only by the engine: Liquid's `renderLimit` is checked as it +renders. + ## CSS ```js diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index e023bc4..4910610 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -488,9 +488,11 @@ export interface LiquidEngineOptions { /** * `outputEscape` ('escape', where liquidjs escapes nothing), * `ownPropertyOnly` (true), `strictFilters`, `strictVariables`, `jsTruthy`, - * `parseLimit`, `renderLimit`, `memoryLimit`. `root`, `partials`, `layouts`, - * `fs`, `extname`, `relativeReference`, `templates` and `cache` are the - * engine's, and throw. + * and the limits, where liquidjs has none: `parseLimit` (1048576, the + * characters of a template), `renderLimit` (1000, the milliseconds of a + * render) and `memoryLimit` (1e8). `root`, `partials`, `layouts`, `fs`, + * `extname`, `relativeReference`, `templates` and `cache` are the engine's, + * and throw. */ readonly options?: Readonly>; readonly filters?: Readonly unknown>>; @@ -507,7 +509,8 @@ export interface LiquidEngineOptions { /** * The Liquid engine: `include`, `render` and `layout` through the loader, from * the root, then the directories of partials, `./x` against the file. With - * its limits, the one engine of these for templates a tenant writes. + * its limits, on by default, the engine of these for templates a tenant + * writes; see the trust matrix of the guide. */ export declare function createLiquidEngine(options: LiquidEngineOptions): TemplateEngine; diff --git a/src/templates/liquid.js b/src/templates/liquid.js index f02bfaa..8afdb86 100644 --- a/src/templates/liquid.js +++ b/src/templates/liquid.js @@ -24,6 +24,13 @@ const RESERVED = new Set([ 'cache', ]); +// The limits of liquidjs, which has none of its own: the characters of a +// template parsed, as many as the file loader reads; the milliseconds of a +// render, which liquidjs checks as it goes, so that a loop of a template stops +// where no timeout can stop it; and what a render may allocate, in liquidjs's +// units (characters, and the items of a range). +const LIMITS = Object.freeze({ parseLimit: 1024 * 1024, renderLimit: 1000, memoryLimit: 1e8 }); + // The options of liquidjs that turn off its escaping, or replace it. const UNESCAPED = new Map([['outputEscape', (value) => value !== 'escape']]); @@ -106,9 +113,10 @@ const fileSystemOf = (includes, roots, extname) => { * Makes the Liquid engine. * * @param options `Liquid`, the `Liquid` of `liquidjs`; `options` of - * liquidjs, such as `strictVariables`, `ownPropertyOnly` (true) or - * `outputEscape` ('escape'); `filters`; `configure(liquid)`; `extensions` - * (`['.liquid']`). + * liquidjs, such as `strictVariables`, `ownPropertyOnly` (true), + * `outputEscape` ('escape'), `parseLimit` (1 MiB), `renderLimit` (1000 ms) or + * `memoryLimit` (1e8); `filters`; `configure(liquid)`; `extensions` + * (`['.liquid']`); `allowUnescaped`. * @returns The engine. * @throws {TypeError} If `Liquid` is not a class or an option is reserved. */ @@ -131,6 +139,7 @@ const createLiquidEngine = (options = {}) => { liquid = new Liquid({ outputEscape: 'escape', ownPropertyOnly: true, + ...LIMITS, ...given, root: [''], partials: roots, diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 5820c1d..6ec6ce2 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -565,6 +565,37 @@ test('Templates: Liquid escapes by default, its includes and layouts through the ); }); +test('Templates: Liquid has limits by default, which its options override', async () => { + // Before: liquidjs's own, none, so a template a tenant wrote could loop or + // allocate until the process fell over. + let liquid; + const engine = createLiquidEngine({ Liquid, configure: (instance) => (liquid = instance) }); + await renderWith(engine, { 'a.liquid': 'x' }, { template: 'a' }); + assert.deepStrictEqual( + [liquid.options.parseLimit, liquid.options.renderLimit, liquid.options.memoryLimit], + [1024 * 1024, 1000, 1e8], + ); + const huge = await failureOf( + renderWith( + engine, + { 'a.liquid': '{% for i in (1..200000000) %}{% endfor %}' }, + { template: 'a' }, + ), + ); + assert.ok(huge instanceof TemplateRenderError); + assert.match(huge.message, /memory alloc limit exceeded/); + const small = createLiquidEngine({ Liquid, options: { memoryLimit: 10 } }); + await assert.rejects( + renderWith(small, { 'a.liquid': '{% for i in (1..100) %}{% endfor %}' }, { template: 'a' }), + /memory alloc limit exceeded/, + ); + const long = createLiquidEngine({ Liquid, options: { parseLimit: 4 } }); + await assert.rejects( + renderWith(long, { 'a.liquid': 'too long' }, { template: 'a' }), + /parse length limit exceeded/, + ); +}); + test('Templates: Liquid reads synchronously too, through its loader', async () => { // liquidjs renders synchronously with existsSync and readFileSync of its // file system, which are the loader's too. From a5000863d93c4c3bf3468c0ed260b0b91db4efd1 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:39:36 +0300 Subject: [PATCH 30/83] perf(mongodb): index when the next job of the outbox is due When no job was ready, reserve read when the next one is due, sorted by availableAt alone, which the index of the ready jobs (by priority first) cannot give: MongoDB scanned and sorted every ready job on each empty poll of each lane, a long schedule of delayed jobs making it O(n). ensureIndexes makes { state, availableAt } (mailer_due); the opt-in suite checks by explain() that the plan has no sort. Co-Authored-By: Claude Opus 5.5 --- docs/guide/outbox.md | 2 +- src/mongodb/index.d.ts | 2 +- src/mongodb/outbox.js | 3 +++ tests/mongodb/outbox.mongodb.test.js | 22 ++++++++++++++++++++++ tests/unit/mongodb.test.js | 1 + 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 0afbc1e..e1ccfbc 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -88,7 +88,7 @@ is only the fallback. | Collection | Index | | --- | --- | -| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; a TTL on `expireAt` | +| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; `{ state, availableAt }` for the next one due; a TTL on `expireAt` | | audit | `{ correlationId, at, attempt, rank }`; `{ jobId }` | | idempotency | a TTL on `expiresAt` | diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index 9654ded..02d2118 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -86,7 +86,7 @@ export interface MongoOutboxJob { export interface MongoOutbox extends QueueDriver { readonly transactional: true; readonly source: QueueSource; - /** Makes the indexes of the outbox: the ready jobs, the leases, and the expiry of finished ones (`expireAt`). */ + /** Makes the indexes of the outbox: the ready jobs, the leases, the next one due, and the expiry of finished ones (`expireAt`). */ ensureIndexes(): Promise; get(id: string): Promise; } diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index a4db4e6..c81d18d 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -280,6 +280,9 @@ const createMongoOutbox = (options = {}) => { { name: 'mailer_ready' }, ); await collection.createIndex({ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }); + // When the next job is due, read when none is: sorted by time alone, + // which the index of the ready jobs, by priority first, cannot give. + await collection.createIndex({ state: 1, availableAt: 1 }, { name: 'mailer_due' }); // A finished job goes at its time, a completed one and with it the // memory of its key after the window of duplicates, a failed one later. await collection.createIndex( diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index 7051cbe..b8d6af3 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -220,3 +220,25 @@ sourceContract( }, { skip }, ); + +test( + 'MongoDB (replica set): when the next job is due is read from an index, not a sort', + { skip }, + async (t) => { + const { outbox, queue, db } = await setup(t); + for (let index = 0; index < 50; index++) { + await queue.enqueue(job({ scheduledAt: new Date(Date.now() + 60_000 + index) })); + } + assert.ok(outbox); + const plan = await db + .collection('mail_outbox') + .find({ state: 'ready' }) + .sort({ availableAt: 1 }) + .project({ availableAt: 1 }) + .limit(1) + .explain(); + const text = JSON.stringify(plan.queryPlanner.winningPlan); + assert.match(text, /IXSCAN/); + assert.doesNotMatch(text, /"stage":"SORT"/); + }, +); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index ce9159b..9450a64 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -184,6 +184,7 @@ test('MongoDB: the outbox makes its indexes, and wakes its workers on a change s assert.deepStrictEqual(collection.indexes, [ [{ state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, { name: 'mailer_ready' }], [{ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }], + [{ state: 1, availableAt: 1 }, { name: 'mailer_due' }], [{ expireAt: 1 }, { name: 'mailer_expire', expireAfterSeconds: 0 }], ]); let woken = 0; From b757bc2822b27b80879e0bc2eb153749225c964b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:43:01 +0300 Subject: [PATCH 31/83] perf(mongodb): take back leases and read the pause once a while Every reserve of every lane of every worker read the pause of the queue and wrote an updateMany on the primary to take back the leases that ran out, before taking a job and reading when the next is due: four round trips an idle poll, 4 x workers x lanes a second. A process takes back leases once a third of the shortest lease (a lease cannot run out sooner), and reads the pause once a poll interval, a pause of its own at once: an idle poll is the findOneAndUpdate and the read of the next due time. Co-Authored-By: Claude Opus 5.5 --- docs/guide/outbox.md | 5 +++- src/mongodb/outbox.js | 52 +++++++++++++++++++++++++++----------- tests/unit/mongodb.test.js | 27 ++++++++++++++++++++ 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index e1ccfbc..81bd2d5 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -78,7 +78,10 @@ Leases are written by the clock of each worker, so a lease is taken back only clock is behind still holds it; keep the clocks of the workers in sync (NTP), within that tolerance, and the lease more than three times longer. -Workers poll (`pollIntervalMilliseconds`, a second). With `changeStream: true`, +Workers poll (`pollIntervalMilliseconds`, a second). An idle poll of a lane is +one `findOneAndUpdate` and one read of when the next job is due: a process +takes back the leases that ran out once a third of a lease, and reads the +pause of the queue once a poll interval (a pause of its own at once). With `changeStream: true`, on a replica set, a change of the collection wakes them at once, and the poll is only the fallback. diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index c81d18d..3192db6 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -74,6 +74,10 @@ class MongoOutboxSource { id = 'mongodb'; #collection; #config; + // When this process last took back the leases that ran out, and read the + // pause of the queue: once a while, not on each reserve of each lane. + #reclaimedAt = -Infinity; + #pause = { readAt: -Infinity, until: 0 }; constructor(collection, config) { this.#collection = collection; @@ -114,23 +118,29 @@ class MongoOutboxSource { return answer; } - async #paused(now) { - const pause = await this.#collection.findOne( - { _id: PAUSE_ID }, - { projection: { pausedUntil: 1 } }, - ); - const until = pause?.pausedUntil; - return until instanceof Date && until > now ? until : undefined; + // Until when the queue is paused: read again once a poll interval, a pause + // of another process seen within it, one of this process at once. + async #pausedUntil(now) { + if (now - this.#pause.readAt >= this.#config.pauseReadMilliseconds) { + const pause = await this.#collection.findOne( + { _id: PAUSE_ID }, + { projection: { pausedUntil: 1 } }, + ); + const until = pause?.pausedUntil; + this.#pause = { readAt: now, until: until instanceof Date ? until.getTime() : 0 }; + } + return this.#pause.until; } - async reserve(leaseMilliseconds) { - const now = new Date(this.#config.now()); - const paused = await this.#paused(now); - if (paused !== undefined) return { nextAvailableAt: paused.getTime() }; - // A job whose lease ran out is ready again, its attempt to run again: - // by this worker's clock, as long ago as two clocks may be apart, lest a - // worker whose clock is behind still holds it. - const ranOut = new Date(now.getTime() - this.#config.clockSkewToleranceMilliseconds); + // A job whose lease ran out is ready again, its attempt to run again: by + // this worker's clock, as long ago as two clocks may be apart, lest a worker + // whose clock is behind still holds it. Once a third of a lease, as a lease + // cannot run out sooner, a write on the primary saved on every other poll. + async #reclaim(now, leaseMilliseconds) { + const every = Math.min(this.#config.leaseMilliseconds, leaseMilliseconds) / 3; + if (now - this.#reclaimedAt < every) return; + this.#reclaimedAt = now; + const ranOut = new Date(now - this.#config.clockSkewToleranceMilliseconds); await this.#collection.updateMany( { state: 'active', leaseUntil: { $lte: ranOut } }, { @@ -138,6 +148,13 @@ class MongoOutboxSource { $inc: { attempts: -1, redeliveries: 1 }, }, ); + } + + async reserve(leaseMilliseconds) { + const now = new Date(this.#config.now()); + const pausedUntil = await this.#pausedUntil(now.getTime()); + if (pausedUntil > now.getTime()) return { nextAvailableAt: pausedUntil }; + await this.#reclaim(now.getTime(), leaseMilliseconds); const token = randomUUID(); const taken = documentOf( await this.#collection.findOneAndUpdate( @@ -209,6 +226,7 @@ class MongoOutboxSource { { $max: { pausedUntil: new Date(until) } }, { upsert: true }, ); + this.#pause.until = Math.max(this.#pause.until, until); } } @@ -261,6 +279,10 @@ const createMongoOutbox = (options = {}) => { } const config = { now: checkOptionalFunction(options.now, 'now') ?? Date.now, + // The polling driver checks it; the pause is read again as often. + pauseReadMilliseconds: options.pollIntervalMilliseconds ?? 1000, + // The lease of the workers: none of theirs runs out sooner. + leaseMilliseconds: options.leaseMilliseconds ?? DEFAULT_LEASE_MS, retainData: checkRetainData(options.retainData), clockSkewToleranceMilliseconds, changeStream, diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 9450a64..d8c472c 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -499,3 +499,30 @@ test('MongoDB: the audit sink tells whether it holds an event', async () => { { projection: { _id: 1 } }, ]); }); + +test('MongoDB: a reserve takes back leases once a third of a lease, and reads the pause once a poll', async () => { + const { collection, source, advance } = outboxOf({ extra: { pollIntervalMilliseconds: 1000 } }); + const count = (name) => collection.calls.filter((call) => call[0] === name).length; + for (let index = 0; index < 5; index++) await source.reserve(60_000); + // Idle, five reserves of five lanes: one write to take back leases, one read of the pause. + assert.strictEqual(count('updateMany'), 1); + assert.strictEqual( + collection.calls.filter( + (call) => call[0] === 'findOne' && JSON.stringify(call[1]) === '{"_id":"mailer:pause"}', + ).length, + 1, + ); + advance(20_000); + await source.reserve(60_000); + assert.strictEqual(count('updateMany'), 2); + // A pause of another process is seen within a poll interval. + const other = createMongoOutbox({ collection, now: () => 0 }); + await source.enqueue(requestOf('a')); + await other.source.pause(90_000_000); + assert.strictEqual((await source.reserve(60_000)).lease?.job.id, 'a'); + advance(1000); + await source.enqueue(requestOf('b')); + assert.deepStrictEqual(await source.reserve(60_000), { + nextAvailableAt: 90_000_000, + }); +}); From e692f9319433bcd9ec6e60d28e9c7f5ecc88ba8b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:43:56 +0300 Subject: [PATCH 32/83] feat(templates): bound the includes, their depth and the output of a render Only the file loader bounded anything, the size of a template, so a template that included in a loop, a chain of includes or a template that repeated a value read and gave as much as it named. A renderer takes maxIncludes (10000), the includes, partials and layouts an engine looks up in one render, counted while it runs synchronously; maxIncludeDepth (32), a chain of includes each written in the one before, as the engine names it; and maxOutputBytes (10 MiB), each part in UTF-8 after each step. Each takes Infinity for none, and going past one is a TemplateRenderError. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 18 ++++ src/templates/includes.js | 77 ++++++++++++++-- src/templates/index.d.ts | 14 +++ src/templates/renderer.js | 127 +++++++++++++++++++------- tests/types/templates.test-d.ts | 3 + tests/unit/templates-engines.test.js | 60 ++++++++++++ tests/unit/templates-renderer.test.js | 49 ++++++++++ 7 files changed, 306 insertions(+), 42 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index f6bfd95..1915028 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -239,6 +239,24 @@ wait for the one compile. A template that is missing or fails is not kept, so one deployed later is found. `clear()` forgets everything; `cache: false` reads and compiles on every render, for development. +## Bounds + +A render is bounded, so that a template, or a context, that runs away fails +the render rather than the process: + +| Option | Default | Bounds | +| --- | --- | --- | +| `maxIncludes` | `10000` | the includes, partials and layouts an engine looks up in one render | +| `maxIncludeDepth` | `32` | a chain of includes, each written in the one before | +| `maxOutputBytes` | 10 MiB | the HTML, the text and the subject, each in bytes of UTF-8, after each step | + +Each takes `Infinity` for no bound, and going past one is a +`TemplateRenderError`. The includes are counted as an engine runs +synchronously: those EJS, Eta and Mustache look up as they render, and those +Pug and Handlebars look up as they compile, until their first await. The depth +is counted where the engine names the template an include is written in (Eta, +Handlebars); a partial that names itself is no deeper for it. The file loader reads no template over its `maxBytes` (1 MiB). + ## Errors | Thrown | When | `retryable` | diff --git a/src/templates/includes.js b/src/templates/includes.js index e335311..972c16b 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -89,20 +89,74 @@ const loadFailure = (name, error) => ? error : new TemplateLoadError(name, { cause: error }); +// The bounds of the render whose engine runs now, synchronously: set around +// each call of an engine, so that the includes it looks up before it returns +// are counted against that render. What an engine looks up in a continuation +// of its own, after an await, is not counted, as no render is running then. +let current; + +/** + * Calls an engine with the bounds of a render. + * + * @param bounds `{ includes, maxIncludes }`: the includes read so far. + * @param step The call of the engine. + * @returns What it gives. + * @internal + */ +const withBounds = (bounds, step) => { + const outer = current; + current = bounds; + try { + return step(); + } finally { + current = outer; + } +}; + +const spend = () => { + if (current === undefined) return; + current.includes++; + if (current.includes > current.maxIncludes) { + throw new RangeError( + `The render reads more than ${current.maxIncludes} includes (maxIncludes).`, + ); + } +}; + /** * Makes the includes of a renderer: the reads of its loader, kept while its * cache is on, and the lookup of a name an engine includes. * * @param loader The loader of the renderer. * @param options `partials`, the directories searched by a bare name after the - * root, `cache` and `maxEntries`. - * @returns The includes, one object per renderer and per `clear()`. + * root, `cache`, `maxEntries` and `maxIncludeDepth`. + * @returns `{ includes, top }`: the includes, one object per renderer and per + * `clear()`, and the mark of a template the renderer compiles itself, which + * is no include deep. * @internal */ const createIncludes = (loader, options) => { - const { partials, cache } = options; + const { partials, cache, maxIncludeDepth } = options; const sources = createCache(cache ? options.maxEntries : 0); const syncSources = createCache(cache ? options.maxEntries : 0); + // How many includes deep each template is, at least, as an include is + // written in a template: a chain of them deeper than maxIncludeDepth is + // refused, and a template that includes itself is as deep as before. + const depths = createCache(options.maxEntries); + const depthOf = (include, from) => { + const depth = (from === undefined ? 0 : (depths.get(from) ?? 0)) + 1; + if (depth > maxIncludeDepth) { + throw new RangeError( + `The include ${JSON.stringify(include)} of ${from} is more than ${maxIncludeDepth} ` + + 'includes deep (maxIncludeDepth).', + ); + } + return depth; + }; + const reached = (name, depth) => { + const known = depths.get(name); + if (known === undefined || depth < known) depths.set(name, depth); + }; const read = (name) => memoize(sources, name, async () => { try { @@ -128,29 +182,38 @@ const createIncludes = (loader, options) => { syncSources.set(name, { source }); return source; }; - return Object.freeze({ + const includes = Object.freeze({ partials, cache, join: joinName, read, readSync, async lookup(include, from, extensions = ['']) { + spend(); + const depth = depthOf(include, from); const names = candidatesOf(include, from, extensions, partials); for (const name of names) { const source = await read(name); - if (source !== undefined) return source; + if (source === undefined) continue; + reached(source.name, depth); + return source; } throw new TemplateNotFoundError(include, names, { from }); }, lookupSync(include, from, extensions = ['']) { + spend(); + const depth = depthOf(include, from); const names = candidatesOf(include, from, extensions, partials); for (const name of names) { const source = readSync(name); - if (source !== undefined) return source; + if (source === undefined) continue; + reached(source.name, depth); + return source; } throw new TemplateNotFoundError(include, names, { from }); }, }); + return { includes, top: (name) => depths.set(name, 0) }; }; -module.exports = { createIncludes, joinName }; +module.exports = { createIncludes, joinName, withBounds }; diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 4910610..f1f3665 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -237,6 +237,20 @@ export interface TemplateRendererOptions { readonly autoText?: boolean; /** false reads and compiles every template on each render, for development. @default true */ readonly cache?: boolean | TemplateCacheOptions; + /** + * The includes, partials and layouts an engine looks up in one render, as + * it runs synchronously (what it looks up after an await of its own is not + * counted); more fail the render. Infinity for no bound. @default 10000 + */ + readonly maxIncludes?: number; + /** + * How deep a chain of includes goes, each written in the one before, as the + * engine names the template an include is written in. Infinity for no + * bound. @default 32 + */ + readonly maxIncludeDepth?: number; + /** The largest HTML, text or subject a render gives, in bytes of UTF-8. Infinity for no bound. @default 10485760 */ + readonly maxOutputBytes?: number; } /** A renderer of templates by name, a `Renderer` of `./render`. */ diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 00dddff..6049d2d 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -12,10 +12,15 @@ const { checkOptionalFunction, checkPositiveInteger } = require('../internal/opt const { createCache, memoize } = require('./cache.js'); const { TemplateNotFoundError, TemplateRenderError, templateErrorOf } = require('./errors.js'); const { parseFrontMatter } = require('./front-matter.js'); -const { createIncludes } = require('./includes.js'); +const { createIncludes, withBounds } = require('./includes.js'); const { checkTemplateName } = require('./loader.js'); const DEFAULT_MAX_ENTRIES = 1000; +// The bounds of a render: the includes it reads, how deep they go, and the +// bytes of each part it gives. +const DEFAULT_MAX_INCLUDES = 10_000; +const DEFAULT_MAX_INCLUDE_DEPTH = 32; +const DEFAULT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024; const NO_LAYOUT = new Set(['none', 'false']); const defaultVariantName = (template, variant) => { @@ -66,6 +71,15 @@ const enginesOf = (options) => { return { engines, suffixes }; }; +// A whole number above zero, or Infinity for none. +const checkBound = (value, name, fallback) => { + if (value === undefined) return fallback; + if (value !== Number.POSITIVE_INFINITY && (!Number.isSafeInteger(value) || value < 1)) { + throw new TypeError(`${name} must be a positive whole number or Infinity: ${String(value)}`); + } + return value; +}; + const checkLocale = (locale, what) => { if (locale === undefined) return undefined; if (typeof locale !== 'string') throw new TypeError(`${what} must be a string.`); @@ -113,6 +127,13 @@ const createRendererConfig = (options) => { autoText: options.autoText ?? true, cache: cache !== false, maxEntries, + maxIncludes: checkBound(options.maxIncludes, 'maxIncludes', DEFAULT_MAX_INCLUDES), + maxIncludeDepth: checkBound( + options.maxIncludeDepth, + 'maxIncludeDepth', + DEFAULT_MAX_INCLUDE_DEPTH, + ), + maxOutputBytes: checkBound(options.maxOutputBytes, 'maxOutputBytes', DEFAULT_MAX_OUTPUT_BYTES), }; }; @@ -160,14 +181,29 @@ const wrapError = (error, template, phase, engine) => { }); }; -const run = async (phase, template, engine, step) => { +// Calls an engine, with the bounds of the render while it runs, and makes +// what it throws an error of ./templates. +const run = async (phase, template, engine, step, bounds) => { try { - return await step(); + return await withBounds(bounds, step); } catch (error) { throw wrapError(error, template, phase, engine); } }; +// A part of a message, held to maxOutputBytes of UTF-8, at most three bytes +// for each UTF-16 unit, so that most need no count. +const checkOutput = (config, phase, name, engine, value) => { + const max = config.maxOutputBytes; + if (value.length * 3 <= max || Buffer.byteLength(value) <= max) return value; + throw new TemplateRenderError( + name, + phase, + `The template gave more than ${max} bytes (maxOutputBytes).`, + { engine: engine?.name }, + ); +}; + const normalizeSubject = (subject) => { if (subject === undefined) return undefined; const collapsed = decodeEntities(subject).replace(/\s+/g, ' ').trim(); @@ -181,7 +217,9 @@ const isParts = (value) => typeof value === 'object' && value !== null; * * @param options `engine` or `engines`, `loader`, `partials`, `layout`, * `globals`, `defaultLocale`, `variantName`, `frontMatter` (true), - * `inlineCss`, `autoText` (true) and `cache` (true, or `{ maxEntries }`). + * `inlineCss`, `autoText` (true), `cache` (true, or `{ maxEntries }`), + * `maxIncludes` (10000), `maxIncludeDepth` (32) and `maxOutputBytes` + * (10 MiB). * @returns The renderer: `render({ template, context, locale })` and * `clear()`. * @throws {TypeError} If an option is invalid. @@ -193,20 +231,22 @@ const createTemplateRenderer = (options = {}) => { partials: config.partials, cache: config.cache, maxEntries: config.maxEntries, + maxIncludeDepth: config.maxIncludeDepth, }); const compiled = createCache(config.cache ? config.maxEntries : 0); const resolved = createCache(config.cache ? config.maxEntries : 0); - const state = { config, compiled, resolved, includes: makeIncludes() }; + const state = { config, compiled, resolved, files: makeIncludes() }; return { async render(input) { const checked = checkInput(input); // Without a cache, each render reads and compiles anew, includes too. - const current = config.cache ? state : { ...state, includes: makeIncludes() }; - return renderTemplate(current, checked, input); + const current = config.cache ? state : { ...state, files: makeIncludes() }; + const bounds = { includes: 0, maxIncludes: config.maxIncludes }; + return renderTemplate(current, checked, input, bounds); }, clear() { // Engines key what they keep by the includes, so they make it anew too. - state.includes = makeIncludes(); + state.files = makeIncludes(); compiled.clear(); resolved.clear(); }, @@ -238,7 +278,7 @@ const findPart = async (state, base, locale, part, choices, tried) => { for (const extension of choice.extensions) { const name = stem + extension; tried?.push(name); - const source = await state.includes.read(name); + const source = await state.files.includes.read(name); if (source !== undefined) return { name, source, engine: choice.engine, choice }; } } @@ -273,7 +313,7 @@ const resolvedFor = (state, template, locale) => resolveTemplate(state, template, locale), ); -const compileEntry = async (state, found, part) => { +const compileEntry = async (state, found, part, bounds) => { const { name, engine } = found; let body = found.source.source; let fields; @@ -292,9 +332,10 @@ const compileEntry = async (state, found, part) => { path: found.source.path, part, cache: state.config.cache, - includes: state.includes, + includes: state.files.includes, }; - const render = await run('compile', name, engine, () => engine.compile(body, context)); + state.files.top(name); + const render = await run('compile', name, engine, () => engine.compile(body, context), bounds); if (typeof render !== 'function') { throw new TemplateRenderError(name, 'compile', 'The engine gave no function.', { engine: engine.name, @@ -303,25 +344,29 @@ const compileEntry = async (state, found, part) => { const subject = fields?.subject === undefined ? undefined - : await run('compile', name, engine, () => - engine.compile(fields.subject, { ...context, part: 'subject' }), + : await run( + 'compile', + name, + engine, + () => engine.compile(fields.subject, { ...context, part: 'subject' }), + bounds, ); return { name, engine, render, subject, fields, context }; }; -const compiledFor = (state, found, part) => +const compiledFor = (state, found, part, bounds) => memoize(state.compiled, `${found.engine.name}\u0000${part}\u0000${found.name}`, () => - compileEntry(state, found, part), + compileEntry(state, found, part, bounds), ); -const renderString = async (phase, entry, render, scope) => { - const output = await run(phase, entry.name, entry.engine, () => render(scope)); +const renderString = async (state, phase, entry, render, scope, bounds) => { + const output = await run(phase, entry.name, entry.engine, () => render(scope), bounds); if (typeof output !== 'string') { throw new TemplateRenderError(entry.name, phase, 'The template gave no string.', { engine: entry.engine.name, }); } - return output; + return checkOutput(state.config, phase, entry.name, entry.engine, output); }; const globalsOf = async (config, input) => { @@ -343,7 +388,7 @@ const layoutNameOf = (config, entry) => { }; // Renders the HTML of a template into its layout, when it has one. -const applyLayout = async (state, entry, html, scope, locale) => { +const applyLayout = async (state, entry, html, scope, locale, bounds) => { const name = layoutNameOf(state.config, entry); if (name === undefined) return { html, engine: entry.engine, context: entry.context }; let resolution; @@ -356,15 +401,22 @@ const applyLayout = async (state, entry, html, scope, locale) => { if (resolution.html === undefined) { throw new TemplateRenderError(entry.name, 'layout', `The layout ${name} has no HTML.`); } - const layout = await compiledFor(state, resolution.html, 'html'); + const layout = await compiledFor(state, resolution.html, 'html', bounds); if (layout.fields?.layout !== undefined) { throw new TemplateRenderError(layout.name, 'layout', 'A layout has no layout of its own.'); } - const wrapped = await renderString('layout', layout, layout.render, { ...scope, body: html }); + const wrapped = await renderString( + state, + 'layout', + layout, + layout.render, + { ...scope, body: html }, + bounds, + ); return { html: wrapped, engine: layout.engine, context: layout.context }; }; -const renderTemplate = async (state, checked, input) => { +const renderTemplate = async (state, checked, input, bounds) => { const { config } = state; const resolution = await resolvedFor(state, checked.template, checked.locale); const scope = { ...(await globalsOf(config, input)), ...checked.context }; @@ -372,11 +424,12 @@ const renderTemplate = async (state, checked, input) => { state, resolution.html ?? resolution.text, resolution.html ? 'html' : 'text', + bounds, ); let html; let text; let subject; - const output = await run('render', main.name, main.engine, () => main.render(scope)); + const output = await run('render', main.name, main.engine, () => main.render(scope), bounds); if (typeof output === 'string') { if (resolution.html === undefined) text = decodeEntities(output); else html = output; @@ -388,27 +441,29 @@ const renderTemplate = async (state, checked, input) => { }); } if (html !== undefined && typeof output === 'string') { - const outer = await applyLayout(state, main, html, scope, resolution.locale); + checkOutput(config, 'render', main.name, main.engine, html); + const outer = await applyLayout(state, main, html, scope, resolution.locale, bounds); html = outer.html; if (typeof outer.engine.finish === 'function') { - html = await run('finish', main.name, outer.engine, () => - outer.engine.finish(html, outer.context), - ); + const finish = () => outer.engine.finish(html, outer.context); + html = await run('finish', main.name, outer.engine, finish, bounds); + checkOutput(config, 'finish', main.name, outer.engine, html); } } if (html !== undefined && config.inlineCss !== undefined) { - html = await run('inline-css', main.name, undefined, () => config.inlineCss(html)); + html = await run('inline-css', main.name, undefined, () => config.inlineCss(html), bounds); + checkOutput(config, 'inline-css', main.name, undefined, html); } if (text === undefined && resolution.html !== undefined && resolution.text !== undefined) { - const entry = await compiledFor(state, resolution.text, 'text'); - text = decodeEntities(await renderString('render', entry, entry.render, scope)); + const entry = await compiledFor(state, resolution.text, 'text', bounds); + text = decodeEntities(await renderString(state, 'render', entry, entry.render, scope, bounds)); } if (subject === undefined && main.subject !== undefined) { - subject = await renderString('render', main, main.subject, scope); + subject = await renderString(state, 'render', main, main.subject, scope, bounds); } if (subject === undefined && resolution.subject !== undefined) { - const entry = await compiledFor(state, resolution.subject, 'subject'); - subject = await renderString('render', entry, entry.render, scope); + const entry = await compiledFor(state, resolution.subject, 'subject', bounds); + subject = await renderString(state, 'render', entry, entry.render, scope, bounds); } return outputOf(config, main, resolution, { html, text, subject }); }; @@ -416,13 +471,15 @@ const renderTemplate = async (state, checked, input) => { const outputOf = (config, main, resolution, parts) => { const { html } = parts; for (const key of ['html', 'text', 'subject']) { - if (parts[key] !== undefined && typeof parts[key] !== 'string') { + if (parts[key] === undefined) continue; + if (typeof parts[key] !== 'string') { throw new TemplateRenderError( main.name, 'render', `The template gave its ${key} as no string.`, ); } + checkOutput(config, 'render', main.name, main.engine, parts[key]); } const text = parts.text ?? (config.autoText && html !== undefined ? htmlToText(html) : undefined); if (html === undefined && text === undefined) { diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index dc15822..9cf5fc3 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -90,6 +90,9 @@ const templates = createTemplateRenderer({ defaultLocale: 'en', inlineCss: cssInline, cache: { maxEntries: 100 }, + maxIncludes: 100, + maxIncludeDepth: 8, + maxOutputBytes: Infinity, }); expectAssignable>(templates); const output = await templates.render({ diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 6ec6ce2..8bfa613 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -174,6 +174,66 @@ test('Templates: no engine option turns off escaping unless allowUnescaped', asy } }); +test('Templates: a render reads at most maxIncludes includes, at most maxIncludeDepth deep', async () => { + // Before: no bound, so a template that included in a loop, or a chain of + // includes, read as many as it named. + const items = [1, 2, 3, 4, 5]; + const looped = [ + [ + createMustacheEngine({ mustache }), + { 'a.mustache': '{{#items}}{{> row}}{{/items}}', 'row.mustache': '.' }, + ], + [ + createEjsEngine({ ejs }), + { 'a.ejs': '<% for (const i of items) { %><%- include("row") %><% } %>', 'row.ejs': '.' }, + ], + [ + createEtaEngine({ Eta }), + { 'a.eta': '<% for (const i of it.items) { %><%~ include("row") %><% } %>', 'row.eta': '.' }, + ], + ]; + for (const [engine, templates] of looped) { + const options = { maxIncludes: 4 }; + const error = await failureOf( + renderWith(engine, templates, { template: 'a', context: { items } }, options), + ); + assert.ok(error instanceof TemplateRenderError, engine.name); + assert.match(error.message, /more than 4 includes \(maxIncludes\)/, engine.name); + const output = await renderWith( + engine, + templates, + { template: 'a', context: { items } }, + { maxIncludes: 5 }, + ); + assert.strictEqual(output.html, '.....', engine.name); + } + const chain = { + 'a.hbs': 'a{{> b}}', + 'b.hbs': 'b{{> c}}', + 'c.hbs': 'c{{> d}}', + 'd.hbs': 'd', + }; + const engine = createHandlebarsEngine({ handlebars }); + assert.strictEqual( + (await renderWith(engine, chain, { template: 'a' }, { maxIncludeDepth: 3 })).html, + 'abcd', + ); + const deep = await failureOf( + renderWith(engine, chain, { template: 'a' }, { maxIncludeDepth: 2 }), + ); + assert.ok(deep instanceof TemplateRenderError); + assert.match(deep.message, /"d" of c.hbs is more than 2 includes deep/); + // A partial that names itself is no deeper for it. + const tree = { 'a.hbs': '{{> node}}', 'node.hbs': '{{#if next}}{{> node next=false}}{{/if}}n' }; + const recursive = await renderWith( + engine, + tree, + { template: 'a', context: { next: true } }, + { maxIncludeDepth: 2 }, + ); + assert.strictEqual(recursive.html, 'nn'); +}); + test('Templates: the EJS engine refuses what it sets itself', () => { assert.throws(() => createEjsEngine({}), /needs ejs/); for (const name of ['filename', 'root', 'views', 'includer', 'cache', 'client']) { diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index 2fbd34c..9b15c34 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -490,6 +490,51 @@ test('Templates: engines are chosen by extension, the longest first', async () = assert.strictEqual(text.text, 'hbs'); }); +test('Templates: no part of a message is longer than maxOutputBytes', async () => { + // Before: no bound, so a template that repeated a value gave as much as it + // could hold. + const templates = { + 'a.tpl': '{{x}}', + 'a.text.tpl': '{{y}}', + 'b.tpl': '

    ', + 'b.subject.tpl': '{{x}}', + 'c.tpl': '{{x}}', + 'layout.tpl': '{{{body}}}{{{body}}}', + }; + const big = async (template, context, options) => { + const { renderer } = rendererOf(templates, { maxOutputBytes: 30, ...options }); + return renderer.render({ template, context }).catch((e) => e); + }; + for (const [template, context] of [ + ['a', { x: 'x'.repeat(31), y: '' }], + ['a', { x: '', y: 'я'.repeat(16) }], + ['b', { x: 'x'.repeat(31) }], + ]) { + const error = await big(template, context); + assert.ok(error instanceof TemplateRenderError, template); + assert.match(error.message, /more than 30 bytes \(maxOutputBytes\)/); + } + // Ten Cyrillic letters are 20 bytes, and need no count. + assert.strictEqual((await big('a', { x: 'я'.repeat(10), y: 'y' })).html, 'я'.repeat(10)); + // The layout counts too. + const layout = await big('c', { x: 'x'.repeat(20) }, { layout: 'layout' }); + assert.strictEqual(layout.phase, 'layout'); + const parts = createTemplateRenderer({ + engine: { name: 'parts', extensions: [], compile: () => () => ({ html: 'x'.repeat(40) }) }, + loader: createMemoryLoader({ x: '' }), + maxOutputBytes: 30, + }); + assert.match((await parts.render({ template: 'x' }).catch((e) => e)).message, /maxOutputBytes/); + const inlined = await big( + 'b', + {}, + { inlineCss: createCssInliner({ inline: () => 'x'.repeat(31) }) }, + ); + assert.strictEqual(inlined.phase, 'inline-css'); + const finished = await big('b', {}, { engine: fakeEngine({ finish: () => 'x'.repeat(31) }) }); + assert.strictEqual(finished.phase, 'finish'); +}); + test('Templates: the renderer checks its options', () => { const engine = fakeEngine(); const loader = createMemoryLoader({}); @@ -517,11 +562,15 @@ test('Templates: the renderer checks its options', () => { [{ engine, loader, variantName: 1 }, /variantName must be a function/], [{ engine, loader, inlineCss: 1 }, /inlineCss must be a function/], [{ engine, loader, defaultLocale: 1 }, /defaultLocale must be a string/], + [{ engine, loader, maxIncludes: 0 }, /maxIncludes must be a positive whole number/], + [{ engine, loader, maxIncludeDepth: 1.5 }, /maxIncludeDepth must be/], + [{ engine, loader, maxOutputBytes: '1' }, /maxOutputBytes must be/], ]; for (const [options, pattern] of bad) { assert.throws(() => createTemplateRenderer(options), pattern, String(pattern)); } assert.throws(() => createTemplateRenderer(), /needs an engine/); + createTemplateRenderer({ engine, loader, maxIncludes: Infinity, maxOutputBytes: Infinity }); assert.throws(() => createCssInliner({}), /needs inline/); assert.throws(() => createCssInliner({ inline() {}, options: 1 }), /must be an object/); }); From 2b6a46f4848abfc15b776018e9432adbc90183c9 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:44:02 +0300 Subject: [PATCH 33/83] perf(queue): wake one lane per change of the source A change the source told (a job enqueued, a change stream event) woke every lane of the process, and every lane reserved at once for one job, a round trip each on a store. Each change wakes the lane that waited longest; a wake that comes while every lane is busy is kept for the next to wait, so none is lost. Co-Authored-By: Claude Opus 5.5 --- src/internal/polling-driver.js | 37 +++++++++++++++++++++-------- tests/unit/queue-memory.test.js | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/src/internal/polling-driver.js b/src/internal/polling-driver.js index b5fdff1..426bcd9 100644 --- a/src/internal/polling-driver.js +++ b/src/internal/polling-driver.js @@ -47,19 +47,37 @@ const createPollingConfig = (options) => { return config; }; -// A wake that the source's listener and close() both pull, renewed after. +// The wakes of the source's listener, each for one lane: a change (a job +// enqueued) wakes the lane that waited longest rather than every lane of the +// process, which would all reserve at once for one job. A wake that comes +// while every lane is busy is kept for the next one to wait. const createWaker = (source) => { - let controller = new AbortController(); + const waiting = new Set(); + let pending = false; const wake = () => { - controller.abort(); - controller = new AbortController(); + for (const controller of waiting) { + waiting.delete(controller); + controller.abort(); + return; + } + pending = true; }; const unsubscribe = typeof source.subscribe === 'function' ? source.subscribe(wake) : undefined; return { - get signal() { - return controller.signal; + // The signal of one wait, aborted by a wake. + take() { + const controller = new AbortController(); + if (pending) { + pending = false; + controller.abort(); + } else { + waiting.add(controller); + } + return controller; + }, + release(controller) { + waiting.delete(controller); }, - wake, unsubscribe: () => unsubscribe?.(), }; }; @@ -201,13 +219,15 @@ const startConsumer = (source, handler, concurrency, outerSignal, config, log) = outerSignal?.addEventListener('abort', onOuterAbort, { once: true }); if (outerSignal?.aborted) stopping.abort(); const wait = async (milliseconds) => { - const { signal, cleanup } = combineSignals(stopping.signal, waker.signal); + const woken = waker.take(); + const { signal, cleanup } = combineSignals(stopping.signal, woken.signal); try { await config.sleep(milliseconds, signal); } catch { // Woken: a job may be ready, or the consumer stops. } finally { cleanup(); + waker.release(woken); } }; const lane = async () => { @@ -235,7 +255,6 @@ const startConsumer = (source, handler, concurrency, outerSignal, config, log) = return { async close(closeOptions = {}) { stopping.abort(); - waker.wake(); const timeout = closeOptions.timeoutMilliseconds ?? 30_000; // The jobs still running past the timeout are told to stop: one not // yet sent goes back. One that does not hear it is left, its lease to diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 1c48ef9..bfaa156 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -569,3 +569,44 @@ test('Queue: the memory queue keeps a completed job for the window of duplicates ); assert.throws(() => createMemoryQueue({ dedupeWindowSeconds: 0 }), /dedupeWindowSeconds/); }); + +test('Queue: a change wakes one waiting lane, not every lane, and is kept when all are busy', async () => { + const clock = manualClock(0); + let listener; + let reserves = 0; + const source = { + id: 'waking', + enqueue: async () => ({ id: 'x', duplicate: false }), + reserve: async () => { + reserves++; + return {}; + }, + extend: async () => true, + ack: async () => true, + retry: async () => true, + fail: async () => true, + release: async () => true, + subscribe: (wake) => { + listener = wake; + return () => {}; + }, + }; + const driver = createPollingDriver(source, { + now: clock.now, + sleep: clock.sleep, + pollIntervalMilliseconds: 60_000, + }); + const consumer = driver.consume(async () => ({ type: 'ack' }), { concurrency: 4 }); + const settle = () => new Promise((resolve) => setImmediate(resolve)); + await settle(); + assert.strictEqual(reserves, 4); + listener(); + await settle(); + // One job enqueued: one lane reserves, three wait on. + assert.strictEqual(reserves, 5); + listener(); + listener(); + await settle(); + assert.strictEqual(reserves, 7); + await consumer.close({ timeoutMilliseconds: 10 }); +}); From 3d99803b22f78aa56af6d9078e220048b31b7f3b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:47:08 +0300 Subject: [PATCH 34/83] fix(templates): give a loader template names alone, none starting with a dot The includes passed the names an engine read straight to the loader, and a name variantName gave was only checked to be a string, so a custom loader, such as one that queries a database by name, was given whatever came; and a segment after the first could start with a dot, so `partials/.env` was a name. Both reads of the includes refuse a name that is no template name, as variantName's are refused, and no segment starts with a dot. Co-Authored-By: Claude Opus 5.5 --- src/templates/includes.js | 13 +++-- src/templates/loader.js | 31 ++++++++---- src/templates/renderer.js | 6 ++- tests/unit/templates-loader.test.js | 4 ++ tests/unit/templates-renderer.test.js | 72 ++++++++++++++++++++++++++- 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/src/templates/includes.js b/src/templates/includes.js index 972c16b..9e2bc1c 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -3,7 +3,7 @@ // kept as its templates are, and never read from disk by the engine itself. const { TemplateLoadError, TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); -const { isTemplateName } = require('./loader.js'); +const { checkTemplateName, isTemplateName, notTemplateName } = require('./loader.js'); const { createCache, memoize } = require('./cache.js'); // The segments of a name, `.` dropped and `..` taking the one before; undefined @@ -78,6 +78,8 @@ const candidatesOf = (include, from, extensions, partials) => { return names; }; +const READ = 'The name of a template read'; + // What a loader threw, as an error that names the template: one of // ./templates as it is, and any other, such as EMFILE or a database timeout, // as a TemplateLoadError, which an engine that wraps it does not make a @@ -157,15 +159,20 @@ const createIncludes = (loader, options) => { const known = depths.get(name); if (known === undefined || depth < known) depths.set(name, depth); }; - const read = (name) => - memoize(sources, name, async () => { + // A loader is given template names alone, whoever asks: an engine, or the + // variants of the renderer. + const read = (name) => { + if (!isTemplateName(name)) return Promise.reject(notTemplateName(name, READ)); + return memoize(sources, name, async () => { try { return (await loader.load(name)) ?? undefined; } catch (error) { throw loadFailure(name, error); } }); + }; const readSync = (name) => { + checkTemplateName(name, READ); if (typeof loader.loadSync !== 'function') { throw new TypeError( 'This engine reads its includes synchronously, and the loader has no loadSync.', diff --git a/src/templates/loader.js b/src/templates/loader.js index 372ec85..6c2631d 100644 --- a/src/templates/loader.js +++ b/src/templates/loader.js @@ -16,7 +16,8 @@ const DEFAULT_MAX_BYTES = 1024 * 1024; /** * Whether a value is a template name: `welcome`, `auth/reset-password`, * `welcome.uk.hbs`; letters, digits, `_`, `-`, `@`, `.` and `/` between - * segments, none of them empty, `.` or `..`, at most 256 characters. + * segments, none of them empty or starting with `.` (`..`, `.env`), at most + * 256 characters. * * @internal */ @@ -28,12 +29,23 @@ const isTemplateName = (name) => { for (;;) { const slash = name.indexOf('/', start); const segment = name.slice(start, slash === -1 ? name.length : slash); - if (segment === '' || segment === '.' || segment === '..') return false; + if (segment === '' || segment[0] === '.') return false; if (slash === -1) return true; start = slash + 1; } }; +/** + * The error of a value that is no template name. + * + * @internal + */ +const notTemplateName = (name, what) => + new TypeError( + `${what} must be a template name such as "welcome" or "auth/reset-password", ` + + `not a path: ${JSON.stringify(name)}`, + ); + /** * Checks a template name. * @@ -41,12 +53,7 @@ const isTemplateName = (name) => { * @internal */ const checkTemplateName = (name, what) => { - if (!isTemplateName(name)) { - throw new TypeError( - `${what} must be a template name such as "welcome" or "auth/reset-password", ` + - `not a path: ${JSON.stringify(name)}`, - ); - } + if (!isTemplateName(name)) throw notTemplateName(name, what); return name; }; @@ -160,4 +167,10 @@ const createMemoryLoader = (templates) => { return { load: loadSync, loadSync }; }; -module.exports = { checkTemplateName, createFileLoader, createMemoryLoader, isTemplateName }; +module.exports = { + checkTemplateName, + createFileLoader, + createMemoryLoader, + isTemplateName, + notTemplateName, +}; diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 6049d2d..0b32850 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -272,8 +272,10 @@ const choicesOf = (config, template) => { }; const findPart = async (state, base, locale, part, choices, tried) => { - const stem = state.config.variantName(base, { locale, part }); - if (typeof stem !== 'string') throw new TypeError('variantName must give a name.'); + const stem = checkTemplateName( + state.config.variantName(base, { locale, part }), + 'The name variantName gives', + ); for (const choice of choices) { for (const extension of choice.extensions) { const name = stem + extension; diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index 2eec3e6..0067f71 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -24,6 +24,7 @@ const makeTree = (t) => { fs.writeFileSync(path.join(root, 'notes.txt'), 'not a template'); fs.writeFileSync(path.join(root, 'big.ejs'), 'x'.repeat(2048)); fs.writeFileSync(path.join(base, 'secret.ejs'), 'secret'); + fs.writeFileSync(path.join(root, '.env'), 'SECRET=1'); return { base, root }; }; @@ -44,6 +45,9 @@ test('Templates: the file loader reads a name under its directory, its BOM dropp '../secret.ejs', 'auth/../welcome.ejs', '/etc/passwd', + // Before: read, as a segment after the first could start with a dot. + 'auth/../.env', + 'auth/.hidden.ejs', ]) { assert.strictEqual(await loader.load(name), undefined, name); assert.strictEqual(loader.loadSync(name), undefined, name); diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index 9b15c34..f50d016 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -130,7 +130,19 @@ test('Templates: a missing template names what was tried, and is not retryable', test('Templates: a name is never a path', async () => { const { renderer } = rendererOf({}); - for (const template of ['../secret', '/etc/passwd', 'a//b', 'a/./b', 'a\\b', '', '.hidden', 42]) { + for (const template of [ + '../secret', + '/etc/passwd', + 'a//b', + 'a/./b', + 'a\\b', + '', + '.hidden', + // Before: a segment after the first could start with a dot. + 'partials/.env', + 'a/.git/config', + 42, + ]) { await assert.rejects(renderer.render({ template }), TypeError, String(template)); } await assert.rejects(renderer.render(null), TypeError); @@ -138,6 +150,58 @@ test('Templates: a name is never a path', async () => { await assert.rejects(renderer.render({ template: 'a', locale: 1 }), TypeError); }); +test('Templates: a loader is given template names alone, whoever asks', async () => { + // Before: a name an engine read, or one variantName gave, reached the loader + // unchecked, such as a database query by name. + const asked = []; + const loader = { + load: (name) => { + asked.push(name); + return name === 'a.tpl' ? { name, source: '

    ' } : undefined; + }, + loadSync: (name) => { + asked.push(name); + return undefined; + }, + }; + const reader = (read) => ({ + name: 'reader', + extensions: ['.tpl'], + compile: async (source, context) => { + await read(context.includes); + return () => source; + }, + }); + for (const read of [ + (includes) => includes.read('../secret.tpl'), + (includes) => includes.readSync('partials/.env'), + (includes) => includes.read(42), + ]) { + const renderer = createTemplateRenderer({ engine: reader(read), loader }); + await assert.rejects(renderer.render({ template: 'a' }), /must be a template name/); + } + const variants = createTemplateRenderer({ + engine: fakeEngine(), + loader, + variantName: (template) => `../${template}`, + }); + await assert.rejects( + variants.render({ template: 'a' }), + /variantName gives must be a template name/, + ); + assert.deepStrictEqual(asked, [ + 'a.tpl', + 'a.text.tpl', + 'a.subject.tpl', + 'a.tpl', + 'a.text.tpl', + 'a.subject.tpl', + 'a.tpl', + 'a.text.tpl', + 'a.subject.tpl', + ]); +}); + test('Templates: includes are read through the loader, from the root or the partials', async () => { const { renderer } = rendererOf( { @@ -553,6 +617,7 @@ test('Templates: the renderer checks its options', () => { [{ engine }, /needs a loader/], [{ engine, loader, partials: 'p' }, /partials must be an array/], [{ engine, loader, partials: ['../p'] }, /directory of partials/], + [{ engine, loader, partials: ['p/.hidden'] }, /directory of partials/], [{ engine, loader, layout: '/abs' }, /layout must be a template name/], [{ engine, loader, globals: 1 }, /globals must be an object or a function/], [{ engine, loader, frontMatter: 'yes' }, /frontMatter must be a boolean/], @@ -583,7 +648,10 @@ test('Templates: variantName names the variants, frontMatter: false reads none', const output = await renderer.render({ template: 'welcome', locale: 'uk' }); assert.deepStrictEqual([output.html, output.subject], ['

    uk

    ', 'S']); const { renderer: broken } = rendererOf({}, { variantName: () => 1 }); - await assert.rejects(broken.render({ template: 'a' }), /variantName must give a name/); + await assert.rejects( + broken.render({ template: 'a' }), + /variantName gives must be a template name/, + ); const { renderer: raw } = rendererOf( { 'a.tpl': '---\nsubject: x\n---\n

    ' }, { frontMatter: false }, From 97f3222aa488c112a65ba2c4d541445b192c584a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:47:12 +0300 Subject: [PATCH 35/83] perf(mongodb): wake on the changes that make a job ready, and reopen a stream The change stream of the outbox matched every insert, update and replace, the outbox's own included: each lease, each extension of a running job (every 20 seconds) and each settling woke the workers, who reserved for nothing. It matches the inserts of ready jobs and the changes that set a job ready (a retry, a release, a lease taken back). A stream that failed, a failover the driver could not resume, stayed closed for good with one warning; it is opened again a second later, then up to every 30 seconds, the wait reset by a change. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++ docs/guide/outbox.md | 7 +++- src/mongodb/outbox.js | 78 +++++++++++++++++++++++++++++++++----- tests/unit/mongodb.test.js | 53 +++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4114c30..d855a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. keeping its version and hash, so a duplicate is still told and a reset link does not wait in the store. Every driver keeps a completed job `dedupeWindowSeconds` (a day, its window of duplicates) and a failed one `retainFailedSeconds` (a week); the outbox drops them by a TTL on `expireAt`. + An idle poll of the outbox is two round trips (leases taken back once a third of a lease, the + pause read once a poll interval, the next due time from its own index); its change stream wakes + one lane per change after which a job may be ready, and is opened again when it fails. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 81bd2d5..9bca7d5 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -82,8 +82,11 @@ Workers poll (`pollIntervalMilliseconds`, a second). An idle poll of a lane is one `findOneAndUpdate` and one read of when the next job is due: a process takes back the leases that ran out once a third of a lease, and reads the pause of the queue once a poll interval (a pause of its own at once). With `changeStream: true`, -on a replica set, a change of the collection wakes them at once, and the poll -is only the fallback. +on a replica set, a change after which a job may be ready (an enqueue, a +retry, a release, a lease taken back) wakes one lane at once, and the poll is +only the fallback; the outbox's own leases and settlings wake none. A stream +that fails is opened again a second later, then up to every 30 seconds +(`queue.change_stream_failed`). ## Indexes and retention diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 3192db6..75a7518 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -10,6 +10,7 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { createLog } = require('../internal/log.js'); const { checkOptionalFunction } = require('../internal/options.js'); +const { sleep } = require('../internal/sleep.js'); const { checkNonNegativeInteger, checkRetainData, @@ -230,19 +231,75 @@ class MongoOutboxSource { } } +// The changes after which a job may be ready: a job enqueued, retried, given +// back or whose lease was taken back. The outbox's own leases, extensions and +// settlings, most of its writes, wake no worker. +const READY_CHANGES = [ + { + $match: { + $or: [ + { operationType: 'insert', 'fullDocument.state': 'ready' }, + { operationType: 'update', 'updateDescription.updatedFields.state': 'ready' }, + { operationType: 'replace', 'fullDocument.state': 'ready' }, + ], + }, + }, +]; +const MAX_RESUBSCRIBE_MS = 30_000; + +const closeStream = async (stream) => { + try { + await stream.close(); + } catch { + // Closed already, or the connection is gone: nothing to close. + } +}; + // Wakes the readers on a change of the collection, where MongoDB has change -// streams (a replica set); the polling goes on without them. +// streams (a replica set); the polling goes on without them. A stream that +// fails (a failover the driver could not resume) is opened again, a second +// later, then up to every 30 seconds. const subscribe = (collection, listener, config) => { - const stream = collection.watch( - [{ $match: { operationType: { $in: ['insert', 'update', 'replace'] } } }], - { fullDocument: 'default' }, - ); - stream.on('change', () => listener()); - stream.on('error', (error) => { - config.log.write('warn', () => ({ error: error?.message }), 'queue.change_stream_failed'); - }); + const stopped = new AbortController(); + let stream; + let failures = 0; + // Closes a stream that failed and opens another after the wait. + const reopen = async (current) => { + if (stream === current) stream = undefined; + await closeStream(current); + const delay = Math.min(MAX_RESUBSCRIBE_MS, 1000 * 2 ** failures); + failures += 1; + try { + await config.sleep(delay, stopped.signal); + } catch { + return; + } + open(); + }; + const open = () => { + if (stopped.signal.aborted) return; + const current = collection.watch(READY_CHANGES, { fullDocument: 'default' }); + stream = current; + let failed = false; + current.on('change', () => { + failures = 0; + listener(); + }); + current.on('error', (error) => { + if (failed) return; + failed = true; + config.log.write( + 'warn', + () => ({ error: error?.message, failures }), + 'queue.change_stream_failed', + ); + reopen(current); + }); + }; + open(); return () => { - stream.close().catch(() => {}); + stopped.abort(); + if (stream !== undefined) closeStream(stream); }; }; @@ -279,6 +336,7 @@ const createMongoOutbox = (options = {}) => { } const config = { now: checkOptionalFunction(options.now, 'now') ?? Date.now, + sleep: checkOptionalFunction(options.sleep, 'sleep') ?? sleep, // The polling driver checks it; the pause is read again as often. pauseReadMilliseconds: options.pollIntervalMilliseconds ?? 1000, // The lease of the workers: none of theirs runs out sooner. diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index d8c472c..7d9fa7e 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -190,14 +190,24 @@ test('MongoDB: the outbox makes its indexes, and wakes its workers on a change s let woken = 0; const unsubscribe = outbox.source.subscribe(() => woken++); const [stream] = collection.streams; + // Only a change after which a job may be ready wakes a worker. assert.deepStrictEqual(stream.pipeline, [ - { $match: { operationType: { $in: ['insert', 'update', 'replace'] } } }, + { + $match: { + $or: [ + { operationType: 'insert', 'fullDocument.state': 'ready' }, + { operationType: 'update', 'updateDescription.updatedFields.state': 'ready' }, + { operationType: 'replace', 'fullDocument.state': 'ready' }, + ], + }, + }, ]); await outbox.source.enqueue(requestOf('a')); assert.strictEqual(woken, 1); stream.listeners.get('error')(new Error('not a replica set')); assert.deepStrictEqual(records, [['queue.change_stream_failed', 'not a replica set']]); unsubscribe(); + await new Promise(setImmediate); assert.strictEqual(collection.streams.length, 0); // A close that fails goes unheard. const failing = outboxOf({ @@ -526,3 +536,44 @@ test('MongoDB: a reserve takes back leases once a third of a lease, and reads th nextAvailableAt: 90_000_000, }); }); + +test('MongoDB: a change stream that fails is opened again, later each time', async () => { + const { manualClock } = require('../helpers/queue/clock.js'); + const clock = manualClock(0); + const records = []; + const collection = fakeCollection(); + const outbox = createMongoOutbox({ + collection, + changeStream: true, + now: clock.now, + sleep: clock.sleep, + logger: { warn: (record) => records.push(record.failures) }, + }); + let woken = 0; + const unsubscribe = outbox.source.subscribe(() => woken++); + const settle = () => new Promise(setImmediate); + collection.streams[0].listeners.get('error')(new Error('failover')); + await settle(); + assert.strictEqual(collection.streams.length, 0); + clock.advance(1000); + await settle(); + assert.strictEqual(collection.streams.length, 1); + collection.streams[0].listeners.get('error')(new Error('again')); + await settle(); + clock.advance(1000); + await settle(); + // The second wait is longer. + assert.strictEqual(collection.streams.length, 0); + clock.advance(1000); + await settle(); + assert.strictEqual(collection.streams.length, 1); + assert.deepStrictEqual(records, [0, 1]); + // A change resets the wait, and wakes a worker. + collection.streams[0].listeners.get('change')({ operationType: 'insert' }); + assert.strictEqual(woken, 1); + collection.streams[0].listeners.get('error')(new Error('once more')); + unsubscribe(); + clock.advance(60_000); + await settle(); + assert.strictEqual(collection.streams.length, 0); +}); From 375d4d4fbadb8effd3ada0aa52f15f9436f0abd6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:47:09 +0300 Subject: [PATCH 36/83] fix(templates): read a template file through the descriptor its size came from The file loader took the size of a template by its path, then read the file whole by its path, so a file that grew in between, or was replaced, was read whole, past maxBytes. It opens the file once, without following a link its last segment became where the system can say so, takes the size of that descriptor, and reads at most maxBytes + 1 bytes through it. Co-Authored-By: Claude Opus 5.5 --- src/templates/loader.js | 104 ++++++++++++++++++++++++---- tests/unit/templates-loader.test.js | 43 ++++++++++++ 2 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/templates/loader.js b/src/templates/loader.js index 6c2631d..ac99685 100644 --- a/src/templates/loader.js +++ b/src/templates/loader.js @@ -80,8 +80,65 @@ const hasExtension = (name, extensions) => { const isMissing = (error) => error?.code === 'ENOENT' || error?.code === 'ENOTDIR'; +// A file is opened without following a link its last segment became after +// its real path was checked, where the system can say so. +const OPEN_FLAGS = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0); +const CHUNK = 64 * 1024; + +// The reads of an open file of `size` bytes, up to `limit`, through the +// descriptor its size was taken from, so that a file that grew since is not +// read whole: each read yields where it goes and is given the bytes read, so +// that one loop serves a descriptor and a handle. +const readsOf = function* (size, limit) { + let buffer = Buffer.allocUnsafe(Math.min(size + 1, limit)); + let total = 0; + for (;;) { + if (total === buffer.length) { + if (total === limit) return buffer; + const grown = Buffer.allocUnsafe(Math.min(total + CHUNK, limit)); + buffer.copy(grown, 0, 0, total); + buffer = grown; + } + const bytesRead = yield { buffer, offset: total }; + if (bytesRead === 0) return buffer.subarray(0, total); + total += bytesRead; + } +}; + +const readAtMost = async (handle, size, limit) => { + const reads = readsOf(size, limit); + let step = reads.next(); + while (!step.done) { + const into = step.value; + const read = await handle.read( + into.buffer, + into.offset, + into.buffer.length - into.offset, + null, + ); + step = reads.next(read.bytesRead); + } + return step.value; +}; + +const readAtMostSync = (fd, size, limit) => { + const reads = readsOf(size, limit); + let step = reads.next(); + while (!step.done) { + const into = step.value; + step = reads.next( + fs.readSync(fd, into.buffer, into.offset, into.buffer.length - into.offset, null), + ); + } + return step.value; +}; + const stripBom = (text) => (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text); +// A file opened as the real path of a name: none when it is gone or is a +// directory, which Windows does not open. +const isNoFile = (error) => isMissing(error) || error?.code === 'EISDIR'; + /** * Makes a loader of the templates in a directory. * @@ -104,14 +161,11 @@ const createFileLoader = (options = {}) => { throw new RangeError(`The template ${JSON.stringify(name)} leads out of ${dir}.`); } }; - const checkStats = (name, stats) => { - if (!stats.isFile()) return false; - if (stats.size > maxBytes) { - throw new RangeError( - `The template ${JSON.stringify(name)} holds more than ${maxBytes} bytes.`, - ); - } - return true; + const tooLong = (name) => + new RangeError(`The template ${JSON.stringify(name)} holds more than ${maxBytes} bytes.`); + const sourceOf = (name, real, bytes) => { + if (bytes.length > maxBytes) throw tooLong(name); + return { name, source: stripBom(bytes.toString('utf8')), path: real }; }; const accepts = (name) => isTemplateName(name) && hasExtension(name, extensions); return { @@ -126,8 +180,21 @@ const createFileLoader = (options = {}) => { throw error; } checkWithin(name, real); - if (!checkStats(name, await fsp.stat(real))) return undefined; - return { name, source: stripBom(await fsp.readFile(real, 'utf8')), path: real }; + let handle; + try { + handle = await fsp.open(real, OPEN_FLAGS); + } catch (error) { + if (isNoFile(error)) return undefined; + throw error; + } + try { + const stats = await handle.stat(); + if (!stats.isFile()) return undefined; + if (stats.size > maxBytes) throw tooLong(name); + return sourceOf(name, real, await readAtMost(handle, stats.size, maxBytes + 1)); + } finally { + await handle.close(); + } }, loadSync(name) { if (!accepts(name)) return undefined; @@ -140,8 +207,21 @@ const createFileLoader = (options = {}) => { throw error; } checkWithin(name, real); - if (!checkStats(name, fs.statSync(real))) return undefined; - return { name, source: stripBom(fs.readFileSync(real, 'utf8')), path: real }; + let fd; + try { + fd = fs.openSync(real, OPEN_FLAGS); + } catch (error) { + if (isNoFile(error)) return undefined; + throw error; + } + try { + const stats = fs.fstatSync(fd); + if (!stats.isFile()) return undefined; + if (stats.size > maxBytes) throw tooLong(name); + return sourceOf(name, real, readAtMostSync(fd, stats.size, maxBytes + 1)); + } finally { + fs.closeSync(fd); + } }, }; }; diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index 0067f71..fb87e5a 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -3,6 +3,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const fs = require('node:fs'); +const fsp = require('node:fs/promises'); const os = require('node:os'); const path = require('node:path'); const { @@ -64,6 +65,48 @@ test('Templates: the file loader reads a name under its directory, its BOM dropp ); }); +test('Templates: the file loader reads no more than maxBytes of a file that grew since its size', async (t) => { + // Before: the size was taken by path and the file read whole by path + // after, so a file that grew in between, or was replaced, was read whole. + const { root } = makeTree(t); + const small = { isFile: () => true, size: 10 }; + t.mock.method(fsp, 'stat', async () => small); + t.mock.method(fs, 'statSync', () => small); + const open = fsp.open; + t.mock.method(fsp, 'open', async (...args) => { + const handle = await open(...args); + handle.stat = async () => small; + return handle; + }); + t.mock.method(fs, 'fstatSync', () => small); + const loader = createFileLoader({ dir: root, maxBytes: 1024 }); + await assert.rejects(loader.load('big.ejs'), /more than 1024 bytes/); + assert.throws(() => loader.loadSync('big.ejs'), /more than 1024 bytes/); + // A file that grew within maxBytes is read whole, through the descriptor. + assert.strictEqual((await createFileLoader({ dir: root }).load('big.ejs')).source.length, 2048); + assert.strictEqual(createFileLoader({ dir: root }).loadSync('big.ejs').source.length, 2048); +}); + +test('Templates: the file loader takes a directory it cannot open for none', async (t) => { + // Windows opens no directory: EISDIR, where the others open it and say it + // is no file. + const { root } = makeTree(t); + const denied = (code) => Object.assign(new Error(code), { code }); + let code = 'EISDIR'; + t.mock.method(fsp, 'open', async () => { + throw denied(code); + }); + t.mock.method(fs, 'openSync', () => { + throw denied(code); + }); + const loader = createFileLoader({ dir: root }); + assert.strictEqual(await loader.load('folder.ejs'), undefined); + assert.strictEqual(loader.loadSync('folder.ejs'), undefined); + code = 'EACCES'; + await assert.rejects(loader.load('welcome.ejs'), { code: 'EACCES' }); + assert.throws(() => loader.loadSync('welcome.ejs'), { code: 'EACCES' }); +}); + test('Templates: the file loader follows no symbolic link out of its directory', async (t) => { if (process.platform === 'win32') { t.skip('symbolic links need privileges on Windows'); From 7f1a5d279f49bcad7501359b4952016ce3414af6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:50:16 +0300 Subject: [PATCH 37/83] feat(templates): tell the step, engine and input of every failure A TemplateRenderError told the template and the phase, and no engine for a layout or for an output that was no string; a TemplateNotFoundError told neither, so a worker had to read the message to tell a missing template from a missing partial. Every error of ./templates now carries `phase` (`resolve` for the lookup of the template asked for), `engine`, `template`, the one that failed, and `requested` and `locale`, what the render was asked for, copied per render, as renders that wait for one compile share its failure. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 6 ++ src/templates/errors.js | 74 ++++++++++++++++++-- src/templates/includes.js | 8 +-- src/templates/index.d.ts | 62 +++++++++++++---- src/templates/renderer.js | 48 ++++++++++--- tests/types/templates.test-d.ts | 4 ++ tests/unit/templates-renderer.test.js | 98 ++++++++++++++++++++++++++- 7 files changed, 261 insertions(+), 39 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 1915028..54ba321 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -266,6 +266,12 @@ Handlebars); a partial that names itself is no deeper for it. The file loader re | `TemplateLoadError` (`MAILER_TEMPLATE_LOAD`) | the loader failed (`EMFILE`, a database timeout); `template` names what it read, `cause` is its error | `true`; unset for a bug of the loader (`TypeError`…) | | `TypeError`, `RangeError` | a name that is not one, a bad locale or context | — | +Each of the three tells what a worker needs without reading its message: +`phase`, the step that failed (`resolve`, `front-matter`, `compile`, `render`, +`layout`, `finish`, `inline-css`); `engine`; `template`, the template that +failed as its loader names it (`welcome.uk.hbs`, or a partial of it); and +`requested` and `locale`, what the render was asked for. + The queue fails a job at once on the first two and the last, and retries an outage of the loader. An engine that wraps what a loader threw in its own error (Nunjucks, Liquid) does not hide it: the error of ./templates is found among its diff --git a/src/templates/errors.js b/src/templates/errors.js index 48712f5..54e56b2 100644 --- a/src/templates/errors.js +++ b/src/templates/errors.js @@ -4,6 +4,17 @@ // its job at once rather than retrying what fails again the same way; a // loader that fails may pass on another attempt. +// What a queue reads of a failure without its text: the step of the render +// that failed, the engine, and the template and locale the render was asked +// for, as the template of the error is the one that failed. +const FIELDS = ['phase', 'engine', 'requested', 'locale']; + +const describe = (error, options) => { + for (const field of FIELDS) { + if (options[field] !== undefined) error[field] = options[field]; + } +}; + // The names a message lists, of all those tried. const MAX_NAMES = 8; @@ -21,9 +32,10 @@ const listOf = (names) => { */ class TemplateNotFoundError extends Error { /** - * @param template The name asked for. + * @param template The name looked up. * @param tried The names looked up, in order. - * @param options `from`, the template that included it, and `cause`. + * @param options `from`, the template that included it; `phase`, `engine`, + * `requested` and `locale`; and `cause`. */ constructor(template, tried, options = {}) { const from = options.from === undefined ? '' : ` (included by ${options.from})`; @@ -35,6 +47,7 @@ class TemplateNotFoundError extends Error { this.template = template; if (options.from !== undefined) this.from = options.from; this.tried = Object.freeze([...tried]); + describe(this, options); } } @@ -49,7 +62,8 @@ class TemplateRenderError extends Error { * @param phase Where: `front-matter`, `compile`, `render`, `layout`, * `finish` or `inline-css`. * @param message What happened. - * @param options `engine`, the name of the engine, and `cause`. + * @param options `engine`, the name of the engine; `requested` and + * `locale`; and `cause`. */ constructor(template, phase, message, options = {}) { super(`The template ${JSON.stringify(template)} failed to ${phase}: ${message}`, options); @@ -57,8 +71,7 @@ class TemplateRenderError extends Error { this.code = 'MAILER_TEMPLATE_RENDER'; this.retryable = false; this.template = template; - this.phase = phase; - if (options.engine !== undefined) this.engine = options.engine; + describe(this, { ...options, phase }); } } @@ -82,8 +95,8 @@ const isBug = (error) => { class TemplateLoadError extends Error { /** * @param template The name the loader was asked for. - * @param options `from`, the template that included it, and `cause`, the - * loader's error. + * @param options `from`, the template that included it; `phase`, + * `engine`, `requested` and `locale`; and `cause`, the loader's error. */ constructor(template, options = {}) { const { cause } = options; @@ -94,6 +107,7 @@ class TemplateLoadError extends Error { if (cause?.retryable !== false && !isBug(cause)) this.retryable = true; this.template = template; if (options.from !== undefined) this.from = options.from; + describe(this, options); } } @@ -115,7 +129,53 @@ const templateErrorOf = (error) => { return undefined; }; +const isTemplateError = (error) => + error instanceof TemplateNotFoundError || + error instanceof TemplateRenderError || + error instanceof TemplateLoadError; + +/** + * Gives an error of ./templates the step and the engine it failed in, when + * it has none: what an include threw is the same for every render that waits + * for the one compile. + * + * @internal + */ +const inPhase = (error, phase, engine) => { + if (!isTemplateError(error)) return error; + error.phase ??= phase; + if (engine !== undefined) error.engine ??= engine; + return error; +}; + +/** + * The error of a render: an error of ./templates, copied with the template + * and the locale the render was asked for, as one error may reach several + * renders that wait for one compile; any other as it is. + * + * @internal + */ +const forRender = (error, requested, locale) => { + if (!isTemplateError(error)) return error; + const copy = Object.create(Object.getPrototypeOf(error)); + for (const key of Object.getOwnPropertyNames(error)) { + Object.defineProperty(copy, key, Object.getOwnPropertyDescriptor(error, key)); + } + // V8 reads the stack of the error it was captured on alone. + Object.defineProperty(copy, 'stack', { + value: error.stack, + writable: true, + configurable: true, + }); + copy.requested = requested; + if (locale !== undefined) copy.locale = locale; + return copy; +}; + module.exports = { + forRender, + inPhase, + isTemplateError, TemplateLoadError, TemplateNotFoundError, TemplateRenderError, diff --git a/src/templates/includes.js b/src/templates/includes.js index 9e2bc1c..d0d635b 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -2,7 +2,7 @@ // template: through the loader of its renderer, so that they are checked and // kept as its templates are, and never read from disk by the engine itself. -const { TemplateLoadError, TemplateNotFoundError, TemplateRenderError } = require('./errors.js'); +const { isTemplateError, TemplateLoadError, TemplateNotFoundError } = require('./errors.js'); const { checkTemplateName, isTemplateName, notTemplateName } = require('./loader.js'); const { createCache, memoize } = require('./cache.js'); @@ -85,11 +85,7 @@ const READ = 'The name of a template read'; // as a TemplateLoadError, which an engine that wraps it does not make a // failure of the template, as it may pass on another attempt. const loadFailure = (name, error) => - error instanceof TemplateNotFoundError || - error instanceof TemplateRenderError || - error instanceof TemplateLoadError - ? error - : new TemplateLoadError(name, { cause: error }); + isTemplateError(error) ? error : new TemplateLoadError(name, { cause: error }); // The bounds of the render whose engine runs now, synchronously: set around // each call of an engine, so that the includes it looks up before it returns diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index f1f3665..9b1e67c 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -290,25 +290,59 @@ export declare function createTemplateRenderer< TSource = string, >(options: TemplateRendererOptions): TemplateRenderer; +/** Where a render failed: looking up its template, or one of its steps. */ +export type TemplateRenderPhase = + | 'resolve' + | 'front-matter' + | 'compile' + | 'render' + | 'layout' + | 'finish' + | 'inline-css'; + +/** + * What an error of ./templates tells of a render, for a worker to read + * without its message. A render sets `requested` and `locale`; the step and + * the engine are set where they are known. + */ +export interface TemplateErrorFields { + /** The step of the render that failed. */ + readonly phase?: TemplateRenderPhase; + /** The name of the engine of the template that failed. */ + readonly engine?: string; + /** The template the render was asked for: `welcome`, where `template` may be `welcome.uk.hbs` or a partial. */ + readonly requested?: string; + /** The locale the render was asked for, canonical (`uk-UA`); none when it was asked for none. */ + readonly locale?: string; +} + /** The options of the errors of ./templates. */ -export interface TemplateNotFoundErrorOptions extends ErrorOptions { +export interface TemplateErrorOptions extends ErrorOptions, TemplateErrorFields {} + +/** The options of {@link TemplateNotFoundError}. */ +export interface TemplateNotFoundErrorOptions extends TemplateErrorOptions { /** The template that included it, for an include, a partial or a layout. */ readonly from?: string; } /** No template, include, partial or layout by that name. Not retryable. */ -export declare class TemplateNotFoundError extends Error { +export declare class TemplateNotFoundError extends Error implements TemplateErrorFields { readonly code: 'MAILER_TEMPLATE_NOT_FOUND'; readonly retryable: false; + /** The name looked up: the template asked for, or an include, partial or layout of it. */ readonly template: string; readonly from?: string; /** The names looked up, in order. */ readonly tried: readonly string[]; + readonly phase?: TemplateRenderPhase; + readonly engine?: string; + readonly requested?: string; + readonly locale?: string; constructor(template: string, tried: readonly string[], options?: TemplateNotFoundErrorOptions); } /** The options of {@link TemplateLoadError}. */ -export interface TemplateLoadErrorOptions extends ErrorOptions { +export interface TemplateLoadErrorOptions extends TemplateErrorOptions { /** The template that included it, for an include, a partial or a layout. */ readonly from?: string; } @@ -320,27 +354,24 @@ export interface TemplateLoadErrorOptions extends ErrorOptions { * (`TypeError`, `RangeError`…) or says `retryable: false`: then `retryable` * is unset, for a queue to classify by the cause. */ -export declare class TemplateLoadError extends Error { +export declare class TemplateLoadError extends Error implements TemplateErrorFields { readonly code: 'MAILER_TEMPLATE_LOAD'; readonly retryable?: true; /** The name the loader was asked for. */ readonly template: string; readonly from?: string; + readonly phase?: TemplateRenderPhase; + readonly engine?: string; + readonly requested?: string; + readonly locale?: string; constructor(template: string, options?: TemplateLoadErrorOptions); } -/** Where a template failed. */ -export type TemplateRenderPhase = - | 'front-matter' - | 'compile' - | 'render' - | 'layout' - | 'finish' - | 'inline-css'; - /** The options of {@link TemplateRenderError}. */ export interface TemplateRenderErrorOptions extends ErrorOptions { readonly engine?: string; + readonly requested?: string; + readonly locale?: string; } /** @@ -348,12 +379,15 @@ export interface TemplateRenderErrorOptions extends ErrorOptions { * helper that threw, MJML that does not validate. The engine's error is its * cause. Not retryable: the same input fails again. */ -export declare class TemplateRenderError extends Error { +export declare class TemplateRenderError extends Error implements TemplateErrorFields { readonly code: 'MAILER_TEMPLATE_RENDER'; readonly retryable: false; + /** The template that failed, as its loader names it: `welcome.uk.hbs`, a layout, a partial. */ readonly template: string; readonly phase: TemplateRenderPhase; readonly engine?: string; + readonly requested?: string; + readonly locale?: string; constructor( template: string, phase: TemplateRenderPhase, diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 0b32850..e048da3 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -10,7 +10,13 @@ const { decodeEntities } = require('../render/entities.js'); const { messageOf } = require('../internal/decorator.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { createCache, memoize } = require('./cache.js'); -const { TemplateNotFoundError, TemplateRenderError, templateErrorOf } = require('./errors.js'); +const { + forRender, + inPhase, + TemplateNotFoundError, + TemplateRenderError, + templateErrorOf, +} = require('./errors.js'); const { parseFrontMatter } = require('./front-matter.js'); const { createIncludes, withBounds } = require('./includes.js'); const { checkTemplateName } = require('./loader.js'); @@ -170,11 +176,11 @@ const checkInput = (input) => { }; const wrapError = (error, template, phase, engine) => { - if (error instanceof TemplateRenderError) return error; + if (error instanceof TemplateRenderError) return inPhase(error, phase, engine?.name); // An engine may wrap what its include threw: a missing include stays one, // and a loader's failure stays the loader's, which may pass on another try. const found = templateErrorOf(error); - if (found !== undefined) return found; + if (found !== undefined) return inPhase(found, phase, engine?.name); return new TemplateRenderError(template, phase, messageOf(error), { cause: error, engine: engine?.name, @@ -242,7 +248,11 @@ const createTemplateRenderer = (options = {}) => { // Without a cache, each render reads and compiles anew, includes too. const current = config.cache ? state : { ...state, files: makeIncludes() }; const bounds = { includes: 0, maxIncludes: config.maxIncludes }; - return renderTemplate(current, checked, input, bounds); + try { + return await renderTemplate(current, checked, input, bounds); + } catch (error) { + throw forRender(error, checked.template, checked.locale); + } }, clear() { // Engines key what they keep by the includes, so they make it anew too. @@ -307,7 +317,7 @@ const resolveTemplate = async (state, template, locale) => { }; } } - throw new TemplateNotFoundError(template, tried); + throw new TemplateNotFoundError(template, tried, { phase: 'resolve' }); }; const resolvedFor = (state, template, locale) => @@ -397,15 +407,23 @@ const applyLayout = async (state, entry, html, scope, locale, bounds) => { try { resolution = await resolvedFor(state, name, locale); } catch (error) { - if (!(error instanceof TemplateNotFoundError)) throw error; - throw new TemplateNotFoundError(name, error.tried, { from: entry.name, cause: error }); + if (!(error instanceof TemplateNotFoundError)) throw inPhase(error, 'layout'); + throw new TemplateNotFoundError(name, error.tried, { + from: entry.name, + cause: error, + phase: 'layout', + }); } if (resolution.html === undefined) { - throw new TemplateRenderError(entry.name, 'layout', `The layout ${name} has no HTML.`); + throw new TemplateRenderError(entry.name, 'layout', `The layout ${name} has no HTML.`, { + engine: entry.engine.name, + }); } const layout = await compiledFor(state, resolution.html, 'html', bounds); if (layout.fields?.layout !== undefined) { - throw new TemplateRenderError(layout.name, 'layout', 'A layout has no layout of its own.'); + throw new TemplateRenderError(layout.name, 'layout', 'A layout has no layout of its own.', { + engine: layout.engine.name, + }); } const wrapped = await renderString( state, @@ -420,7 +438,12 @@ const applyLayout = async (state, entry, html, scope, locale, bounds) => { const renderTemplate = async (state, checked, input, bounds) => { const { config } = state; - const resolution = await resolvedFor(state, checked.template, checked.locale); + let resolution; + try { + resolution = await resolvedFor(state, checked.template, checked.locale); + } catch (error) { + throw inPhase(error, 'resolve'); + } const scope = { ...(await globalsOf(config, input)), ...checked.context }; const main = await compiledFor( state, @@ -479,13 +502,16 @@ const outputOf = (config, main, resolution, parts) => { main.name, 'render', `The template gave its ${key} as no string.`, + { engine: main.engine.name }, ); } checkOutput(config, 'render', main.name, main.engine, parts[key]); } const text = parts.text ?? (config.autoText && html !== undefined ? htmlToText(html) : undefined); if (html === undefined && text === undefined) { - throw new TemplateRenderError(main.name, 'render', 'The template gave neither html nor text.'); + throw new TemplateRenderError(main.name, 'render', 'The template gave neither html nor text.', { + engine: main.engine.name, + }); } const output = { template: main.name }; if (html !== undefined) output.html = html; diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index 9cf5fc3..bc8273d 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -28,6 +28,7 @@ import { type TemplateInputOf, type TemplateOutput, type TemplateRenderer, + type TemplateRenderPhase, } from '../../src/templates/index.js'; // The engines of the real packages, as typed, fit the structural modules. @@ -129,6 +130,9 @@ expectType(missing.retryable); expectType(missing.tried); const broken = new TemplateRenderError('welcome', 'compile', 'x', { engine: 'handlebars' }); expectType<'MAILER_TEMPLATE_RENDER'>(broken.code); +expectType(broken.requested); +expectType(missing.phase); +expectType(missing.locale); const failed = new TemplateLoadError('welcome.hbs', { from: 'layout', cause: new Error('EMFILE') }); expectType<'MAILER_TEMPLATE_LOAD'>(failed.code); expectType(failed.retryable); diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index f50d016..025ea76 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -336,6 +336,100 @@ test('Templates: a template that is missing at first is found once it is there', assert.strictEqual((await renderer.render({ template: 'late' })).html, '

    late

    '); }); +test('Templates: every failure tells its step, engine, template and the input it was for', async () => { + // Before: the template and the phase alone, and no engine for a layout or + // an output that was no string, so a worker had to read the message. + const templates = { + 'welcome.uk.tpl': '

    {{x}}

    ', + 'include.tpl': '{{> missing}}', + 'layout.tpl': '---\nlayout: other\n---\n{{{body}}}', + 'other.tpl': '{{{body}}}', + 'framed.tpl': '---\nlayout: layout\n---\n

    ', + 'lost.tpl': '---\nlayout: gone\n---\n

    ', + }; + const { renderer } = rendererOf(templates); + const fields = (error) => ({ + name: error.name, + phase: error.phase, + engine: error.engine, + template: error.template, + requested: error.requested, + locale: error.locale, + }); + const failure = (input) => renderer.render(input).catch(fields); + assert.deepStrictEqual(await failure({ template: 'nope', locale: 'uk' }), { + name: 'TemplateNotFoundError', + phase: 'resolve', + engine: undefined, + template: 'nope', + requested: 'nope', + locale: 'uk', + }); + assert.deepStrictEqual(await failure({ template: 'include' }), { + name: 'TemplateNotFoundError', + phase: 'compile', + engine: 'fake', + template: 'missing', + requested: 'include', + locale: undefined, + }); + assert.deepStrictEqual( + await failure({ template: 'welcome', locale: 'uk-UA', context: { explode: true } }), + { + name: 'TemplateRenderError', + phase: 'render', + engine: 'fake', + template: 'welcome.uk.tpl', + requested: 'welcome', + locale: 'uk-UA', + }, + ); + assert.deepStrictEqual(await failure({ template: 'framed' }), { + name: 'TemplateRenderError', + phase: 'layout', + engine: 'fake', + template: 'layout.tpl', + requested: 'framed', + locale: undefined, + }); + assert.deepStrictEqual((await failure({ template: 'lost' })).phase, 'layout'); + // Renders that wait for one compile each get their own input. + const { renderer: shared } = rendererOf( + { 'a.tpl': '{{!syntax' }, + { engine: fakeEngine({ async: true }) }, + ); + const [first, second] = await Promise.all([ + shared.render({ template: 'a', locale: 'uk' }).catch((e) => e), + shared.render({ template: 'a.tpl', locale: 'de' }).catch((e) => e), + ]); + assert.deepStrictEqual( + [first.requested, first.locale, second.requested, second.locale], + ['a', 'uk', 'a.tpl', 'de'], + ); + assert.ok(first.cause instanceof SyntaxError); + assert.match(first.stack, /SyntaxError|TemplateRenderError/); + const load = await createTemplateRenderer({ + engine: fakeEngine(), + loader: { + load: () => { + throw new Error('down'); + }, + }, + }) + .render({ template: 'x' }) + .catch(fields); + assert.deepStrictEqual( + [load.name, load.phase, load.requested], + ['TemplateLoadError', 'resolve', 'x'], + ); + // The parts of an engine that are no strings name it too. + const parts = createTemplateRenderer({ + engine: { name: 'parts', extensions: [], compile: () => () => ({}) }, + loader: createMemoryLoader({ x: '' }), + }); + assert.strictEqual((await parts.render({ template: 'x' }).catch((e) => e)).engine, 'parts'); +}); + test('Templates: failures are classified by phase, the engine error their cause', async () => { const { renderer } = rendererOf({ 'syntax.tpl': '

    {{!syntax

    ', @@ -418,7 +512,9 @@ test('Templates: a loader that fails fails as a load of its template, which may }, }, }); - assert.strictEqual(await own.render({ template: 'x' }).catch((e) => e), refusal); + const refused = await own.render({ template: 'x' }).catch((e) => e); + assert.ok(refused instanceof TemplateNotFoundError); + assert.deepStrictEqual([refused.template, refused.message], ['x', refusal.message]); }); test('Templates: finish makes the HTML of the outermost template, then the CSS is inlined', async () => { From f29f9abc58124cb4611bb525fc26eac903ddfd37 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:55:37 +0300 Subject: [PATCH 38/83] feat(templates): time out a render, stop it at its signal, and keep nothing that hangs A render had no timeout and took no signal, and a read or a compile that never settled was kept by the cache and waited for by every render after it, until clear(). A render takes renderTimeoutMilliseconds (30000; 0 and Infinity for none), after which it fails with a retryable TemplateRenderError of phase `timeout`; a read or a compile still pending then is forgotten and aborted, the loader told through `load(name, { signal })`. `render({ ..., signal })` rejects with its reason at its abort; without the cache, the reads are the render's own and the loader is given that signal. The file loader stops between its reads. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 20 +++++ src/templates/cache.js | 36 ++++++-- src/templates/errors.js | 10 ++- src/templates/includes.js | 123 ++++++++++++++++---------- src/templates/index.d.ts | 40 ++++++++- src/templates/loader.js | 8 +- src/templates/renderer.js | 86 +++++++++++++++--- tests/types/templates.test-d.ts | 10 +++ tests/unit/templates-loader.test.js | 23 +++++ tests/unit/templates-renderer.test.js | 106 ++++++++++++++++++++++ 10 files changed, 389 insertions(+), 73 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 54ba321..d017b5e 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -257,6 +257,26 @@ Pug and Handlebars look up as they compile, until their first await. The depth is counted where the engine names the template an include is written in (Eta, Handlebars); a partial that names itself is no deeper for it. The file loader reads no template over its `maxBytes` (1 MiB). +## Timeouts and aborts + +A render takes `renderTimeoutMilliseconds` (30000) at most; then it fails with +a `TemplateRenderError` of phase `timeout`, which is retryable, as what it +waited for, a loader or an async helper, may answer on another attempt. A read +or a compile still pending then is not kept: the next render makes it anew, +rather than wait for one that hangs, and the loader is told to stop through +the `signal` of `load(name, { signal })`. + +`render({ template, context, locale, signal })` stops at the abort of +`signal`, rejecting with its reason, as the worker of the queue passes it when +it shuts down. With the cache on, a read or a compile that other renders wait +for goes on for them; without it, the reads are the render's own, and the +loader is given its signal. + +Neither can stop a template that loops without ever awaiting: JavaScript runs +it to its end. Liquid's `renderLimit` (on by default) is checked as it +renders; with the other engines, keep such templates out of the hands of +those who could write one (see [who may write a template](#who-may-write-a-template)). + ## Errors | Thrown | When | `retryable` | diff --git a/src/templates/cache.js b/src/templates/cache.js index 995e2e7..60ebd20 100644 --- a/src/templates/cache.js +++ b/src/templates/cache.js @@ -2,6 +2,8 @@ // used entry dropped first, on a Map, which iterates in the order of // insertion: an entry read is written anew, so the first one is the oldest. +const { timerMs } = require('../internal/sleep.js'); + /** * Makes a cache of at most `maxEntries` entries. With `maxEntries` 0 it keeps * nothing. @@ -34,19 +36,43 @@ const createCache = (maxEntries) => { }; }; +// A timeout of none: 0, Infinity or undefined. +const hasTimeout = (milliseconds) => + milliseconds !== undefined && milliseconds > 0 && milliseconds !== Number.POSITIVE_INFINITY; + /** * The entry of `key`, made by `make` once while it is kept; a promise that - * rejects is not kept, so the next call makes it again. + * rejects is not kept, so the next call makes it again, nor is one still + * pending after `timeoutMilliseconds`, which is aborted: what hangs, a read + * or a compile, is made anew by the next call rather than waited for by + * every one after. * + * @param make Makes the entry, given the signal of its timeout, if any. * @internal */ -const memoize = (cache, key, make) => { +const memoize = (cache, key, make, timeoutMilliseconds) => { const kept = cache.get(key); if (kept !== undefined) return kept; - const made = Promise.resolve().then(make); + const controller = hasTimeout(timeoutMilliseconds) ? new AbortController() : undefined; + const made = Promise.resolve().then(() => make(controller?.signal)); cache.set(key, made); - made.catch(() => cache.forget(key, made)); + let timer; + if (controller !== undefined) { + timer = setTimeout(() => { + cache.forget(key, made); + controller.abort(new DOMException('The read or compile took too long.', 'TimeoutError')); + }, timerMs(timeoutMilliseconds)); + // The render that waits for it keeps the process, by a timer of its own. + timer.unref?.(); + } + made.then( + () => clearTimeout(timer), + () => { + clearTimeout(timer); + cache.forget(key, made); + }, + ); return made; }; -module.exports = { createCache, memoize }; +module.exports = { createCache, hasTimeout, memoize }; diff --git a/src/templates/errors.js b/src/templates/errors.js index 54e56b2..a8e8dc3 100644 --- a/src/templates/errors.js +++ b/src/templates/errors.js @@ -54,22 +54,24 @@ class TemplateNotFoundError extends Error { /** * A template that does not render: its syntax, a value it cannot read, a * helper that threw, MJML that does not validate. The engine's error is its - * cause. + * cause. It is not retryable, but for a render that ran out of time, whose + * loader or helper may answer on another attempt. */ class TemplateRenderError extends Error { /** * @param template The template that failed. * @param phase Where: `front-matter`, `compile`, `render`, `layout`, - * `finish` or `inline-css`. + * `finish`, `inline-css`, or `timeout` for a render that ran out of time. * @param message What happened. * @param options `engine`, the name of the engine; `requested` and * `locale`; and `cause`. */ constructor(template, phase, message, options = {}) { - super(`The template ${JSON.stringify(template)} failed to ${phase}: ${message}`, options); + const failed = phase === 'timeout' ? 'did not render in time' : `failed to ${phase}`; + super(`The template ${JSON.stringify(template)} ${failed}: ${message}`, options); this.name = 'TemplateRenderError'; this.code = 'MAILER_TEMPLATE_RENDER'; - this.retryable = false; + this.retryable = phase === 'timeout'; this.template = template; describe(this, { ...options, phase }); } diff --git a/src/templates/includes.js b/src/templates/includes.js index d0d635b..8407a59 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -121,51 +121,32 @@ const spend = () => { } }; -/** - * Makes the includes of a renderer: the reads of its loader, kept while its - * cache is on, and the lookup of a name an engine includes. - * - * @param loader The loader of the renderer. - * @param options `partials`, the directories searched by a bare name after the - * root, `cache`, `maxEntries` and `maxIncludeDepth`. - * @returns `{ includes, top }`: the includes, one object per renderer and per - * `clear()`, and the mark of a template the renderer compiles itself, which - * is no include deep. - * @internal - */ -const createIncludes = (loader, options) => { - const { partials, cache, maxIncludeDepth } = options; - const sources = createCache(cache ? options.maxEntries : 0); - const syncSources = createCache(cache ? options.maxEntries : 0); - // How many includes deep each template is, at least, as an include is - // written in a template: a chain of them deeper than maxIncludeDepth is - // refused, and a template that includes itself is as deep as before. - const depths = createCache(options.maxEntries); - const depthOf = (include, from) => { - const depth = (from === undefined ? 0 : (depths.get(from) ?? 0)) + 1; - if (depth > maxIncludeDepth) { - throw new RangeError( - `The include ${JSON.stringify(include)} of ${from} is more than ${maxIncludeDepth} ` + - 'includes deep (maxIncludeDepth).', - ); - } - return depth; +// The reads of a loader, kept while the cache is on. A loader is given +// template names alone, whoever asks: an engine, or the variants of the +// renderer; and it is told when to stop: at the timeout of a read, which +// renders that wait for it share, or when the one render it is for is +// aborted. +const readsOf = (loader, options) => { + const sources = createCache(options.cache ? options.maxEntries : 0); + const syncSources = createCache(options.cache ? options.maxEntries : 0); + const signalOf = (timeout) => { + if (options.signal === undefined) return timeout; + return timeout === undefined ? options.signal : AbortSignal.any([timeout, options.signal]); }; - const reached = (name, depth) => { - const known = depths.get(name); - if (known === undefined || depth < known) depths.set(name, depth); - }; - // A loader is given template names alone, whoever asks: an engine, or the - // variants of the renderer. const read = (name) => { if (!isTemplateName(name)) return Promise.reject(notTemplateName(name, READ)); - return memoize(sources, name, async () => { - try { - return (await loader.load(name)) ?? undefined; - } catch (error) { - throw loadFailure(name, error); - } - }); + return memoize( + sources, + name, + async (timeout) => { + try { + return (await loader.load(name, { signal: signalOf(timeout) })) ?? undefined; + } catch (error) { + throw loadFailure(name, error); + } + }, + options.timeoutMilliseconds, + ); }; const readSync = (name) => { checkTemplateName(name, READ); @@ -185,38 +166,84 @@ const createIncludes = (loader, options) => { syncSources.set(name, { source }); return source; }; + return { read, readSync }; +}; + +// How many includes deep each template is, at least, as an include is +// written in a template: a chain of them deeper than maxIncludeDepth is +// refused, and a template that includes itself is as deep as before. +const depthsOf = (options) => { + const { maxIncludeDepth } = options; + const depths = createCache(options.maxEntries); + return { + of(include, from) { + const depth = (from === undefined ? 0 : (depths.get(from) ?? 0)) + 1; + if (depth > maxIncludeDepth) { + throw new RangeError( + `The include ${JSON.stringify(include)} of ${from} is more than ${maxIncludeDepth} ` + + 'includes deep (maxIncludeDepth).', + ); + } + return depth; + }, + reached(name, depth) { + const known = depths.get(name); + if (known === undefined || depth < known) depths.set(name, depth); + }, + top: (name) => depths.set(name, 0), + }; +}; + +/** + * Makes the includes of a renderer: the reads of its loader, kept while its + * cache is on, and the lookup of a name an engine includes. + * + * @param loader The loader of the renderer. + * @param options `partials`, the directories searched by a bare name after the + * root, `cache`, `maxEntries`, `maxIncludeDepth`, `timeoutMilliseconds`, after + * which a read that hangs is aborted and read anew, and `signal`, that of the + * one render the includes are for, without a cache. + * @returns `{ includes, top }`: the includes, one object per renderer and per + * `clear()`, and the mark of a template the renderer compiles itself, which + * is no include deep. + * @internal + */ +const createIncludes = (loader, options) => { + const { partials } = options; + const { read, readSync } = readsOf(loader, options); + const depths = depthsOf(options); const includes = Object.freeze({ partials, - cache, + cache: options.cache, join: joinName, read, readSync, async lookup(include, from, extensions = ['']) { spend(); - const depth = depthOf(include, from); + const depth = depths.of(include, from); const names = candidatesOf(include, from, extensions, partials); for (const name of names) { const source = await read(name); if (source === undefined) continue; - reached(source.name, depth); + depths.reached(source.name, depth); return source; } throw new TemplateNotFoundError(include, names, { from }); }, lookupSync(include, from, extensions = ['']) { spend(); - const depth = depthOf(include, from); + const depth = depths.of(include, from); const names = candidatesOf(include, from, extensions, partials); for (const name of names) { const source = readSync(name); if (source === undefined) continue; - reached(source.name, depth); + depths.reached(source.name, depth); return source; } throw new TemplateNotFoundError(include, names, { from }); }, }); - return { includes, top: (name) => depths.set(name, 0) }; + return { includes, top: depths.top }; }; module.exports = { createIncludes, joinName, withBounds }; diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 9b1e67c..1687cd0 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -19,6 +19,15 @@ export interface TemplateInput { * `defaultLocale` so, then the template without a locale. */ readonly locale?: string; + /** The tenant the message is for, as the worker of `./queue` gives it: for `globals`, which the renderer reads nothing of. */ + readonly tenantId?: string; + /** + * Stops the render, which rejects with its reason. With the cache on, a + * read or a compile that other renders wait for goes on for them; without + * it, the reads of the render are its own, and the loader is given the + * signal. + */ + readonly signal?: AbortSignal; } /** @@ -46,14 +55,25 @@ export interface TemplateSource { readonly path?: string; } +/** What a loader is told as it reads. */ +export interface TemplateLoadOptions { + /** + * Aborted when the read is no longer waited for: at the timeout of the + * render (`TimeoutError`), or, without a cache, at the abort of the one + * render it is for. A loader that can stop, a query or a fetch, stops. + */ + readonly signal?: AbortSignal; +} + /** * Where templates are read by name: relative to the root of the loader, `/` - * between directories, no `.` or `..` segment. + * between directories, no segment starting with `.`. */ export interface TemplateLoader { /** The template of that name, or undefined when there is none. */ load( name: string, + options?: TemplateLoadOptions, ): TemplateSource | undefined | Promise | undefined>; /** * The same, synchronously: EJS, Pug, Eta and Mustache read their includes @@ -251,6 +271,14 @@ export interface TemplateRendererOptions { readonly maxIncludeDepth?: number; /** The largest HTML, text or subject a render gives, in bytes of UTF-8. Infinity for no bound. @default 10485760 */ readonly maxOutputBytes?: number; + /** + * How long a render may take: then it fails with a retryable + * `TemplateRenderError` of phase `timeout`, and a read or a compile it + * waited for is made anew by the next render rather than kept. 0 or + * Infinity for none. A template that loops without awaiting cannot be + * stopped so: Liquid's `renderLimit` stops one. @default 30000 + */ + readonly renderTimeoutMilliseconds?: number; } /** A renderer of templates by name, a `Renderer` of `./render`. */ @@ -298,7 +326,8 @@ export type TemplateRenderPhase = | 'render' | 'layout' | 'finish' - | 'inline-css'; + | 'inline-css' + | 'timeout'; /** * What an error of ./templates tells of a render, for a worker to read @@ -377,11 +406,14 @@ export interface TemplateRenderErrorOptions extends ErrorOptions { /** * A template that does not render: its syntax, a value it cannot read, a * helper that threw, MJML that does not validate. The engine's error is its - * cause. Not retryable: the same input fails again. + * cause. Not retryable, as the same input fails again, but for phase + * `timeout`: what the render waited for, a loader or a helper, may answer on + * another attempt. */ export declare class TemplateRenderError extends Error implements TemplateErrorFields { readonly code: 'MAILER_TEMPLATE_RENDER'; - readonly retryable: false; + /** false; true for phase `timeout`. */ + readonly retryable: boolean; /** The template that failed, as its loader names it: `welcome.uk.hbs`, a layout, a partial. */ readonly template: string; readonly phase: TemplateRenderPhase; diff --git a/src/templates/loader.js b/src/templates/loader.js index ac99685..ceccac1 100644 --- a/src/templates/loader.js +++ b/src/templates/loader.js @@ -143,7 +143,8 @@ const isNoFile = (error) => isMissing(error) || error?.code === 'EISDIR'; * Makes a loader of the templates in a directory. * * @param options `dir`, `extensions` (any by default) and `maxBytes` (1 MiB). - * @returns The loader. + * @returns The loader: `load(name, { signal })`, which stops between its + * reads when the signal aborts, and `loadSync(name)`. * @throws {TypeError} If an option is invalid. */ const createFileLoader = (options = {}) => { @@ -169,8 +170,10 @@ const createFileLoader = (options = {}) => { }; const accepts = (name) => isTemplateName(name) && hasExtension(name, extensions); return { - async load(name) { + async load(name, loadOptions) { if (!accepts(name)) return undefined; + const signal = loadOptions?.signal; + signal?.throwIfAborted(); root ??= await fsp.realpath(dir); let real; try { @@ -180,6 +183,7 @@ const createFileLoader = (options = {}) => { throw error; } checkWithin(name, real); + signal?.throwIfAborted(); let handle; try { handle = await fsp.open(real, OPEN_FLAGS); diff --git a/src/templates/renderer.js b/src/templates/renderer.js index e048da3..de9cd57 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -9,7 +9,9 @@ const { htmlToText } = require('../render/html-to-text.js'); const { decodeEntities } = require('../render/entities.js'); const { messageOf } = require('../internal/decorator.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); -const { createCache, memoize } = require('./cache.js'); +const { abortReason } = require('../internal/abort.js'); +const { timerMs } = require('../internal/sleep.js'); +const { createCache, hasTimeout, memoize } = require('./cache.js'); const { forRender, inPhase, @@ -27,6 +29,9 @@ const DEFAULT_MAX_ENTRIES = 1000; const DEFAULT_MAX_INCLUDES = 10_000; const DEFAULT_MAX_INCLUDE_DEPTH = 32; const DEFAULT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024; +// How long a render may take; what it waits for after that, a read or a +// compile, is made anew by the next render rather than waited for. +const DEFAULT_RENDER_TIMEOUT_MS = 30_000; const NO_LAYOUT = new Set(['none', 'false']); const defaultVariantName = (template, variant) => { @@ -86,6 +91,15 @@ const checkBound = (value, name, fallback) => { return value; }; +// 0 and Infinity mean no timeout, as for the transports. +const checkTimeout = (value) => { + if (value === undefined) return DEFAULT_RENDER_TIMEOUT_MS; + if (typeof value !== 'number' || Number.isNaN(value) || value < 0) { + throw new TypeError(`renderTimeoutMilliseconds must be a non-negative number: ${value}`); + } + return value; +}; + const checkLocale = (locale, what) => { if (locale === undefined) return undefined; if (typeof locale !== 'string') throw new TypeError(`${what} must be a string.`); @@ -140,6 +154,7 @@ const createRendererConfig = (options) => { DEFAULT_MAX_INCLUDE_DEPTH, ), maxOutputBytes: checkBound(options.maxOutputBytes, 'maxOutputBytes', DEFAULT_MAX_OUTPUT_BYTES), + renderTimeoutMilliseconds: checkTimeout(options.renderTimeoutMilliseconds), }; }; @@ -172,7 +187,11 @@ const checkInput = (input) => { if (typeof context !== 'object' || context === null || Array.isArray(context)) { throw new TypeError('context must be an object.'); } - return { template, context, locale: checkLocale(input.locale, 'locale') }; + const { signal } = input; + if (signal !== undefined && typeof signal?.addEventListener !== 'function') { + throw new TypeError('signal must be an AbortSignal.'); + } + return { template, context, locale: checkLocale(input.locale, 'locale'), signal }; }; const wrapError = (error, template, phase, engine) => { @@ -218,6 +237,39 @@ const normalizeSubject = (subject) => { const isParts = (value) => typeof value === 'object' && value !== null; +// A render, settled at its timeout or at the abort of its signal, whichever +// comes first: what it waits for goes on, for the renders that share it, and +// is not kept if it hangs. The timer keeps the process, as its caller waits. +const inTime = (config, checked, rendering) => { + const timeout = config.renderTimeoutMilliseconds; + const { signal } = checked; + if (!hasTimeout(timeout) && signal === undefined) return rendering; + return new Promise((resolve, reject) => { + let timer; + const onAbort = () => { + settle(); + reject(abortReason(signal)); + }; + const settle = () => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + if (hasTimeout(timeout)) { + timer = setTimeout(() => { + settle(); + reject( + new TemplateRenderError(checked.template, 'timeout', `It took over ${timeout} ms.`, { + requested: checked.template, + locale: checked.locale, + }), + ); + }, timerMs(timeout)); + } + signal?.addEventListener('abort', onAbort, { once: true }); + rendering.then(resolve, reject).finally(settle); + }); +}; + /** * Makes a renderer of templates by name. * @@ -232,12 +284,14 @@ const isParts = (value) => typeof value === 'object' && value !== null; */ const createTemplateRenderer = (options = {}) => { const config = createRendererConfig(options); - const makeIncludes = () => + const makeIncludes = (signal) => createIncludes(config.loader, { partials: config.partials, cache: config.cache, maxEntries: config.maxEntries, maxIncludeDepth: config.maxIncludeDepth, + timeoutMilliseconds: config.renderTimeoutMilliseconds, + signal, }); const compiled = createCache(config.cache ? config.maxEntries : 0); const resolved = createCache(config.cache ? config.maxEntries : 0); @@ -245,11 +299,13 @@ const createTemplateRenderer = (options = {}) => { return { async render(input) { const checked = checkInput(input); - // Without a cache, each render reads and compiles anew, includes too. - const current = config.cache ? state : { ...state, files: makeIncludes() }; + checked.signal?.throwIfAborted(); + // Without a cache, each render reads and compiles anew, includes too, + // which its signal stops. + const current = config.cache ? state : { ...state, files: makeIncludes(checked.signal) }; const bounds = { includes: 0, maxIncludes: config.maxIncludes }; try { - return await renderTemplate(current, checked, input, bounds); + return await inTime(config, checked, renderTemplate(current, checked, input, bounds)); } catch (error) { throw forRender(error, checked.template, checked.locale); } @@ -321,8 +377,11 @@ const resolveTemplate = async (state, template, locale) => { }; const resolvedFor = (state, template, locale) => - memoize(state.resolved, `${template}\u0000${locale ?? ''}`, () => - resolveTemplate(state, template, locale), + memoize( + state.resolved, + `${template}\u0000${locale ?? ''}`, + () => resolveTemplate(state, template, locale), + state.config.renderTimeoutMilliseconds, ); const compileEntry = async (state, found, part, bounds) => { @@ -367,8 +426,11 @@ const compileEntry = async (state, found, part, bounds) => { }; const compiledFor = (state, found, part, bounds) => - memoize(state.compiled, `${found.engine.name}\u0000${part}\u0000${found.name}`, () => - compileEntry(state, found, part, bounds), + memoize( + state.compiled, + `${found.engine.name}\u0000${part}\u0000${found.name}`, + () => compileEntry(state, found, part, bounds), + state.config.renderTimeoutMilliseconds, ); const renderString = async (state, phase, entry, render, scope, bounds) => { @@ -444,6 +506,8 @@ const renderTemplate = async (state, checked, input, bounds) => { } catch (error) { throw inPhase(error, 'resolve'); } + // An aborted render stops at its next step, as it rejected already. + checked.signal?.throwIfAborted(); const scope = { ...(await globalsOf(config, input)), ...checked.context }; const main = await compiledFor( state, @@ -454,7 +518,9 @@ const renderTemplate = async (state, checked, input, bounds) => { let html; let text; let subject; + checked.signal?.throwIfAborted(); const output = await run('render', main.name, main.engine, () => main.render(scope), bounds); + checked.signal?.throwIfAborted(); if (typeof output === 'string') { if (resolution.html === undefined) text = decodeEntities(output); else html = output; diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index bc8273d..9375cb0 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -100,6 +100,16 @@ const output = await templates.render({ template: 'welcome', context: { name: 'Ada' }, locale: 'uk', + tenantId: 'acme', + signal: AbortSignal.timeout(1000), +}); +expectError(templates.render({ template: 'welcome', signal: 'now' })); +createTemplateRenderer({ + engine: hbs, + loader: { + load: (name, options) => (options?.signal?.aborted ? undefined : { name, source: '' }), + }, + renderTimeoutMilliseconds: 5000, }); expectType(output); expectType(output.subject); diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index fb87e5a..31f06a6 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -87,6 +87,29 @@ test('Templates: the file loader reads no more than maxBytes of a file that grew assert.strictEqual(createFileLoader({ dir: root }).loadSync('big.ejs').source.length, 2048); }); +test('Templates: the file loader stops at the abort of its signal', async (t) => { + const { root } = makeTree(t); + const loader = createFileLoader({ dir: root }); + const reason = new Error('stop'); + await assert.rejects( + loader.load('welcome.ejs', { signal: AbortSignal.abort(reason) }), + (e) => e === reason, + ); + // Aborted after it found the file, before it opens it. + const controller = new AbortController(); + const realpath = fsp.realpath; + t.mock.method(fsp, 'realpath', async (...args) => { + const real = await realpath(...args); + if (args[0].endsWith('welcome.ejs')) controller.abort(reason); + return real; + }); + await assert.rejects( + loader.load('welcome.ejs', { signal: controller.signal }), + (e) => e === reason, + ); + assert.strictEqual((await loader.load('welcome.ejs', {})).source, '

    Hi <%= name %>

    '); +}); + test('Templates: the file loader takes a directory it cannot open for none', async (t) => { // Windows opens no directory: EISDIR, where the others open it and say it // is no file. diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index 025ea76..1b86550 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -695,6 +695,108 @@ test('Templates: no part of a message is longer than maxOutputBytes', async () = assert.strictEqual(finished.phase, 'finish'); }); +test('Templates: a render that hangs times out, and what it waited for is read anew', async () => { + // Before: no timeout, and a read or a compile that never settled was kept + // and waited for by every render after it, until clear(). + let hang = true; + const signals = new Map(); + const loader = { + load: (name, options) => { + signals.set(name, options.signal); + if (name !== 'a.tpl') return undefined; + return hang ? new Promise(() => {}) : { name, source: '

    {{n}}

    ' }; + }, + }; + const renderer = createTemplateRenderer({ + engine: fakeEngine(), + loader, + renderTimeoutMilliseconds: 50, + }); + const error = await renderer.render({ template: 'a', locale: 'uk' }).catch((e) => e); + assert.ok(error instanceof TemplateRenderError); + assert.deepStrictEqual( + [error.phase, error.retryable, error.template, error.requested, error.locale], + ['timeout', true, 'a', 'a', 'uk'], + ); + assert.match(error.message, /did not render in time: It took over 50 ms/); + // The loader is told to stop, at the timeout of its read, which may come a + // tick after that of the render. + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.ok(signals.get('a.tpl').aborted); + assert.strictEqual(signals.get('a.tpl').reason.name, 'TimeoutError'); + assert.ok(!signals.get('a.uk.tpl').aborted); + hang = false; + assert.strictEqual( + (await renderer.render({ template: 'a', context: { n: 1 } })).html, + '

    1

    ', + ); + // A compile that hangs is made anew too. + let stalls = 1; + const engine = fakeEngine(); + const compile = engine.compile; + engine.compile = (source, context) => + stalls-- > 0 ? new Promise(() => {}) : compile(source, context); + const { renderer: stalled } = rendererOf( + { 'b.tpl': '

    ' }, + { engine, renderTimeoutMilliseconds: 50 }, + ); + assert.strictEqual((await stalled.render({ template: 'b' }).catch((e) => e)).phase, 'timeout'); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.strictEqual((await stalled.render({ template: 'b' })).html, '

    '); +}); + +test('Templates: a render stops at the abort of its signal, with its reason', async () => { + // Before: render took no signal, so a worker shutting down waited for it. + const seen = []; + const loader = { + load: (name, options) => { + seen.push(options.signal); + return name === 'a.tpl' ? new Promise(() => {}) : undefined; + }, + }; + const renderer = createTemplateRenderer({ engine: fakeEngine(), loader }); + const controller = new AbortController(); + const rendering = renderer.render({ template: 'a', signal: controller.signal }); + const reason = new Error('shutting down'); + setTimeout(() => controller.abort(reason), 10); + assert.strictEqual(await rendering.catch((e) => e), reason); + // An aborted signal stops it before it reads anything. + const before = seen.length; + const aborted = AbortSignal.abort(reason); + assert.strictEqual( + await renderer.render({ template: 'a', signal: aborted }).catch((e) => e), + reason, + ); + assert.strictEqual(seen.length, before); + // Without a cache, the reads are the render's own, and its signal stops them. + const uncached = createTemplateRenderer({ engine: fakeEngine(), loader, cache: false }); + const own = new AbortController(); + const pending = uncached.render({ template: 'a', signal: own.signal }); + await new Promise((resolve) => setTimeout(resolve, 10)); + own.abort(reason); + assert.strictEqual(await pending.catch((e) => e), reason); + assert.ok(seen.at(-1).aborted); + // A render that ends first is not touched by a later abort. + const done = new AbortController(); + const { renderer: quick } = rendererOf({ 'q.tpl': '

    ' }); + assert.strictEqual((await quick.render({ template: 'q', signal: done.signal })).html, '

    '); + done.abort(); + // An abort between the steps of a render stops it there. + const later = new AbortController(); + const { renderer: steps } = rendererOf( + { 's.tpl': '

    ' }, + { globals: () => later.abort(reason) }, + ); + assert.strictEqual( + await steps.render({ template: 's', signal: later.signal }).catch((e) => e), + reason, + ); + await assert.rejects( + renderer.render({ template: 'a', signal: {} }), + /signal must be an AbortSignal/, + ); +}); + test('Templates: the renderer checks its options', () => { const engine = fakeEngine(); const loader = createMemoryLoader({}); @@ -726,12 +828,16 @@ test('Templates: the renderer checks its options', () => { [{ engine, loader, maxIncludes: 0 }, /maxIncludes must be a positive whole number/], [{ engine, loader, maxIncludeDepth: 1.5 }, /maxIncludeDepth must be/], [{ engine, loader, maxOutputBytes: '1' }, /maxOutputBytes must be/], + [{ engine, loader, renderTimeoutMilliseconds: -1 }, /renderTimeoutMilliseconds must be/], + [{ engine, loader, renderTimeoutMilliseconds: Number.NaN }, /renderTimeoutMilliseconds/], ]; for (const [options, pattern] of bad) { assert.throws(() => createTemplateRenderer(options), pattern, String(pattern)); } assert.throws(() => createTemplateRenderer(), /needs an engine/); createTemplateRenderer({ engine, loader, maxIncludes: Infinity, maxOutputBytes: Infinity }); + createTemplateRenderer({ engine, loader, renderTimeoutMilliseconds: 0 }); + createTemplateRenderer({ engine, loader, renderTimeoutMilliseconds: Infinity }); assert.throws(() => createCssInliner({}), /needs inline/); assert.throws(() => createCssInliner({ inline() {}, options: 1 }), /must be an object/); }); From f4feae63787dd8cdc29265a9a725714b6ad5dbf3 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:55:55 +0300 Subject: [PATCH 39/83] feat(queue): tenants in turn, and a pause of one tenant's jobs Jobs went by priority, then time, so one tenant's backlog of newsletters held every other tenant's normal mail, and a rate limit could pause only the whole queue, every tenant with it, until its time. A job request carries its tenantId and a job gives it back. fairness: 'tenant', on the memory queue and the outbox, takes a high job of any tenant first, then the tenants with jobs ready in turn (the memory queue a heap per tenant; the outbox the tenants read with distinct once a poll interval, a new index mailer_tenant). rateLimitScope: 'tenant' pauses the jobs of the job's tenant alone (a pause document per tenant, read with the queue's in one range of _id), and resume() of either ends a pause early. The contract of a source holds the new cases, and runs whole under fairness on memory, the fake collection and a replica set. BullMQ shares a queue by tenant only in its Pro edition: the docs give a queue per tenant or stream. ./queue goes from 17.6 to 18.3 KB min+gzip and ./mongodb from 5.6 to 6.6: their budgets go from 18 to 18.5 and from 6 to 7 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + docs/guide/outbox.md | 9 +- docs/guide/queue.md | 17 ++ docs/transports/bullmq.md | 10 ++ index.d.ts | 11 +- scripts/size-budgets.json | 4 +- src/internal/polling-driver.js | 8 + src/mongodb/index.d.ts | 8 + src/mongodb/outbox.js | 191 ++++++++++++++++++----- src/queue/handler.js | 6 +- src/queue/index.d.ts | 15 +- src/queue/job.js | 1 + src/queue/memory-queue.js | 142 +++++++++++++++-- src/queue/outcome.js | 15 +- tests/helpers/mongodb/fake-collection.js | 30 +++- tests/helpers/queue/source-contract.js | 95 ++++++++++- tests/mongodb/outbox.mongodb.test.js | 23 ++- tests/types/mongodb.test-d.ts | 2 + tests/types/queue.test-d.ts | 4 +- tests/unit/mongodb.test.js | 8 +- tests/unit/queue-handler.test.js | 12 ++ tests/unit/queue-memory.test.js | 30 ++++ tests/unit/queue-source-contract.test.js | 22 ++- 23 files changed, 582 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d855a29..db31cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. An idle poll of the outbox is two round trips (leases taken back once a third of a lease, the pause read once a poll interval, the next due time from its own index); its change stream wakes one lane per change after which a job may be ready, and is opened again when it fails. + `fairness: 'tenant'` (memory queue, outbox) takes a high job of any tenant first, then the + tenants in turn, so one tenant's backlog holds no other's mail; `rateLimitScope: 'tenant'` pauses + one tenant's jobs alone, and `resume()` ends a pause early. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 9bca7d5..2092e7b 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -71,7 +71,12 @@ The outbox is a `QueueSource`, read by the polling driver of `./queue`: - a job whose lease ran out (its worker died) is ready again, its attempt run again, `redelivered`; - a rate limit of the whole queue (`rateLimitScope: 'queue'`) pauses every - worker, through a document of its own. + worker, and one of a tenant (`'tenant'`) that tenant's jobs alone, through + documents of their own (`mailer:pause`, `mailer:pause:`), which + `outbox.resume()` ends early; +- with `fairness: 'tenant'`, a lane takes a `high` job of any tenant first, + then the tenants with jobs ready in turn (read with `distinct` once a poll + interval): two round trips a job. Leases are written by the clock of each worker, so a lease is taken back only `clockSkewToleranceMilliseconds` (5 s) after it ran out, lest a worker whose @@ -94,7 +99,7 @@ that fails is opened again a second later, then up to every 30 seconds | Collection | Index | | --- | --- | -| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; `{ state, availableAt }` for the next one due; a TTL on `expireAt` | +| outbox | `{ state, rank, availableAt, enqueuedAt }` for the ready jobs; `{ state, leaseUntil }` for the leases; `{ state, availableAt }` for the next one due; `{ state, tenantId, rank, availableAt, enqueuedAt }` for the tenants in turn; a TTL on `expireAt` | | audit | `{ correlationId, at, attempt, rank }`; `{ jobId }` | | idempotency | a TTL on `expiresAt` | diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 1522137..652df87 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -209,6 +209,23 @@ over it, so that a rate limit does not bring every job back at once. driver can. `classify({ receipt, job })` decides in the place of these rules (`'retry'`, `'fail'`, `'delivery_unknown'`, or `undefined` for the rules), and `checkJob(data, job)` refuses a job for good by throwing. +`rateLimitScope: 'tenant'` makes a rate limit pause the jobs of the job's +tenant alone, on the memory queue and the outbox; `resume()` of either ends a +pause, of the queue or of a tenant (`{ tenantId }`), before its time. A +`'queue'` pause suits one provider; with failover among several, let +[`./breaker`](/transports/breaker) and [`./ratelimit`](/transports/ratelimit) +take a provider out instead. + +## Tenants in turn + +Jobs go by priority, then time: a tenant who enqueues a million newsletters +holds every other tenant's `normal` mail behind them. `fairness: 'tenant'`, on +the memory queue and the outbox, takes a `high` job of any tenant first (a code +of sign-in waits for no one), then the tenants in turn, a job of each, the jobs +of no tenant as one more. It is exact within a process and close across them. +Without it, a queue per stream (transactional and bulk, each with its own +workers) keeps a campaign from holding a password reset. BullMQ shares a queue +by tenant only in its Pro edition: use a queue per tenant or per stream there. **The queue retries; the transport should not.** Each attempt of a `RetryTransport` inside the worker holds the job and multiplies its attempts: diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index 46f9cb5..51a14fe 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -39,6 +39,16 @@ it holds only while the first job is not finished. A completed job keeps only `{ v, hash }` of its data (`job.updateData`, by `retainData: 'failed'`, the default; `'all'` or `'none'`), so its message does not wait a day in Redis. +A rate limit of a tenant's scope (`rateLimitScope: 'tenant'`) is a retry of the +job on BullMQ, which pauses a queue only whole; and tenants share a queue fairly +only in BullMQ Pro (groups): give each tenant, or each stream (transactional and +bulk), a queue and workers of its own: + +```js +const transactional = createMailQueue({ driver: createBullMQDriver({ bullmq, queueName: 'mail-transactional' }) }); +const bulk = createMailQueue({ driver: createBullMQDriver({ bullmq, queueName: 'mail-bulk' }) }); +``` + A job BullMQ fails itself, which the handler never settled (stalled more than `maxStalledCount`), is logged as `queue.job.lost` and told to the queue's audit sink as `delivery_unknown` (code `queue.lost`): its run may have sent it before diff --git a/index.d.ts b/index.d.ts index db64739..44b0491 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1496,6 +1496,8 @@ export interface QueueJobRequest { readonly enqueuedAt: number; /** Its id comes from a key of the caller's: an enqueue again is expected, and told. */ readonly unique: boolean; + /** The tenant of the job: a source may share its lanes among tenants, and pause one. */ + readonly tenantId?: string; } /** The options of an enqueue. */ export interface QueueEnqueueOptions { @@ -1529,6 +1531,8 @@ export interface QueueJob { * not count them. */ readonly redeliveries?: number; + /** The tenant of the job, from its request, where the source keeps it. */ + readonly tenantId?: string; } /** What a handler makes of a run of a job. */ export type QueueOutcome = @@ -1539,7 +1543,8 @@ export type QueueOutcome = readonly delayMilliseconds: number; readonly code: string; readonly reason: string; - readonly scope?: 'job' | 'queue'; + /** `queue`: the whole queue waits; `tenant`: the jobs of the job's tenant wait. */ + readonly scope?: 'job' | 'queue' | 'tenant'; } /** No attempt again: the job is failed for good. */ | { readonly type: 'fail'; readonly code: string; readonly reason: string } @@ -1624,7 +1629,9 @@ export interface QueueSource { /** Back to ready at `availableAt` (now by default), its attempt not counted. */ release(lease: QueueLease, availableAt?: number): Promise; /** The whole queue waits until then; without it, a retry of the queue is one of the job. */ - pause?(until: number): Promise; + pause?(until: number, scope?: { readonly tenantId?: string }): Promise; + /** Ends a pause of the whole queue, or of one tenant's jobs, before its time. */ + resume?(scope?: { readonly tenantId?: string }): Promise; /** Wakes the readers when a job may be ready: an enqueue, a change stream. */ subscribe?(listener: () => void): () => void; } diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 3b30f76..090eb59 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 18, + "./queue": 18.5, "./bullmq": 4.5, - "./mongodb": 6 + "./mongodb": 7 } diff --git a/src/internal/polling-driver.js b/src/internal/polling-driver.js index 426bcd9..4daaf90 100644 --- a/src/internal/polling-driver.js +++ b/src/internal/polling-driver.js @@ -100,6 +100,14 @@ const settle = async (source, lease, outcome, config) => { if (outcome.scope === 'queue' && typeof source.pause === 'function') { await source.pause(now + outcome.delayMilliseconds); } + // The tenant's jobs wait, where the source keeps whose a job is. + if ( + outcome.scope === 'tenant' && + job.tenantId !== undefined && + typeof source.pause === 'function' + ) { + await source.pause(now + outcome.delayMilliseconds, { tenantId: job.tenantId }); + } return source.retry(lease, now + outcome.delayMilliseconds, failure); } case 'fail': diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index 02d2118..0cd5f6c 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -64,6 +64,12 @@ export interface MongoOutboxOptions extends PollingDriverOptions { readonly retainFailedSeconds?: number; /** The finished jobs whose data is kept: the others keep their version and hash. @default 'failed' */ readonly retainData?: 'all' | 'failed' | 'none'; + /** + * `tenant`: high jobs of any tenant first, then the tenants' jobs in turn, + * so that one tenant's backlog holds no other's mail; needs `distinct()`. + * @default 'none' + */ + readonly fairness?: 'none' | 'tenant'; readonly logger?: Logger; } @@ -89,6 +95,8 @@ export interface MongoOutbox extends QueueDriver { /** Makes the indexes of the outbox: the ready jobs, the leases, the next one due, and the expiry of finished ones (`expireAt`). */ ensureIndexes(): Promise; get(id: string): Promise; + /** Ends a pause of the queue, or of one tenant's jobs, before its time. */ + resume(scope?: { readonly tenantId?: string }): Promise; } /** diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 75a7518..32d9f64 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -19,8 +19,15 @@ const { const { isJobRequest, rankOf } = require('../internal/priority.js'); const { checkCollection, documentOf, sessionOf } = require('./collection.js'); -// The document of a pause of the whole queue: no job id has a colon. +// The documents of the pauses, of the whole queue and of one tenant's jobs: +// their ids in one range of _id, which no job id, with no colon, falls in. const PAUSE_ID = 'mailer:pause'; +const pauseIdOf = (tenantId) => (tenantId === undefined ? PAUSE_ID : `${PAUSE_ID}:${tenantId}`); +const PAUSES = { _id: { $gte: PAUSE_ID, $lt: `${PAUSE_ID};` } }; +// The order of the ready jobs, and of those of one rank. +const READY_ORDER = { rank: 1, availableAt: 1, enqueuedAt: 1 }; +const RANK_ORDER = { availableAt: 1, enqueuedAt: 1 }; +const FAIRNESS = new Set(['none', 'tenant']); // How far the clocks of two workers may be apart: a lease is taken back only // that long after it ran out by the clock of the worker that takes it. const DEFAULT_CLOCK_SKEW_MS = 5000; @@ -46,12 +53,14 @@ const documentOfRequest = (request, now) => ({ redeliveries: 0, token: null, leaseUntil: null, + // Null for none, so that an index and a filter of tenants take it as one. + tenantId: request.tenantId ?? null, updatedAt: new Date(now), }); const jobOf = (document) => { const { _id: id } = document; - return { + const job = { id, name: document.name, data: document.data, @@ -61,6 +70,8 @@ const jobOf = (document) => { redelivered: document.redelivered === true, redeliveries: document.redeliveries ?? 0, }; + if (typeof document.tenantId === 'string') job.tenantId = document.tenantId; + return job; }; // The filter of a write under a lease: the job is still lent to it. @@ -78,7 +89,9 @@ class MongoOutboxSource { // When this process last took back the leases that ran out, and read the // pause of the queue: once a while, not on each reserve of each lane. #reclaimedAt = -Infinity; - #pause = { readAt: -Infinity, until: 0 }; + #pauses = { readAt: -Infinity, until: 0, tenants: new Map() }; + // The tenants with jobs ready, read once a poll interval, taken in turn. + #tenants = { readAt: -Infinity, list: [], cursor: 0, found: false }; constructor(collection, config) { this.#collection = collection; @@ -119,18 +132,36 @@ class MongoOutboxSource { return answer; } - // Until when the queue is paused: read again once a poll interval, a pause - // of another process seen within it, one of this process at once. - async #pausedUntil(now) { - if (now - this.#pause.readAt >= this.#config.pauseReadMilliseconds) { - const pause = await this.#collection.findOne( - { _id: PAUSE_ID }, - { projection: { pausedUntil: 1 } }, - ); - const until = pause?.pausedUntil; - this.#pause = { readAt: now, until: until instanceof Date ? until.getTime() : 0 }; + // The pauses, of the queue and of tenants: read again once a poll + // interval, a pause of another process seen within it, one of this process + // at once. + async #readPauses(now) { + if (now - this.#pauses.readAt < this.#config.pauseReadMilliseconds) return this.#pauses; + const documents = await this.#collection + // oxlint-disable-next-line no-restricted-properties -- a MongoDB collection, not an array + .find(PAUSES, { projection: { pausedUntil: 1, tenantId: 1 } }) + .toArray(); + let until = 0; + const tenants = new Map(); + for (const document of documents) { + const at = document.pausedUntil instanceof Date ? document.pausedUntil.getTime() : 0; + if (typeof document.tenantId === 'string') tenants.set(document.tenantId, at); + else until = at; + } + this.#pauses = { readAt: now, until, tenants }; + return this.#pauses; + } + + // The tenants whose jobs wait now, and when the first of them resumes. + #pausedTenants(now) { + const paused = []; + let resumesAt; + for (const [tenant, until] of this.#pauses.tenants) { + if (until <= now) continue; + paused.push(tenant); + if (resumesAt === undefined || until < resumesAt) resumesAt = until; } - return this.#pause.until; + return { paused, resumesAt }; } // A job whose lease ran out is ready again, its attempt to run again: by @@ -151,15 +182,12 @@ class MongoOutboxSource { ); } - async reserve(leaseMilliseconds) { - const now = new Date(this.#config.now()); - const pausedUntil = await this.#pausedUntil(now.getTime()); - if (pausedUntil > now.getTime()) return { nextAvailableAt: pausedUntil }; - await this.#reclaim(now.getTime(), leaseMilliseconds); + // Lends the first job a filter meets, in an order. + async #take(filter, sort, now, leaseMilliseconds) { const token = randomUUID(); const taken = documentOf( await this.#collection.findOneAndUpdate( - { state: 'ready', availableAt: { $lte: now } }, + filter, { $set: { state: 'active', @@ -171,21 +199,75 @@ class MongoOutboxSource { $inc: { attempts: 1 }, }, // As it was: whether it comes back, and its attempt before this one. - { - sort: { rank: 1, availableAt: 1, enqueuedAt: 1 }, - returnDocument: 'before', - includeResultMetadata: false, - }, + { sort, returnDocument: 'before', includeResultMetadata: false }, ), ); - if (taken !== null) { - return { lease: { job: { ...jobOf(taken), attempt: taken.attempts + 1 }, token } }; + if (taken === null) return undefined; + return { lease: { job: { ...jobOf(taken), attempt: taken.attempts + 1 }, token } }; + } + + // The tenants with jobs ready: read again once a poll interval, or when + // those read are done with and some were found then. + async #tenantsReady(ready, now, again) { + const tenants = this.#tenants; + const stale = now - tenants.readAt >= this.#config.pauseReadMilliseconds; + if (stale || (again && tenants.found)) { + const list = await this.#collection.distinct('tenantId', ready); + this.#tenants = { readAt: now, list, cursor: 0, found: list.length > 0 }; } - const next = await this.#collection.findOne( - { state: 'ready' }, - { sort: { availableAt: 1 }, projection: { availableAt: 1 } }, - ); - return next?.availableAt instanceof Date ? { nextAvailableAt: next.availableAt.getTime() } : {}; + return this.#tenants; + } + + // A job by tenant in turn: a high one of any tenant first, a code of + // sign-in waiting for no one's newsletter, then the tenants one after the + // other, so that the backlog of one holds no other's mail. + async #takeFairly(ready, paused, now, leaseMilliseconds) { + const high = await this.#take({ ...ready, rank: 0 }, RANK_ORDER, now, leaseMilliseconds); + if (high !== undefined) return high; + for (let round = 0; round < 2; round++) { + const tenants = await this.#tenantsReady(ready, now.getTime(), round > 0); + while (tenants.list.length > 0) { + const index = tenants.cursor % tenants.list.length; + const tenant = tenants.list[index]; + // A tenant paused since its list was read waits. + if (!paused.has(tenant)) { + const filter = { state: 'ready', availableAt: ready.availableAt, tenantId: tenant }; + const taken = await this.#take(filter, READY_ORDER, now, leaseMilliseconds); + if (taken !== undefined) { + tenants.cursor = index + 1; + return taken; + } + } + // Nothing of this tenant is ready now. + tenants.list.splice(index, 1); + tenants.cursor = index; + } + } + return undefined; + } + + async reserve(leaseMilliseconds) { + const now = new Date(this.#config.now()); + const pauses = await this.#readPauses(now.getTime()); + if (pauses.until > now.getTime()) return { nextAvailableAt: pauses.until }; + await this.#reclaim(now.getTime(), leaseMilliseconds); + const { paused, resumesAt } = this.#pausedTenants(now.getTime()); + const ready = { state: 'ready', availableAt: { $lte: now } }; + if (paused.length > 0) ready.tenantId = { $nin: paused }; + const taken = + this.#config.fairness === 'tenant' + ? await this.#takeFairly(ready, new Set(paused), now, leaseMilliseconds) + : await this.#take(ready, READY_ORDER, now, leaseMilliseconds); + if (taken !== undefined) return taken; + const waiting = { state: 'ready' }; + if (paused.length > 0) waiting.tenantId = { $nin: paused }; + const next = await this.#collection.findOne(waiting, { + sort: { availableAt: 1 }, + projection: { availableAt: 1 }, + }); + let due = next?.availableAt instanceof Date ? next.availableAt.getTime() : undefined; + if (resumesAt !== undefined && (due === undefined || resumesAt < due)) due = resumesAt; + return due === undefined ? {} : { nextAvailableAt: due }; } async extend(lease, leaseMilliseconds) { @@ -221,13 +303,26 @@ class MongoOutboxSource { return this.#settle(lease, { state: 'ready', availableAt: at }, { attempts: -1 }); } - async pause(until) { - await this.#collection.updateOne( - { _id: PAUSE_ID }, - { $max: { pausedUntil: new Date(until) } }, - { upsert: true }, - ); - this.#pause.until = Math.max(this.#pause.until, until); + async pause(until, scope = {}) { + const { tenantId } = scope; + const update = { $max: { pausedUntil: new Date(until) } }; + if (tenantId !== undefined) update.$set = { tenantId }; + await this.#collection.updateOne({ _id: pauseIdOf(tenantId) }, update, { upsert: true }); + if (tenantId === undefined) { + this.#pauses.until = Math.max(this.#pauses.until, until); + } else { + const held = this.#pauses.tenants.get(tenantId) ?? 0; + this.#pauses.tenants.set(tenantId, Math.max(held, until)); + } + } + + async resume(scope = {}) { + const { tenantId } = scope; + await this.#collection.deleteOne({ _id: pauseIdOf(tenantId) }); + if (tenantId === undefined) this.#pauses.until = 0; + else this.#pauses.tenants.delete(tenantId); + // Its jobs are ready again: the tenants are read again. + this.#tenants.readAt = -Infinity; } } @@ -308,7 +403,8 @@ const subscribe = (collection, listener, config) => { * * @param options `collection`, `db.collection('mail_outbox')`; * `changeStream` (false), to wake the workers on a change (a replica set); - * `retainData` ('failed'), the finished jobs whose data is kept; + * `retainData` ('failed'), the finished jobs whose data is kept; `fairness` + * ('none'), or 'tenant' for the tenants' jobs in turn; * `dedupeWindowSeconds` (a day), the time a completed job is kept, and so * the window of its duplicates; `retainFailedSeconds` (a week), that of a * failed one; `clockSkewToleranceMilliseconds` (5000), how @@ -318,7 +414,12 @@ const subscribe = (collection, listener, config) => { * @throws {TypeError} If an option is invalid. */ const createMongoOutbox = (options = {}) => { + const fairness = options.fairness ?? 'none'; + if (!FAIRNESS.has(fairness)) throw new TypeError("fairness must be 'none' or 'tenant'."); const collection = checkCollection(options.collection, 'createMongoOutbox'); + if (fairness === 'tenant' && typeof collection.distinct !== 'function') { + throw new TypeError("fairness 'tenant' needs a collection with distinct()."); + } const changeStream = options.changeStream ?? false; if (typeof changeStream !== 'boolean') throw new TypeError('changeStream must be a boolean.'); if (changeStream && typeof collection.watch !== 'function') { @@ -339,6 +440,7 @@ const createMongoOutbox = (options = {}) => { sleep: checkOptionalFunction(options.sleep, 'sleep') ?? sleep, // The polling driver checks it; the pause is read again as often. pauseReadMilliseconds: options.pollIntervalMilliseconds ?? 1000, + fairness, // The lease of the workers: none of theirs runs out sooner. leaseMilliseconds: options.leaseMilliseconds ?? DEFAULT_LEASE_MS, retainData: checkRetainData(options.retainData), @@ -354,6 +456,7 @@ const createMongoOutbox = (options = {}) => { // An enqueue writes in the transaction it is given. transactional: true, source, + resume: (scope) => source.resume(scope), async ensureIndexes() { await collection.createIndex( { state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, @@ -363,6 +466,11 @@ const createMongoOutbox = (options = {}) => { // When the next job is due, read when none is: sorted by time alone, // which the index of the ready jobs, by priority first, cannot give. await collection.createIndex({ state: 1, availableAt: 1 }, { name: 'mailer_due' }); + // The tenants with jobs ready, and the jobs of one, for fairness. + await collection.createIndex( + { state: 1, tenantId: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, + { name: 'mailer_tenant' }, + ); // A finished job goes at its time, a completed one and with it the // memory of its key after the window of duplicates, a failed one later. await collection.createIndex( @@ -373,7 +481,8 @@ const createMongoOutbox = (options = {}) => { async get(id) { const document = await collection.findOne({ _id: id }); // The pause of the queue is no job. - if (document === null || document === undefined || id === PAUSE_ID) return undefined; + // The pauses of the queue are no jobs. + if (document === null || document === undefined || id.startsWith(PAUSE_ID)) return undefined; const job = jobOf(document); return { ...job, diff --git a/src/queue/handler.js b/src/queue/handler.js index a42edb2..02b214c 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -67,8 +67,8 @@ const createHandlerConfig = (options = {}) => { throw new TypeError('The audit sink of a mail worker must have append().'); } const rateLimitScope = options.rateLimitScope ?? 'job'; - if (rateLimitScope !== 'job' && rateLimitScope !== 'queue') { - throw new TypeError("rateLimitScope must be 'job' or 'queue'."); + if (rateLimitScope !== 'job' && rateLimitScope !== 'queue' && rateLimitScope !== 'tenant') { + throw new TypeError("rateLimitScope must be 'job', 'queue' or 'tenant'."); } if (options.auditStrict !== undefined && typeof options.auditStrict !== 'boolean') { throw new TypeError('auditStrict must be a boolean.'); @@ -202,7 +202,7 @@ const release = (run, why) => { const settleFailure = async (run, receipt, code) => { const { config, job, data } = run; const expiresAt = data.expiresAt === undefined ? undefined : Date.parse(data.expiresAt); - const decided = outcomeOfFailure(receipt, job, { ...config, expiresAt }); + const decided = outcomeOfFailure(receipt, job, { ...config, expiresAt, tenantId: data.tenantId }); let { outcome } = decided; const generic = outcome.code === 'queue.attempts_exhausted' || outcome.code === 'queue.expired'; if (code !== undefined && !generic) outcome = { ...outcome, code }; diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index d4c7395..65d96dd 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -339,8 +339,12 @@ export interface MailWorkerOptions { readonly backoff?: MailBackoff; /** The wait before an attempt again after an `auth` or `configuration` failure. @default 300000 */ readonly operatorDelayMilliseconds?: number; - /** `queue`: a rate limit makes the whole queue wait, where the driver can. @default 'job' */ - readonly rateLimitScope?: 'job' | 'queue'; + /** + * What a rate limit makes wait, where the driver can: the job, the whole + * queue, or the jobs of the job's tenant (the memory queue and the outbox). + * @default 'job' + */ + readonly rateLimitScope?: 'job' | 'queue' | 'tenant'; /** * The runs of a job that may end unsettled (its worker killed, its event * loop blocked) before the job is given up as `delivery_unknown` @@ -494,6 +498,11 @@ export interface MemoryQueueOptions extends PollingDriverOptions { readonly retainFailedSeconds?: number; /** See {@link MailQueueRetainData}. @default 'failed' */ readonly retainData?: MailQueueRetainData; + /** + * `tenant`: high jobs of any tenant first, then the tenants' jobs in turn, + * so that one tenant's backlog holds no other's mail. @default 'none' + */ + readonly fairness?: 'none' | 'tenant'; } /** @@ -513,6 +522,8 @@ export interface MemoryQueue extends QueueDriver { drain(): Promise; /** Wakes its readers, after a clock of the tests moved. */ wake(): void; + /** Ends a pause of the queue, or of one tenant's jobs, before its time. */ + resume(scope?: { readonly tenantId?: string }): Promise; } /** diff --git a/src/queue/job.js b/src/queue/job.js index aa2d46c..e463c35 100644 --- a/src/queue/job.js +++ b/src/queue/job.js @@ -338,6 +338,7 @@ const prepareMailJob = (input, options = {}) => { unique: keyed !== undefined, }; if (scheduledAt !== undefined) request.availableAt = scheduledAt.getTime(); + if (tenantId !== undefined) request.tenantId = tenantId; return { data, request }; }; diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index 53792dc..e97d926 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -61,7 +61,13 @@ class MemorySource { // The jobs waiting, by when they are due; those due, by the order they run // in; and the leases, by when they run out. #delayed = new Heap(dueBefore); - #ready = new Heap(readyBefore); + // The jobs due: the high ones of every tenant on one heap, the others on a + // heap per tenant under fairness, or one for all, the tenants in turn. + #high = new Heap(readyBefore); + #buckets = new Map(); + // The jobs due of tenants whose jobs wait, set aside until their pause ends. + #parked = new Map(); + #tenantPauses = new Map(); #leases = new Heap(leaseBefore); #open = 0; #active = 0; @@ -174,18 +180,96 @@ class MemorySource { const current = isCurrent(node, 'waiting'); if (current && node.availableAt > now) return; this.#delayed.pop(); - if (current) this.#ready.push(node); + if (current) this.#ready(node); } } - #nextReady() { + // A job due, onto the heap it is taken from. + #ready(node) { + if (node.rank === 0) { + this.#high.push(node); + return; + } + const key = this.#config.fairness === 'tenant' ? (node.entry.tenantId ?? '') : ''; + let heap = this.#buckets.get(key); + if (heap === undefined) { + heap = new Heap(readyBefore); + this.#buckets.set(key, heap); + } + heap.push(node); + } + + #isTenantPaused(entry, now) { + return entry.tenantId !== undefined && (this.#tenantPauses.get(entry.tenantId) ?? 0) > now; + } + + // The first job of a heap that may run now: stale nodes dropped, those of a + // paused tenant set aside. + #firstOf(heap, now) { for (;;) { - const node = this.#ready.peek(); - if (node === undefined || isCurrent(node, 'waiting')) return node; - this.#ready.pop(); + const node = heap.peek(); + if (node === undefined) return undefined; + if (!isCurrent(node, 'waiting')) { + heap.pop(); + } else if (this.#isTenantPaused(node.entry, now)) { + heap.pop(); + const parked = this.#parked.get(node.entry.tenantId) ?? []; + parked.push(node); + this.#parked.set(node.entry.tenantId, parked); + } else { + return node; + } + } + } + + // The tenants whose pause ended give their jobs back. + #unpark(now) { + for (const [tenant, until] of this.#tenantPauses) { + if (until > now) continue; + this.#tenantPauses.delete(tenant); + this.#releaseParked(tenant); + } + } + + #releaseParked(tenant) { + const parked = this.#parked.get(tenant); + if (parked === undefined) return; + this.#parked.delete(tenant); + for (const node of parked) { + if (isCurrent(node, 'waiting')) this.#ready(node); } } + // The next job to run, and the heap it is on: a high one first, then the + // tenants in turn, the one taken from going last when `take`. + #nextReady(now, take) { + const high = this.#firstOf(this.#high, now); + if (high !== undefined) return { node: high, heap: this.#high }; + for (const [key, heap] of this.#buckets) { + const node = this.#firstOf(heap, now); + if (node === undefined) { + this.#buckets.delete(key); + continue; + } + if (take) { + this.#buckets.delete(key); + this.#buckets.set(key, heap); + } + return { node, heap }; + } + return undefined; + } + + // When the first paused tenant resumes, if its jobs wait. + #nextResume() { + let next; + for (const tenant of this.#parked.keys()) { + const until = this.#tenantPauses.get(tenant); + if (until !== undefined && (next === undefined || until < next)) next = until; + } + return next; + } + #nextDue() { for (;;) { const node = this.#delayed.peek(); @@ -221,6 +305,7 @@ class MemorySource { token: undefined, leaseUntil: 0, version: 0, + tenantId: request.tenantId, }; this.jobs.set(entry.id, entry); this.#open += 1; @@ -234,14 +319,17 @@ class MemorySource { this.#sweep(now); this.#expire(now); if (this.#pausedUntil > now) return { nextAvailableAt: this.#pausedUntil }; + this.#unpark(now); this.#promote(now); - const node = this.#nextReady(); - if (node === undefined) { - const due = this.#nextDue(); + const found = this.#nextReady(now, true); + if (found === undefined) { + let due = this.#nextDue(); + const resumes = this.#nextResume(); + if (resumes !== undefined && (due === undefined || resumes < due)) due = resumes; return due === undefined ? {} : { nextAvailableAt: due }; } - this.#ready.pop(); - const next = node.entry; + found.heap.pop(); + const next = found.node.entry; next.state = 'active'; next.attempt += 1; next.token = randomUUID(); @@ -259,6 +347,7 @@ class MemorySource { redelivered: next.redelivered, redeliveries: next.redeliveries, }; + if (next.tenantId !== undefined) job.tenantId = next.tenantId; next.redelivered = false; this.#notify(); return { lease: { job, token: next.token } }; @@ -313,8 +402,24 @@ class MemorySource { return true; } - async pause(until) { - this.#pausedUntil = Math.max(this.#pausedUntil, until); + async pause(until, scope = {}) { + const { tenantId } = scope; + if (tenantId === undefined) { + this.#pausedUntil = Math.max(this.#pausedUntil, until); + return; + } + this.#tenantPauses.set(tenantId, Math.max(this.#tenantPauses.get(tenantId) ?? 0, until)); + } + + async resume(scope = {}) { + const { tenantId } = scope; + if (tenantId === undefined) { + this.#pausedUntil = 0; + } else { + this.#tenantPauses.delete(tenantId); + this.#releaseParked(tenantId); + } + this.#notify(); } subscribe(listener) { @@ -325,8 +430,9 @@ class MemorySource { #busy(now) { if (this.#active > 0) return true; if (this.#pausedUntil > now) return false; + this.#unpark(now); this.#promote(now); - return this.#nextReady() !== undefined; + return this.#nextReady(now, false) !== undefined; } // Settles when no job is ready now and none runs. @@ -370,7 +476,8 @@ const viewOf = (entry, now) => { * @param options `now`, `sleep`, `leaseMilliseconds` (60000), * `pollIntervalMilliseconds` (1000), `maxJobs` (100000), `retainFinished` * (10000), `dedupeWindowSeconds` (a day), `retainFailedSeconds` (a week), - * `retainData` ('failed'), `logger`. + * `retainData` ('failed'), `fairness` ('none', or 'tenant' for the tenants' + * jobs in turn), `logger`. * @returns The driver, with `source`, `list()`, `get(id)`, `drain()` and * `wake()`. * @throws {TypeError} If an option is invalid. @@ -381,8 +488,12 @@ const createMemoryQueue = (options = {}) => { maxJobs: checkPositiveInteger(options.maxJobs ?? 100_000, 'maxJobs'), retainFinished: checkPositiveInteger(options.retainFinished ?? 10_000, 'retainFinished'), retainData: checkRetainData(options.retainData), + fairness: options.fairness ?? 'none', ...retentionOf(options), }; + if (config.fairness !== 'none' && config.fairness !== 'tenant') { + throw new TypeError("fairness must be 'none' or 'tenant'."); + } const source = new MemorySource(config); const driver = createPollingDriver(source, { ...options, now: config.now }); return { @@ -401,6 +512,7 @@ const createMemoryQueue = (options = {}) => { return entry === undefined ? undefined : viewOf(entry, config.now()); }, drain: () => source.drain(), + resume: (scope) => source.resume(scope), wake: () => source.wake(), }; }; diff --git a/src/queue/outcome.js b/src/queue/outcome.js index 92203e5..15c1714 100644 --- a/src/queue/outcome.js +++ b/src/queue/outcome.js @@ -70,7 +70,7 @@ const retryDelayOf = (attempt, receipt, config) => { * @param receipt The failed receipt. * @param job The run of the job. * @param config `backoff`, `random`, `now`, `operatorDelayMilliseconds`, - * `rateLimitScope`, `classify`, and `expiresAt` of the job, if any. + * `rateLimitScope`, `classify`, and `expiresAt` and `tenantId` of the job, if any. * @returns `{ outcome, state, error }`. * @internal */ @@ -102,8 +102,17 @@ const outcomeOfFailure = (receipt, job, config) => { error, }; } - const scope = - error.category === 'rate-limit' && config.rateLimitScope === 'queue' ? 'queue' : 'job'; + // A rate limit makes the queue, or the job's tenant, wait where asked; a + // job of no tenant waits alone. + let scope = 'job'; + if (error.category === 'rate-limit' && config.rateLimitScope === 'queue') scope = 'queue'; + if ( + error.category === 'rate-limit' && + config.rateLimitScope === 'tenant' && + config.tenantId !== undefined + ) { + scope = 'tenant'; + } return { outcome: { type: 'retry', delayMilliseconds: delay, code, reason, scope }, state: 'queued', diff --git a/tests/helpers/mongodb/fake-collection.js b/tests/helpers/mongodb/fake-collection.js index a0232c6..f62ff83 100644 --- a/tests/helpers/mongodb/fake-collection.js +++ b/tests/helpers/mongodb/fake-collection.js @@ -1,5 +1,6 @@ // Not ported: a collection of MongoDB in memory for the tests of ./mongodb, -// with what the stores call of it: filters of equality, `$lte` and `$or`; +// with what the stores call of it: filters of equality, `$lte`, `$lt`, `$gt`, +// `$gte`, `$in`, `$nin` and `$or`; `distinct`; // `$set`, `$setOnInsert`, `$unset`, `$inc` and `$max`; an upsert that meets an id held // fails as MongoDB does (E11000); sorts; and a change stream. @@ -37,6 +38,23 @@ const matches = (document, filter) => { if (!condition.$in.some((candidate) => equal(value, candidate))) return false; continue; } + if ('$nin' in condition) { + if (condition.$nin.some((candidate) => equal(value, candidate))) return false; + continue; + } + const bounds = [ + ['$gt', (a, b) => a > b], + ['$gte', (a, b) => a >= b], + ['$lt', (a, b) => a < b], + ]; + let bounded = false; + for (const [operator, holds] of bounds) { + if (!(operator in condition)) continue; + bounded = true; + if (value === null || value === undefined) return false; + if (!holds(comparable(value), comparable(condition[operator]))) return false; + } + if (bounded) continue; } if (!equal(value, condition)) return false; } @@ -201,6 +219,16 @@ const fakeCollection = (options = {}) => { }; return cursor; }, + async distinct(field, filter = {}) { + calls.push(['distinct', field, filter]); + const values = []; + for (const document of documents.values()) { + if (!matches(document, filter)) continue; + const value = document[field] ?? null; + if (!values.some((held) => equal(held, value))) values.push(value); + } + return values; + }, async createIndex(spec, opts = {}) { indexes.push([spec, opts]); return opts.name ?? 'index'; diff --git a/tests/helpers/queue/source-contract.js b/tests/helpers/queue/source-contract.js index cabf904..df4d09c 100644 --- a/tests/helpers/queue/source-contract.js +++ b/tests/helpers/queue/source-contract.js @@ -31,6 +31,21 @@ const requestOf = (id, extra = {}) => ({ * @param options `skip`, to register the cases skipped. */ const sourceContract = (name, makeSource, options = {}) => { + const fair = (title, body) => { + const fairOptions = { ...options, fairness: 'tenant' }; + test(`Queue source (${name}, fair): ${title}`, { skip: options.skip }, async (t) => { + let time = START; + const clock = { + now: () => time, + advance(milliseconds) { + time += milliseconds; + }, + }; + const made = await makeSource({ now: clock.now, fairness: fairOptions.fairness }); + if (made.cleanup !== undefined) t.after(made.cleanup); + await body(made.source, clock); + }); + }; const contract = (title, body) => test(`Queue source (${name}): ${title}`, { skip: options.skip }, async (t) => { let time = START; @@ -40,7 +55,7 @@ const sourceContract = (name, makeSource, options = {}) => { time += milliseconds; }, }; - const made = await makeSource({ now: clock.now }); + const made = await makeSource({ now: clock.now, fairness: options.fairness }); if (made.cleanup !== undefined) t.after(made.cleanup); await body(made.source, clock); }); @@ -189,6 +204,84 @@ const sourceContract = (name, makeSource, options = {}) => { clock.advance(2000); assert.strictEqual((await source.reserve(LEASE)).lease.job.id, 'a'); }); + + contract('a job gives back the tenant of its request', async (source) => { + await source.enqueue(requestOf('a', { tenantId: 'acme' })); + await source.enqueue(requestOf('b')); + const tenants = {}; + for (let index = 0; index < 2; index++) { + const { lease } = await source.reserve(LEASE); + tenants[lease.job.id] = lease.job.tenantId; + } + assert.deepStrictEqual(tenants, { a: 'acme', b: undefined }); + }); + + contract( + 'a pause of a tenant holds its jobs alone, until it ends or is resumed', + async (source, clock) => { + await source.enqueue(requestOf('a1', { tenantId: 'a' })); + await source.enqueue( + requestOf('b1', { tenantId: 'b', enqueuedAt: START + 1, availableAt: START }), + ); + await source.pause(START + 2000, { tenantId: 'a' }); + const { lease } = await source.reserve(LEASE); + assert.strictEqual(lease.job.id, 'b1'); + await source.ack(lease, null); + assert.deepStrictEqual(await source.reserve(LEASE), { nextAvailableAt: START + 2000 }); + clock.advance(2000); + const resumed = (await source.reserve(LEASE)).lease; + assert.strictEqual(resumed.job.id, 'a1'); + await source.retry(resumed, START + 2000, { code: 'x', reason: 'y' }); + await source.pause(START + 60_000, { tenantId: 'a' }); + assert.strictEqual((await source.reserve(LEASE)).lease, undefined); + await source.resume({ tenantId: 'a' }); + assert.strictEqual((await source.reserve(LEASE)).lease.job.id, 'a1'); + }, + ); + + contract('a pause of the queue ends early when resumed', async (source) => { + await source.enqueue(requestOf('a')); + await source.pause(START + 60_000); + assert.deepStrictEqual(await source.reserve(LEASE), { nextAvailableAt: START + 60_000 }); + await source.resume(); + assert.strictEqual((await source.reserve(LEASE)).lease.job.id, 'a'); + }); + + fair("the backlog of one tenant holds no other tenant's jobs", async (source) => { + for (let index = 0; index < 20; index++) { + await source.enqueue( + requestOf(`a${index}`, { tenantId: 'a', enqueuedAt: START + index, availableAt: START }), + ); + } + await source.enqueue( + requestOf('b0', { tenantId: 'b', enqueuedAt: START + 100, availableAt: START }), + ); + await source.enqueue(requestOf('n0', { enqueuedAt: START + 101, availableAt: START })); + await source.enqueue( + requestOf('h0', { + tenantId: 'a', + priority: 'high', + enqueuedAt: START + 102, + availableAt: START, + }), + ); + const order = []; + for (let index = 0; index < 6; index++) { + const { lease } = await source.reserve(LEASE); + order.push(lease.job.id); + await source.ack(lease, null); + } + // A high job first, of any tenant; then the tenants in turn, a tenant of none among them. + assert.strictEqual(order[0], 'h0'); + assert.ok(order.indexOf('b0') <= 3, order.join()); + assert.ok(order.indexOf('n0') <= 3, order.join()); + assert.deepStrictEqual( + order.filter((id) => id.startsWith('a')), + order + .filter((id) => id.startsWith('a')) + .toSorted((x, y) => Number(x.slice(1)) - Number(y.slice(1))), + ); + }); }; module.exports = { LEASE, START, requestOf, sourceContract }; diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index b8d6af3..7e3fa52 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -204,11 +204,11 @@ const { sourceContract } = require('../helpers/queue/source-contract.js'); sourceContract( 'mongodb, a replica set', - async ({ now }) => { + async ({ now, fairness }) => { const client = new MongoClient(url); await client.connect(); const db = client.db(`mailer_test_${randomUUID().replaceAll('-', '')}`); - const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), now }); + const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), now, fairness }); await outbox.ensureIndexes(); return { source: outbox.source, @@ -242,3 +242,22 @@ test( assert.doesNotMatch(text, /"stage":"SORT"/); }, ); + +sourceContract( + 'mongodb, a replica set, tenants in turn', + async ({ now, fairness }) => { + const client = new MongoClient(url); + await client.connect(); + const db = client.db(`mailer_test_${randomUUID().replaceAll('-', '')}`); + const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), now, fairness }); + await outbox.ensureIndexes(); + return { + source: outbox.source, + cleanup: async () => { + await db.dropDatabase(); + await client.close(); + }, + }; + }, + { skip, fairness: 'tenant' }, +); diff --git a/tests/types/mongodb.test-d.ts b/tests/types/mongodb.test-d.ts index 5704d25..39a5990 100644 --- a/tests/types/mongodb.test-d.ts +++ b/tests/types/mongodb.test-d.ts @@ -23,6 +23,8 @@ expectAssignable(typed); const typedOutbox = createMongoOutbox({ collection: db.collection('x') }); expectType(typedOutbox.transactional); +expectType>(typedOutbox.resume()); +createMongoOutbox({ collection: db.collection('x'), fairness: 'tenant' }); const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), changeStream: true, diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 39685f4..fac4bdb 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -32,7 +32,9 @@ declare const engine: TemplateEngine; const driver = createMemoryQueue({ maxJobs: 10, pollIntervalMilliseconds: 100 }); expectType(driver); expectType(driver.source); -createMemoryQueue({ retainData: 'none' }); +createMemoryQueue({ retainData: 'none', fairness: 'tenant' }); +expectType>(driver.resume({ tenantId: 'acme' })); +expectError(createMemoryQueue({ fairness: 'fifo' })); expectError(createMemoryQueue({ retainData: 'some' })); expectAssignable(driver); const queue = createMailQueue({ diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 7d9fa7e..07003a1 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -185,6 +185,7 @@ test('MongoDB: the outbox makes its indexes, and wakes its workers on a change s [{ state: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, { name: 'mailer_ready' }], [{ state: 1, leaseUntil: 1 }, { name: 'mailer_leases' }], [{ state: 1, availableAt: 1 }, { name: 'mailer_due' }], + [{ state: 1, tenantId: 1, rank: 1, availableAt: 1, enqueuedAt: 1 }, { name: 'mailer_tenant' }], [{ expireAt: 1 }, { name: 'mailer_expire', expireAfterSeconds: 0 }], ]); let woken = 0; @@ -516,12 +517,7 @@ test('MongoDB: a reserve takes back leases once a third of a lease, and reads th for (let index = 0; index < 5; index++) await source.reserve(60_000); // Idle, five reserves of five lanes: one write to take back leases, one read of the pause. assert.strictEqual(count('updateMany'), 1); - assert.strictEqual( - collection.calls.filter( - (call) => call[0] === 'findOne' && JSON.stringify(call[1]) === '{"_id":"mailer:pause"}', - ).length, - 1, - ); + assert.strictEqual(count('find'), 1); advance(20_000); await source.reserve(60_000); assert.strictEqual(count('updateMany'), 2); diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index 508cb8f..fb0a51e 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -644,3 +644,15 @@ test('Queue: a template that fails leaves its text in the log, not the store or records.some(([message, reason]) => message === 'queue.render.failed' && /4242/.test(reason)), ); }); + +test('Queue: a rate limit makes the job’s tenant wait under rateLimitScope tenant', async () => { + const transport = fakeTransport(() => + failure('rate-limit', { retryable: true, retryAfterMilliseconds: 60_000 }), + ); + const { handler } = handlerOf({ transport, rateLimitScope: 'tenant' }); + const tenanted = await run(handler, jobOf({ tenantId: 'acme' })); + assert.deepStrictEqual([tenanted.type, tenanted.scope], ['retry', 'tenant']); + // A job of no tenant waits alone. + const alone = await run(handler, jobOf({ idempotencyKey: 'n' })); + assert.strictEqual(alone.scope, 'job'); +}); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index bfaa156..f179b0f 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -610,3 +610,33 @@ test('Queue: a change wakes one waiting lane, not every lane, and is kept when a assert.strictEqual(reserves, 7); await consumer.close({ timeoutMilliseconds: 10 }); }); + +test('Queue: a retry of a tenant’s scope pauses that tenant’s jobs, not the others', async () => { + const clock = manualClock(10); + const queue = createMemoryQueue({ now: clock.now, sleep: clock.sleep }); + await queue.enqueue(requestOf('a1', { tenantId: 'a' })); + await queue.enqueue(requestOf('a2', { tenantId: 'a', enqueuedAt: 1 })); + await queue.enqueue(requestOf('b1', { tenantId: 'b', enqueuedAt: 2 })); + const runs = []; + const consumer = queue.consume(async (job) => { + runs.push(job.id); + if (job.id === 'a1' && job.attempt === 1) { + return { + type: 'retry', + delayMilliseconds: 60_000, + code: 'rate', + reason: 'limited', + scope: 'tenant', + }; + } + return { type: 'ack' }; + }); + await queue.drain(); + // a2 waits with a1 for the pause of tenant a; b1 goes. + assert.deepStrictEqual(runs, ['a1', 'b1']); + clock.advance(60_000); + queue.wake(); + await queue.drain(); + await consumer.close(); + assert.deepStrictEqual(runs.toSorted(), ['a1', 'a1', 'a2', 'b1']); +}); diff --git a/tests/unit/queue-source-contract.test.js b/tests/unit/queue-source-contract.test.js index b76feff..9149567 100644 --- a/tests/unit/queue-source-contract.test.js +++ b/tests/unit/queue-source-contract.test.js @@ -6,7 +6,23 @@ const { createMemoryQueue } = require('../../src/queue/index.js'); const { fakeCollection } = require('../helpers/mongodb/fake-collection.js'); const { sourceContract } = require('../helpers/queue/source-contract.js'); -sourceContract('memory', async ({ now }) => ({ source: createMemoryQueue({ now }).source })); -sourceContract('mongodb', async ({ now }) => ({ - source: createMongoOutbox({ collection: fakeCollection(), now }).source, +sourceContract('memory', async ({ now, fairness }) => ({ + source: createMemoryQueue({ now, fairness }).source, })); +sourceContract('mongodb', async ({ now, fairness }) => ({ + source: createMongoOutbox({ collection: fakeCollection(), now, fairness }).source, +})); + +// Fairness among tenants keeps every other rule of a source. +sourceContract( + 'memory, tenants in turn', + async ({ now }) => ({ source: createMemoryQueue({ now, fairness: 'tenant' }).source }), + { fairness: 'tenant' }, +); +sourceContract( + 'mongodb, tenants in turn', + async ({ now }) => ({ + source: createMongoOutbox({ collection: fakeCollection(), now, fairness: 'tenant' }).source, + }), + { fairness: 'tenant' }, +); From 6ed3d8e337952d8292509bcfda81a303c3ad938f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:56:30 +0300 Subject: [PATCH 40/83] test(mongodb): a context with keys of $ and dots goes through the outbox The review suspected that a template context with a key such as $amount or a.b would be refused by $setOnInsert and abort the application's transaction. MongoDB 7 stores such keys in a nested value and gives them back as they went, so the hash of the job holds and the worker sends it; the opt-in suite pins it on a replica set, in a transaction, rather than the outbox refusing keys MongoDB takes. Co-Authored-By: Claude Opus 5.5 --- tests/mongodb/outbox.mongodb.test.js | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index 7e3fa52..79e50ca 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -261,3 +261,38 @@ sourceContract( }, { skip, fairness: 'tenant' }, ); + +test( + 'MongoDB (replica set): a context with keys of $ and dots is stored, read back and sent', + { skip }, + async (t) => { + const { client, outbox, queue, worker } = await setup(t); + const { createMemoryLoader, createTemplateRenderer } = require('../../src/templates/index.js'); + const { fakeEngine } = require('../helpers/templates/fake-engine.js'); + const renderer = createTemplateRenderer({ + engine: fakeEngine(), + loader: createMemoryLoader({ 'price.tpl': '---\nsubject: Price\n---\n

    {{name}}

    ' }), + }); + const session = client.startSession(); + let enqueued; + await session.withTransaction(async () => { + enqueued = await queue.enqueue( + { + kind: 'template', + template: 'price', + context: { name: 'Ada', $amount: 5, 'a.b': 1, price: { $gt: 1 } }, + from: 'app@example.com', + to: 'ada@example.com', + idempotencyKey: randomUUID(), + }, + { transaction: session }, + ); + }); + await session.endSession(); + const transport = fakeTransport(); + worker({ transport, renderer }); + await until(async () => (await outbox.get(enqueued.id))?.state === 'completed'); + // Its hash held after the round trip: the keys came back as they went. + assert.equal(transport.calls.length, 1); + }, +); From df74957968c9597deb2a75162ae2daf34e430a87 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:59:01 +0300 Subject: [PATCH 41/83] refactor(templates): share what the engines keep per renderer and the checks of extensions Five engines kept an instance per includes in a WeakMap of their own, four made a TemplateNotFoundError of the one name they read, and the loader and the engines each had a check of extensions and a test of a name's extension. They are perIncludes, missingTemplate, checkExtensions and hasExtension of src/templates/environment.js; the file loader reads through two functions of its own rather than two copies inside one. The messages of a bad list of extensions are one for the loader and engines. Co-Authored-By: Claude Opus 5.5 --- src/templates/environment.js | 68 ++++++++++ src/templates/eta.js | 14 +- src/templates/handlebars.js | 22 ++-- src/templates/includes.js | 10 +- src/templates/liquid.js | 15 +-- src/templates/loader.js | 183 +++++++++++++-------------- src/templates/mustache.js | 24 ++-- src/templates/nunjucks.js | 12 +- src/templates/options.js | 14 +- src/templates/pug.js | 3 +- tests/unit/templates-engines.test.js | 3 +- tests/unit/templates-loader.test.js | 2 +- 12 files changed, 202 insertions(+), 168 deletions(-) create mode 100644 src/templates/environment.js diff --git a/src/templates/environment.js b/src/templates/environment.js new file mode 100644 index 0000000..347df5e --- /dev/null +++ b/src/templates/environment.js @@ -0,0 +1,68 @@ +// Not ported: what the engines of ./templates share: what an engine keeps for +// each renderer, the template it found missing by name, and the extensions +// templates are found by. + +const { TemplateNotFoundError } = require('./errors.js'); + +/** + * What an engine keeps for each renderer, such as its environment: made by + * `make` for the first template of a renderer's includes, and kept by them in + * a WeakMap, so that it goes after `clear()`, which makes them anew, and with + * each render without a cache. + * + * @param make Makes it, given the includes. + * @returns `(includes) => instance`. + * @internal + */ +const perIncludes = (make) => { + const instances = new WeakMap(); + return (includes) => { + let instance = instances.get(includes); + if (instance === undefined) { + instance = make(includes); + instances.set(includes, instance); + } + return instance; + }; +}; + +/** + * A template an engine read by its name, which is not there. + * + * @internal + */ +const missingTemplate = (name, from) => + new TemplateNotFoundError(name, [name], from === undefined ? undefined : { from }); + +/** + * Whether a name ends with one of the extensions, the empty one aside. + * + * @internal + */ +const hasExtension = (name, extensions) => { + for (const extension of extensions) { + if (extension !== '' && name.endsWith(extension)) return true; + } + return false; +}; + +/** + * Checks a list of extensions, each with its dot. + * + * @returns A frozen copy. + * @throws {TypeError} If it is not an array of them. + * @internal + */ +const checkExtensions = (extensions, what) => { + if (!Array.isArray(extensions)) { + throw new TypeError(`${what} must be an array of extensions, such as [".hbs"].`); + } + for (const extension of extensions) { + if (typeof extension !== 'string' || extension.length < 2 || extension[0] !== '.') { + throw new TypeError(`An extension starts with its dot, such as ".hbs": ${String(extension)}`); + } + } + return Object.freeze([...extensions]); +}; + +module.exports = { checkExtensions, hasExtension, missingTemplate, perIncludes }; diff --git a/src/templates/eta.js b/src/templates/eta.js index 8b3dcf1..ecf99a1 100644 --- a/src/templates/eta.js +++ b/src/templates/eta.js @@ -3,7 +3,7 @@ // so `include()` and `layout()` read nothing from disk, a name relative to the // template it is written in; output is escaped, as Eta does by default. -const { TemplateNotFoundError } = require('./errors.js'); +const { missingTemplate, perIncludes } = require('./environment.js'); const { checkEngineExtensions, checkEngineOptions, @@ -38,21 +38,17 @@ const createEtaEngine = (options = {}) => { const async = options.async ?? false; if (typeof async !== 'boolean') throw new TypeError('async must be a boolean.'); const extensions = checkEngineExtensions(options.extensions, ['.eta']); - const instances = new WeakMap(); - const instanceOf = (includes) => { - let eta = instances.get(includes); - if (eta !== undefined) return eta; - eta = new Eta({ ...given, cache: includes.cache, cacheFilepaths: false }); + const instanceOf = perIncludes((includes) => { + const eta = new Eta({ ...given, cache: includes.cache, cacheFilepaths: false }); eta.resolvePath = (template, resolveOptions) => includes.lookupSync(template, resolveOptions?.filepath, extensions).name; eta.readFile = (name) => { const found = includes.readSync(name); - if (found === undefined) throw new TemplateNotFoundError(name, [name]); + if (found === undefined) throw missingTemplate(name); return found.source; }; - instances.set(includes, eta); return eta; - }; + }); return { name: 'eta', extensions, diff --git a/src/templates/handlebars.js b/src/templates/handlebars.js index 4b76c84..30534c8 100644 --- a/src/templates/handlebars.js +++ b/src/templates/handlebars.js @@ -13,6 +13,7 @@ const { checkModule, } = require('./options.js'); const { checkOptionalFunction } = require('../internal/options.js'); +const { perIncludes } = require('./environment.js'); const PARTIALS = new Set(['PartialStatement', 'PartialBlockStatement']); // The options of compile that turn off its escaping. @@ -91,20 +92,13 @@ const createHandlebarsEngine = (options = {}) => { checkEscaping(options, given, UNESCAPED, 'handlebars'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.hbs', '.handlebars']); - // The environment of each renderer, keyed by its includes: a new one after - // clear(), and for each render without a cache. - const environments = new WeakMap(); - const environmentOf = (includes) => { - let environment = environments.get(includes); - if (environment === undefined) { - const env = handlebars.create(); - for (const name of Object.keys(helpers)) env.registerHelper(name, helpers[name]); - configure?.(env); - environment = { env, partials: new Map() }; - environments.set(includes, environment); - } - return environment; - }; + // The environment of each renderer. + const environmentOf = perIncludes(() => { + const env = handlebars.create(); + for (const name of Object.keys(helpers)) env.registerHelper(name, helpers[name]); + configure?.(env); + return { env, partials: new Map() }; + }); // Compiles and registers a partial once per environment however many // templates name it at once, under the name of its template, and gives the // partials it names, which it does not wait for: a partial that names diff --git a/src/templates/includes.js b/src/templates/includes.js index 8407a59..5eac7af 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -5,6 +5,7 @@ const { isTemplateError, TemplateLoadError, TemplateNotFoundError } = require('./errors.js'); const { checkTemplateName, isTemplateName, notTemplateName } = require('./loader.js'); const { createCache, memoize } = require('./cache.js'); +const { hasExtension } = require('./environment.js'); // The segments of a name, `.` dropped and `..` taking the one before; undefined // when `..` leads out of the root. @@ -45,13 +46,6 @@ const joinName = (from, include) => { return normalize(`${directoryOf(from)}/${include}`.split('/')); }; -const hasAnyExtension = (name, extensions) => { - for (const extension of extensions) { - if (extension !== '' && name.endsWith(extension)) return true; - } - return false; -}; - // The names an include may be: as joined, then under each partials directory // for a bare name; each with an extension of the engine unless it has one, // then as it is. @@ -68,7 +62,7 @@ const candidatesOf = (include, from, extensions, partials) => { } const names = []; for (const base of bases) { - if (hasAnyExtension(base, extensions)) { + if (hasExtension(base, extensions)) { names.push(base); continue; } diff --git a/src/templates/liquid.js b/src/templates/liquid.js index 8afdb86..fde177a 100644 --- a/src/templates/liquid.js +++ b/src/templates/liquid.js @@ -12,6 +12,7 @@ const { checkModule, } = require('./options.js'); const { checkOptionalFunction } = require('../internal/options.js'); +const { missingTemplate, perIncludes } = require('./environment.js'); const RESERVED = new Set([ 'root', @@ -88,13 +89,13 @@ const fileSystemOf = (includes, roots, extname) => { async readFile(name) { if (isMissing(name)) throw missing(name); const found = await includes.read(name); - if (found === undefined) throw new TemplateNotFoundError(name, [name]); + if (found === undefined) throw missingTemplate(name); return found.source; }, readFileSync(name) { if (isMissing(name)) throw missing(name); const found = includes.readSync(name); - if (found === undefined) throw new TemplateNotFoundError(name, [name]); + if (found === undefined) throw missingTemplate(name); return found.source; }, resolve, @@ -131,12 +132,9 @@ const createLiquidEngine = (options = {}) => { const filters = checkFunctions(options.filters, 'filters'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.liquid']); - const instances = new WeakMap(); - const instanceOf = (includes) => { - let liquid = instances.get(includes); - if (liquid !== undefined) return liquid; + const instanceOf = perIncludes((includes) => { const roots = ['', ...includes.partials]; - liquid = new Liquid({ + const liquid = new Liquid({ outputEscape: 'escape', ownPropertyOnly: true, ...LIMITS, @@ -151,9 +149,8 @@ const createLiquidEngine = (options = {}) => { }); for (const name of Object.keys(filters)) liquid.registerFilter(name, filters[name]); configure?.(liquid); - instances.set(includes, liquid); return liquid; - }; + }); return { name: 'liquid', extensions, diff --git a/src/templates/loader.js b/src/templates/loader.js index ceccac1..d1883ab 100644 --- a/src/templates/loader.js +++ b/src/templates/loader.js @@ -8,6 +8,7 @@ const fsp = require('node:fs/promises'); const path = require('node:path'); const { isWithin } = require('../internal/path.js'); const { checkPositiveInteger } = require('../internal/options.js'); +const { checkExtensions, hasExtension } = require('./environment.js'); const NAME = /^[A-Za-z0-9_@-][A-Za-z0-9_.@/-]*$/; const MAX_NAME_LENGTH = 256; @@ -57,27 +58,6 @@ const checkTemplateName = (name, what) => { return name; }; -const checkExtensions = (extensions, what) => { - if (extensions === undefined) return undefined; - if (!Array.isArray(extensions)) throw new TypeError(`${what} must be an array.`); - const set = new Set(); - for (const extension of extensions) { - if (typeof extension !== 'string' || extension.length < 2 || extension[0] !== '.') { - throw new TypeError(`${what} holds extensions with their dot, such as ".hbs".`); - } - set.add(extension); - } - return set; -}; - -const hasExtension = (name, extensions) => { - if (extensions === undefined) return true; - for (const extension of extensions) { - if (name.endsWith(extension)) return true; - } - return false; -}; - const isMissing = (error) => error?.code === 'ENOENT' || error?.code === 'ENOTDIR'; // A file is opened without following a link its last segment became after @@ -139,6 +119,82 @@ const stripBom = (text) => (text.charCodeAt(0) === 0xfeff ? text.slice(1) : text // directory, which Windows does not open. const isNoFile = (error) => isMissing(error) || error?.code === 'EISDIR'; +// A real path under the root of the files, or a refusal. +const checkWithin = (files, name, real) => { + if (!isWithin(real, files.root)) { + throw new RangeError(`The template ${JSON.stringify(name)} leads out of ${files.dir}.`); + } +}; + +const tooLong = (files, name) => + new RangeError(`The template ${JSON.stringify(name)} holds more than ${files.maxBytes} bytes.`); + +const sourceOf = (files, name, real, bytes) => { + if (bytes.length > files.maxBytes) throw tooLong(files, name); + return { name, source: stripBom(bytes.toString('utf8')), path: real }; +}; + +const accepts = (files, name) => + isTemplateName(name) && (files.extensions === undefined || hasExtension(name, files.extensions)); + +const loadFile = async (files, name, signal) => { + if (!accepts(files, name)) return undefined; + signal?.throwIfAborted(); + files.root ??= await fsp.realpath(files.dir); + let real; + try { + real = await fsp.realpath(path.join(files.root, name)); + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } + checkWithin(files, name, real); + signal?.throwIfAborted(); + let handle; + try { + handle = await fsp.open(real, OPEN_FLAGS); + } catch (error) { + if (isNoFile(error)) return undefined; + throw error; + } + try { + const stats = await handle.stat(); + if (!stats.isFile()) return undefined; + if (stats.size > files.maxBytes) throw tooLong(files, name); + return sourceOf(files, name, real, await readAtMost(handle, stats.size, files.maxBytes + 1)); + } finally { + await handle.close(); + } +}; + +const loadFileSync = (files, name) => { + if (!accepts(files, name)) return undefined; + files.root ??= fs.realpathSync(files.dir); + let real; + try { + real = fs.realpathSync(path.join(files.root, name)); + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } + checkWithin(files, name, real); + let fd; + try { + fd = fs.openSync(real, OPEN_FLAGS); + } catch (error) { + if (isNoFile(error)) return undefined; + throw error; + } + try { + const stats = fs.fstatSync(fd); + if (!stats.isFile()) return undefined; + if (stats.size > files.maxBytes) throw tooLong(files, name); + return sourceOf(files, name, real, readAtMostSync(fd, stats.size, files.maxBytes + 1)); + } finally { + fs.closeSync(fd); + } +}; + /** * Makes a loader of the templates in a directory. * @@ -152,81 +208,20 @@ const createFileLoader = (options = {}) => { if (typeof dir !== 'string' || dir === '') { throw new TypeError('createFileLoader needs dir, the directory of the templates.'); } - const extensions = checkExtensions(options.extensions, 'extensions'); - const maxBytes = checkPositiveInteger(options.maxBytes ?? DEFAULT_MAX_BYTES, 'maxBytes'); - // The root is resolved when it is first read, so that a loader can be made - // before its directory exists, and then kept. - let root; - const checkWithin = (name, real) => { - if (!isWithin(real, root)) { - throw new RangeError(`The template ${JSON.stringify(name)} leads out of ${dir}.`); - } - }; - const tooLong = (name) => - new RangeError(`The template ${JSON.stringify(name)} holds more than ${maxBytes} bytes.`); - const sourceOf = (name, real, bytes) => { - if (bytes.length > maxBytes) throw tooLong(name); - return { name, source: stripBom(bytes.toString('utf8')), path: real }; + const files = { + dir, + extensions: + options.extensions === undefined + ? undefined + : checkExtensions(options.extensions, 'extensions'), + maxBytes: checkPositiveInteger(options.maxBytes ?? DEFAULT_MAX_BYTES, 'maxBytes'), + // The root is resolved when it is first read, so that a loader can be + // made before its directory exists, and then kept. + root: undefined, }; - const accepts = (name) => isTemplateName(name) && hasExtension(name, extensions); return { - async load(name, loadOptions) { - if (!accepts(name)) return undefined; - const signal = loadOptions?.signal; - signal?.throwIfAborted(); - root ??= await fsp.realpath(dir); - let real; - try { - real = await fsp.realpath(path.join(root, name)); - } catch (error) { - if (isMissing(error)) return undefined; - throw error; - } - checkWithin(name, real); - signal?.throwIfAborted(); - let handle; - try { - handle = await fsp.open(real, OPEN_FLAGS); - } catch (error) { - if (isNoFile(error)) return undefined; - throw error; - } - try { - const stats = await handle.stat(); - if (!stats.isFile()) return undefined; - if (stats.size > maxBytes) throw tooLong(name); - return sourceOf(name, real, await readAtMost(handle, stats.size, maxBytes + 1)); - } finally { - await handle.close(); - } - }, - loadSync(name) { - if (!accepts(name)) return undefined; - root ??= fs.realpathSync(dir); - let real; - try { - real = fs.realpathSync(path.join(root, name)); - } catch (error) { - if (isMissing(error)) return undefined; - throw error; - } - checkWithin(name, real); - let fd; - try { - fd = fs.openSync(real, OPEN_FLAGS); - } catch (error) { - if (isNoFile(error)) return undefined; - throw error; - } - try { - const stats = fs.fstatSync(fd); - if (!stats.isFile()) return undefined; - if (stats.size > maxBytes) throw tooLong(name); - return sourceOf(name, real, readAtMostSync(fd, stats.size, maxBytes + 1)); - } finally { - fs.closeSync(fd); - } - }, + load: (name, loadOptions) => loadFile(files, name, loadOptions?.signal), + loadSync: (name) => loadFileSync(files, name), }; }; diff --git a/src/templates/mustache.js b/src/templates/mustache.js index 8430dc3..45944ec 100644 --- a/src/templates/mustache.js +++ b/src/templates/mustache.js @@ -5,6 +5,7 @@ // where Mustache renders it as nothing. const { createCache } = require('./cache.js'); +const { perIncludes } = require('./environment.js'); const { checkEngineExtensions, checkModule } = require('./options.js'); // The parsed templates a writer keeps. @@ -39,21 +40,16 @@ const createMustacheEngine = (options = {}) => { ); const tags = checkTags(options.tags); const extensions = checkEngineExtensions(options.extensions, ['.mustache']); - const writers = new WeakMap(); - const writerOf = (includes) => { - let writer = writers.get(includes); - if (writer === undefined) { - writer = new mustache.Writer(); - const parsed = createCache(includes.cache ? MAX_PARSED : 0); - // Mustache calls set and get; the renderer forgets a writer, not its cache. - writer.templateCache = { - set: (key, value) => parsed.set(key, value), - get: (key) => parsed.get(key), - }; - writers.set(includes, writer); - } + const writerOf = perIncludes((includes) => { + const writer = new mustache.Writer(); + const parsed = createCache(includes.cache ? MAX_PARSED : 0); + // Mustache calls set and get; the renderer forgets a writer, not its cache. + writer.templateCache = { + set: (key, value) => parsed.set(key, value), + get: (key) => parsed.get(key), + }; return writer; - }; + }); return { name: 'mustache', extensions, diff --git a/src/templates/nunjucks.js b/src/templates/nunjucks.js index b73e056..6863757 100644 --- a/src/templates/nunjucks.js +++ b/src/templates/nunjucks.js @@ -13,6 +13,7 @@ const { checkModule, } = require('./options.js'); const { checkOptionalFunction } = require('../internal/options.js'); +const { perIncludes } = require('./environment.js'); const isRelative = (name) => name.startsWith('./') || name.startsWith('../'); @@ -67,10 +68,7 @@ const createNunjucksEngine = (options = {}) => { const filters = checkFunctions(options.filters, 'filters'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.njk', '.nunjucks']); - const environments = new WeakMap(); - const environmentOf = (includes) => { - let state = environments.get(includes); - if (state !== undefined) return state; + const environmentOf = perIncludes((includes) => { // The lookups that found nothing, by name: Nunjucks is told there is no // such template, which it fails on itself unless `ignore missing`, and // its error names the template, so each render finds its own. @@ -104,10 +102,8 @@ const createNunjucksEngine = (options = {}) => { const env = new nunjucks.Environment([loader], { autoescape: true, dev: true, ...given }); for (const name of Object.keys(filters)) env.addFilter(name, filters[name]); configure?.(env); - state = { env, failures }; - environments.set(includes, state); - return state; - }; + return { env, failures }; + }); return { name: 'nunjucks', extensions, diff --git a/src/templates/options.js b/src/templates/options.js index 37a2713..6f30b9d 100644 --- a/src/templates/options.js +++ b/src/templates/options.js @@ -2,6 +2,8 @@ // option an engine keeps for itself, such as the file name its includes are // read by, is refused rather than accepted and never read. +const { checkExtensions } = require('./environment.js'); + /** * Checks the options passed on to an engine, and gives a copy of them. * @@ -64,15 +66,9 @@ const checkEscaping = (options, given, switches, engine) => { */ const checkEngineExtensions = (extensions, defaults) => { if (extensions === undefined) return Object.freeze([...defaults]); - if (!Array.isArray(extensions) || extensions.length === 0) { - throw new TypeError('extensions must be a list such as [".hbs"].'); - } - for (const extension of extensions) { - if (typeof extension !== 'string' || extension.length < 2 || extension[0] !== '.') { - throw new TypeError(`An extension starts with its dot, such as ".hbs": ${String(extension)}`); - } - } - return Object.freeze([...extensions]); + const checked = checkExtensions(extensions, 'extensions'); + if (checked.length === 0) throw new TypeError('extensions must name one at least.'); + return checked; }; /** diff --git a/src/templates/pug.js b/src/templates/pug.js index 7381342..4adb461 100644 --- a/src/templates/pug.js +++ b/src/templates/pug.js @@ -6,6 +6,7 @@ // @nestjs-modules/mailer rendered the file again on each send with the // options of Pug mixed into them. +const { missingTemplate } = require('./environment.js'); const { TemplateNotFoundError } = require('./errors.js'); const { checkEngineExtensions, checkEngineOptions, checkModule } = require('./options.js'); @@ -49,7 +50,7 @@ const createPugEngine = (options = {}) => { const name = nameOf(filename); const found = context.includes.readSync(name); if (found === undefined) { - throw new TemplateNotFoundError(name, [name], { from: context.name }); + throw missingTemplate(name, context.name); } return found.source; }, diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 8bfa613..dc6359b 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -240,7 +240,8 @@ test('Templates: the EJS engine refuses what it sets itself', () => { assert.throws(() => createEjsEngine({ ejs, options: { [name]: 1 } }), new RegExp(name)); } assert.throws(() => createEjsEngine({ ejs, options: 1 }), /must be an object/); - assert.throws(() => createEjsEngine({ ejs, extensions: [] }), /a list/); + assert.throws(() => createEjsEngine({ ejs, extensions: [] }), /name one at least/); + assert.throws(() => createEjsEngine({ ejs, extensions: '.ejs' }), /must be an array/); assert.throws(() => createEjsEngine({ ejs, extensions: ['ejs'] }), /starts with its dot/); assert.deepStrictEqual(createEjsEngine({ ejs, extensions: ['.html'] }).extensions, ['.html']); }); diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index 31f06a6..796be2f 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -175,7 +175,7 @@ test('Templates: the loaders check their options', () => { assert.throws(() => createFileLoader(), /needs dir/); assert.throws(() => createFileLoader({ dir: '' }), /needs dir/); assert.throws(() => createFileLoader({ dir: 'x', extensions: '.ejs' }), /must be an array/); - assert.throws(() => createFileLoader({ dir: 'x', extensions: ['ejs'] }), /with their dot/); + assert.throws(() => createFileLoader({ dir: 'x', extensions: ['ejs'] }), /starts with its dot/); assert.throws(() => createFileLoader({ dir: 'x', maxBytes: 0 }), /maxBytes/); assert.throws(() => createMemoryLoader(), /needs the templates/); assert.throws(() => createMemoryLoader([]), /needs the templates/); From fbfe811e83447d3cc214833e68c32c0ac5cdc879 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 00:59:34 +0300 Subject: [PATCH 42/83] fix(templates): read an EJS include against the template it is written in EJS tells the includer the name written, so the adapter read a relative include against the template rendered: `./b` written in emails/parts/a read emails/b. EJS calls the includer as a method of the options of the template the include is written in, whose file name is now its name in the loader, so that is the template it is read against; error messages name the template as the loader does, not its path on disk. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 7 +++---- src/templates/ejs.js | 18 ++++++++++------- src/templates/index.d.ts | 4 ++-- tests/unit/templates-engines.test.js | 29 ++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index d017b5e..1f96280 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -118,8 +118,7 @@ Every include, partial and layout an engine reads goes through the loader of the renderer: - `/x` is read from the root of the loader; -- `./x` and `../x` against the template it is written in (EJS: the template - rendered, as EJS tells no more); +- `./x` and `../x` against the template it is written in; - any other name from the root, then from each directory of `partials`; - the extensions of the engine are added when the name has none. @@ -254,8 +253,8 @@ Each takes `Infinity` for no bound, and going past one is a `TemplateRenderError`. The includes are counted as an engine runs synchronously: those EJS, Eta and Mustache look up as they render, and those Pug and Handlebars look up as they compile, until their first await. The depth -is counted where the engine names the template an include is written in (Eta, -Handlebars); a partial that names itself is no deeper for it. The file loader reads no template over its `maxBytes` (1 MiB). +is counted where the engine names the template an include is written in (EJS, +Eta, Handlebars); a partial that names itself is no deeper for it. The file loader reads no template over its `maxBytes` (1 MiB). ## Timeouts and aborts diff --git a/src/templates/ejs.js b/src/templates/ejs.js index bdccebe..6d21c7c 100644 --- a/src/templates/ejs.js +++ b/src/templates/ejs.js @@ -39,16 +39,20 @@ const createEjsEngine = (options = {}) => { name: 'ejs', extensions, compile(source, context) { - // EJS tells the includer the name written, not the template it is - // written in: a relative include is read against the template rendered. - const includer = (include) => { - const found = context.includes.lookupSync(include, context.name, extensions); - return { filename: found.path ?? found.name, template: found.source }; + const { includes } = context; + const hooks = { + // EJS calls the includer as a method of the options of the template + // the include is written in, whose file name is its name in the + // loader: a relative include is read against it. + includer(include) { + const found = includes.lookupSync(include, this.filename, extensions); + return { filename: found.name, template: found.source }; + }, }; return ejs.compile(source, { ...given, - filename: context.path ?? context.name, - includer, + filename: context.name, + includer: hooks.includer, cache: false, client: false, }); diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 1687cd0..e7af1f4 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -455,8 +455,8 @@ export interface EjsEngineOptions { } /** * The EJS engine. An include is read through the loader: `/x` from its root, - * `./x` against the template rendered, any other from the root, then the - * directories of partials. + * `./x` against the template it is written in, any other from the root, then + * the directories of partials. */ export declare function createEjsEngine(options: EjsEngineOptions): TemplateEngine; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index dc6359b..b23712c 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -89,6 +89,35 @@ test('Templates: EJS escapes, includes through the loader and keeps raw output r assert.strictEqual(syntax.engine, 'ejs'); }); +test('Templates: an EJS include is read against the template it is written in', async () => { + // Before: against the template rendered, as EJS tells the includer no more + // than the name written, so ./b of emails/parts/a read emails/b. + const engine = createEjsEngine({ ejs }); + const templates = { + 'emails/welcome.ejs': '<%- include("./parts/a") %>', + 'emails/parts/a.ejs': 'a<%- include("./b") %><%- include("../c") %>', + 'emails/parts/b.ejs': 'b', + 'emails/b.ejs': 'wrong', + 'emails/c.ejs': 'c', + }; + assert.strictEqual( + (await renderWith(engine, templates, { template: 'emails/welcome' })).html, + 'abc', + ); + const missing = await failureOf( + renderWith( + engine, + { 'x/a.ejs': '<%- include("./y/b") %>', 'x/y/b.ejs': '<%- include("./nope") %>' }, + { template: 'x/a' }, + ), + ); + assert.ok(missing instanceof TemplateNotFoundError); + assert.deepStrictEqual( + [missing.from, missing.tried], + ['x/y/b.ejs', ['x/y/nope.ejs', 'x/y/nope']], + ); +}); + test('Templates: an async EJS template that throws rejects, rather than hanging the send', async () => { // Before, in @nestjs-modules/mailer: `rendered.then(render)` had no catch. const engine = createEjsEngine({ ejs, options: { async: true } }); From 83fc2b19b2bafb8665ecd28acd992c3633e49bfb Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:00:35 +0300 Subject: [PATCH 43/83] fix(templates): look up a Pug include as every engine's, with its extensions The Pug adapter read a bare name against the file, where every other engine reads it from the root, then the directories of partials, and Pug added `.pug` to a name without an extension before the adapter saw it, so `extensions` was never read. A path written without an extension is marked as it is lexed, the mark taken off where Pug added `.pug` after it, and an include is looked up as the other engines' are, with the extensions of the engine; `include ./footer` reads the file beside. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 4 +++ src/templates/index.d.ts | 6 +++- src/templates/pug.js | 50 +++++++++++++++++----------- tests/unit/templates-engines.test.js | 38 ++++++++++++++++++++- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 1f96280..0181f68 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -312,6 +312,10 @@ causes and thrown as it is. What changes in behavior: +- a bare Pug include (`include footer`) is read from the root, then the + directories of partials, as in every engine here, where Pug reads it against + the file: write `include ./footer` for that; + - a failed CSS inlining rejects once, where the callback was called twice; - an async EJS template that throws rejects, where its promise rejected unheard and the send hung; diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index e7af1f4..75c71e4 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -515,7 +515,11 @@ export interface PugEngineOptions { /** @default ['.pug'] */ readonly extensions?: readonly string[]; } -/** The Pug engine: `include` and `extends` through the loader, `/x` from its root, any other against the file. */ +/** + * The Pug engine: `include` and `extends` through the loader, `/x` from its + * root, `./x` against the file, any other from the root, then the + * directories of partials, with the extensions of the engine. + */ export declare function createPugEngine(options: PugEngineOptions): TemplateEngine; /** What MJML finds wrong in a template. */ diff --git a/src/templates/pug.js b/src/templates/pug.js index 4adb461..57d05e6 100644 --- a/src/templates/pug.js +++ b/src/templates/pug.js @@ -1,17 +1,36 @@ // Not ported: the Pug engine of ./templates, over the `pug` the application // passes in. A plugin of `resolve` and `read` takes the place of Pug's disk: -// `include` and `extends` are read through the loader, a name with `/` from -// its root and any other against the file it is written in. Each template is -// compiled once, and its locals are the context alone, where -// @nestjs-modules/mailer rendered the file again on each send with the -// options of Pug mixed into them. +// `include` and `extends` are looked up through the loader as every engine's +// includes are, `/x` from its root, `./x` against the file it is written in, +// any other from the root, then the directories of partials, with the +// extensions of the engine. Each template is compiled once, and its locals +// are the context alone, where @nestjs-modules/mailer rendered the file again +// on each send with the options of Pug mixed into them. +const path = require('node:path'); const { missingTemplate } = require('./environment.js'); -const { TemplateNotFoundError } = require('./errors.js'); const { checkEngineExtensions, checkEngineOptions, checkModule } = require('./options.js'); const RESERVED = new Set(['filename', 'basedir', 'cache']); +// Pug adds `.pug` to a path written without an extension before it resolves +// it: such a path is marked first, so that the extensions of the engine are +// tried in its place. +const BARE = '\u0000'; +const ADDED = `${BARE}.pug`; + +const markBare = (tokens) => { + for (const token of tokens) { + if (token.type === 'path' && path.extname(token.val) === '') token.val += BARE; + } + return tokens; +}; + +const writtenOf = (filename) => { + const trimmed = filename.trim(); + return trimmed.endsWith(ADDED) ? trimmed.slice(0, -ADDED.length) : trimmed; +}; + // Pug's file names are the loader's names with a `/` before them. const nameOf = (filename) => (filename[0] === '/' ? filename.slice(1) : filename); @@ -36,22 +55,15 @@ const createPugEngine = (options = {}) => { name: 'pug', extensions, compile(source, context) { + const { includes } = context; const loader = { - resolve(filename, from) { - const written = filename.trim(); - const relative = written[0] === '/' || written.startsWith('.') ? written : `./${written}`; - const name = context.includes.join(nameOf(from ?? `/${context.name}`), relative); - if (name === undefined) { - throw new TemplateNotFoundError(written, [], { from: nameOf(from ?? context.name) }); - } - return `/${name}`; - }, + postLex: markBare, + resolve: (filename, from) => + `/${includes.lookupSync(writtenOf(filename), nameOf(from ?? context.name), extensions).name}`, read(filename) { const name = nameOf(filename); - const found = context.includes.readSync(name); - if (found === undefined) { - throw missingTemplate(name, context.name); - } + const found = includes.readSync(name); + if (found === undefined) throw missingTemplate(name, context.name); return found.source; }, }; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index b23712c..3f3fd98 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -460,7 +460,7 @@ test('Templates: Pug reads include and extends through the loader, its locals th const engine = createPugEngine({ pug, options: { doctype: 'html' } }); const templates = { 'emails/welcome.pug': - 'extends ../layouts/main\nblock content\n p= name\n p= filename\n include footer\n include /legal', + 'extends ../layouts/main\nblock content\n p= name\n p= filename\n include ./footer\n include /legal', 'layouts/main.pug': 'html\n body\n block content', 'emails/footer.pug': 'footer bye', 'legal.pug': 'small legal', @@ -497,6 +497,42 @@ test('Templates: Pug reads include and extends through the loader, its locals th assert.deepStrictEqual(seen, ['Block']); }); +test('Templates: Pug looks up a bare name as every engine does, with its extensions', async () => { + // Before: a bare name was read against the file, and \`.pug\` was added + // whatever the extensions of the engine. + const templates = { + 'emails/welcome.pug': 'include footer\ninclude ./sign\ninclude legal.html', + 'emails/footer.pug': 'p wrong', + 'partials/footer.pug': 'footer bye', + 'emails/sign.pug': 'i sign', + // Raw, as Pug includes a file of another extension. + 'legal.html': 'legal', + }; + const engine = createPugEngine({ pug }); + const output = await renderWith( + engine, + templates, + { template: 'emails/welcome' }, + { partials: ['partials'] }, + ); + assert.strictEqual(output.html, '
    bye
    signlegal'); + const jade = createPugEngine({ pug, extensions: ['.jade'] }); + const own = await renderWith( + jade, + { 'a.jade': 'include b', 'b.jade': 'b jade', 'b.pug': 'b pug' }, + { template: 'a' }, + ); + assert.strictEqual(own.html, 'jade'); + const missing = await failureOf( + renderWith(jade, { 'a.jade': 'include nope' }, { template: 'a' }), + ); + assert.ok(missing instanceof TemplateNotFoundError); + assert.deepStrictEqual( + [missing.template, missing.tried, missing.from], + ['nope', ['nope.jade', 'nope'], 'a.jade'], + ); +}); + test('Templates: MJML renders after the engine inside it, whose errors reach it', async () => { const inner = createHandlebarsEngine({ handlebars }); const engine = createMjmlEngine({ mjml2html, engine: inner, onErrors: () => {} }); From 04136de5c052cf84357959931b52dd494c829535 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:01:30 +0300 Subject: [PATCH 44/83] fix(templates): read a Mustache partial against the template it is written in Mustache tells the partials of a template nothing of the template, so the adapter looked up every partial from the root, and `./c` in a partial of emails/parts read c, or emails/c. The writer of the engine renders a partial as Mustache does, its indentation included, with the partials of the template it found, so that each looks up its own against it. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 6 ++++-- src/templates/index.d.ts | 6 +++++- src/templates/mustache.js | 26 +++++++++++++++++++++++--- tests/unit/templates-engines.test.js | 20 ++++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 0181f68..2e8e94c 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -162,8 +162,10 @@ Some engines behave a little differently here than they do on their own: be registered in `configure`. - **Liquid** escapes output by default, where liquidjs escapes nothing unless told. It reads own properties only. -- **Mustache** fails on a partial that is not there, where it renders nothing. - Its writer, and the cache of what it parsed, are per renderer and bounded. +- **Mustache** fails on a partial that is not there, where it renders nothing, + and reads `./x` against the partial it is written in, which Mustache does + not tell. Its writer, and the cache of what it parsed, are per renderer and + bounded. - **MJML** renders after the engine inside it (`.mjml.hbs`), once the layout is applied, so the layout of an MJML template is MJML too. A static `.mjml` is rendered once. What MJML finds wrong under `validationLevel: 'soft'` goes to diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 75c71e4..d4c1ea0 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -667,7 +667,11 @@ export interface MustacheEngineOptions { /** @default ['.mustache'] */ readonly extensions?: readonly string[]; } -/** The Mustache engine: partials through the loader, from the root; a missing one fails the render. */ +/** + * The Mustache engine: partials through the loader, `./x` against the + * template it is written in, any other from the root, then the directories + * of partials; a missing one fails the render. + */ export declare function createMustacheEngine(options: MustacheEngineOptions): TemplateEngine; /** A React Email template: its component, and its subject. */ diff --git a/src/templates/mustache.js b/src/templates/mustache.js index 45944ec..d31d823 100644 --- a/src/templates/mustache.js +++ b/src/templates/mustache.js @@ -1,8 +1,9 @@ // Not ported: the Mustache engine of ./templates, over the `mustache` the // application passes in. Each renderer has a writer of its own, whose cache // of parsed templates is bounded, where `mustache.render` shares one without a -// bound for the process; and a partial that is not there fails the render, -// where Mustache renders it as nothing. +// bound for the process; a partial that is not there fails the render, where +// Mustache renders it as nothing; and a partial is looked up from the +// template it is written in, where Mustache tells its partials nothing of it. const { createCache } = require('./cache.js'); const { perIncludes } = require('./environment.js'); @@ -11,6 +12,13 @@ const { checkEngineExtensions, checkModule } = require('./options.js'); // The parsed templates a writer keeps. const MAX_PARSED = 1000; +// The fields of a partial's token in Mustache 4: its name, and the +// indentation of a standalone partial. +const NAME = 1; +const INDENTATION = 4; +const TAG_INDEX = 5; +const LINE_HAS_NON_SPACE = 6; + const checkTags = (tags) => { if (tags === undefined) return undefined; if ( @@ -42,6 +50,18 @@ const createMustacheEngine = (options = {}) => { const extensions = checkEngineExtensions(options.extensions, ['.mustache']); const writerOf = perIncludes((includes) => { const writer = new mustache.Writer(); + // Mustache's renderPartial, but for the partials it is given, which are + // the template they are written in: each partial renders its own with + // the partials of its name. + writer.renderPartial = function renderPartial(token, context, partials, config) { + const found = includes.lookupSync(token[NAME], partials.from, extensions); + let source = found.source; + if (token[TAG_INDEX] === 0 && token[INDENTATION]) { + source = this.indentPartial(source, token[INDENTATION], token[LINE_HAS_NON_SPACE]); + } + const tokens = this.parse(source, this.getConfigTags(config)); + return this.renderTokens(tokens, context, { from: found.name }, source, config); + }; const parsed = createCache(includes.cache ? MAX_PARSED : 0); // Mustache calls set and get; the renderer forgets a writer, not its cache. writer.templateCache = { @@ -57,7 +77,7 @@ const createMustacheEngine = (options = {}) => { const writer = writerOf(context.includes); // Parsed now, so that a syntax error is one of the compile. writer.parse(source, tags); - const partials = (name) => context.includes.lookupSync(name, undefined, extensions).source; + const partials = { from: context.name }; return (scope) => writer.render(source, scope, partials, tags); }, }; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 3f3fd98..d0194da 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -456,6 +456,26 @@ test('Templates: Mustache fails on a missing partial, where it rendered nothing' assert.strictEqual((await uncached.render({ template: 'a' })).html, 'a'); }); +test('Templates: a Mustache partial is read against the template it is written in', async () => { + // Before: Mustache tells its partials nothing of the template they are + // written in, so every name was read from the root. + const engine = createMustacheEngine({ mustache }); + const templates = { + 'emails/a.mustache': '{{> ./parts/b}}', + 'emails/parts/b.mustache': 'b{{> ./c}}{{> ../d}}', + 'emails/parts/c.mustache': 'c', + 'emails/d.mustache': 'd', + 'emails/c.mustache': 'wrong', + }; + assert.strictEqual((await renderWith(engine, templates, { template: 'emails/a' })).html, 'bcd'); + // A standalone partial is indented as Mustache indents it. + const indented = { 'list.mustache': 'list:\n {{> item}}\nend', 'item.mustache': 'one\ntwo\n' }; + assert.strictEqual( + (await renderWith(engine, indented, { template: 'list' })).html, + mustache.render(indented['list.mustache'], {}, { item: indented['item.mustache'] }), + ); +}); + test('Templates: Pug reads include and extends through the loader, its locals the context alone', async () => { const engine = createPugEngine({ pug, options: { doctype: 'html' } }); const templates = { From af9676f21408218c3df5c1bfb4410cad7bd01c05 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:01:57 +0300 Subject: [PATCH 45/83] perf(queue): tell the sink the queued events of a batch in one write enqueueMany told the audit sink each job's queued event in its own round trip, one after the other, so a batch of a thousand made a thousand writes to MongoDB before or after its one enqueueMany. AuditSink.appendMany, on the memory, MongoDB (one bulkWrite, unordered, in the transaction given) and combined sinks, takes a batch, and the queue uses it for two events or more; a sink without it is told one event at a time, as before. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/guide/audit.md | 4 ++- index.d.ts | 2 ++ src/audit/combine.js | 32 +++++++++++------ src/audit/index.d.ts | 1 + src/audit/memory-sink.js | 16 +++++---- src/mongodb/audit-sink.js | 23 ++++++++++++ src/mongodb/index.d.ts | 9 +++++ src/queue/mail-queue.js | 43 ++++++++++++++-------- tests/helpers/mongodb/fake-collection.js | 11 ++++++ tests/unit/audit.test.js | 13 +++++++ tests/unit/mongodb.test.js | 29 +++++++++++++++ tests/unit/queue.test.js | 46 ++++++++++++++++++++++++ 13 files changed, 199 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db31cfd..04cd50e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. not take every worker down in turn for ever. `redelivered: 'delivery_unknown'` makes a run again of an attempt whose run before died `delivery_unknown` rather than a second send, unless the audit sink tells (`has`, new on the memory, MongoDB and combined sinks) that it died before - its submit. The fields of a job of version 1 are fixed: + its submit. `AuditSink.appendMany` (memory, MongoDB in one `bulkWrite`, combined) writes the + `queued` events of an `enqueueMany` in one call. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`, given back with its attempts not spent, for a week or until it expires) rather than fail its hash during a rolling deploy. diff --git a/docs/guide/audit.md b/docs/guide/audit.md index e1aec85..f7437ae 100644 --- a/docs/guide/audit.md +++ b/docs/guide/audit.md @@ -43,7 +43,9 @@ comes after a `delivered`. `isFinalState` is true for `delivered`, `bounced`, An event holds no recipient: the message is found by its `correlationId`. Its `id` is the same however often it is told, and a sink that can tell whether it holds one (`has(id)`: the memory and MongoDB sinks, and combined sinks when one -of them can) lets a worker know a run that submitted before it died. +of them can) lets a worker know a run that submitted before it died. `appendMany(events)` +writes a batch in one call where a sink can (the MongoDB sink in one +`bulkWrite`); `enqueueMany` tells the sink its batch so. ## Sinks diff --git a/index.d.ts b/index.d.ts index 44b0491..18f978c 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1465,6 +1465,8 @@ export interface AuditContext { /** Where audit events go: append-only, an event written once by its `id`. */ export interface AuditSink { append(event: AuditEvent, context?: AuditContext): void | Promise; + /** Appends several events in one write where it can (a bulk write): a queue's batch. */ + appendMany?(events: readonly AuditEvent[], context?: AuditContext): void | Promise; /** * Whether an event of this id was written: lets a worker tell a run that * submitted its message before it died (the ids are deterministic). diff --git a/src/audit/combine.js b/src/audit/combine.js index dcc2655..2e029af 100644 --- a/src/audit/combine.js +++ b/src/audit/combine.js @@ -17,19 +17,29 @@ const combineAuditSinks = (...sinks) => { // `has`, when a sink can tell: an event any of them holds was written. let readable = false; for (const sink of sinks) readable ||= typeof sink.has === 'function'; + // Tells each sink, whatever another threw, then throws what they threw. + const tellEach = async (tell) => { + const errors = []; + for (const sink of sinks) { + try { + await tell(sink); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, 'Audit sinks failed.'); + }; const combined = { - async append(event, context) { - const errors = []; - for (const sink of sinks) { - try { - await sink.append(event, context); - } catch (error) { - errors.push(error); + append: (event, context) => tellEach((sink) => sink.append(event, context)), + appendMany: (events, context) => + tellEach(async (sink) => { + if (typeof sink.appendMany === 'function') { + await sink.appendMany(events, context); + return; } - } - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, 'Audit sinks failed.'); - }, + for (const event of events) await sink.append(event, context); + }), }; if (readable) { combined.has = async (id) => { diff --git a/src/audit/index.d.ts b/src/audit/index.d.ts index 85d3327..0c9dc9e 100644 --- a/src/audit/index.d.ts +++ b/src/audit/index.d.ts @@ -6,6 +6,7 @@ import type { AuditEvent, AuditSink, DeliveryEvent, Logger, MessageState } from /** An audit sink in memory, with what it holds. */ export interface MemoryAuditSink extends AuditSink { append(event: AuditEvent): void; + appendMany(events: readonly AuditEvent[]): void; has(id: string): boolean; /** Its events, in the order they came, one per id. */ readonly events: readonly AuditEvent[]; diff --git a/src/audit/memory-sink.js b/src/audit/memory-sink.js index d16fd32..b72fcaf 100644 --- a/src/audit/memory-sink.js +++ b/src/audit/memory-sink.js @@ -23,13 +23,17 @@ const createMemoryAuditSink = (options = {}) => { } return found; }; + const append = (event) => { + if (typeof event?.id !== 'string') throw new TypeError('An audit event has an id.'); + // An event told again is written once, where it was first. + if (events.has(event.id)) return; + events.set(event.id, Object.freeze({ ...event })); + if (events.size > maxEvents) events.delete(events.keys().next().value); + }; return { - append(event) { - if (typeof event?.id !== 'string') throw new TypeError('An audit event has an id.'); - // An event told again is written once, where it was first. - if (events.has(event.id)) return; - events.set(event.id, Object.freeze({ ...event })); - if (events.size > maxEvents) events.delete(events.keys().next().value); + append, + appendMany(batch) { + for (const event of batch) append(event); }, has: (id) => events.has(id), get events() { diff --git a/src/mongodb/audit-sink.js b/src/mongodb/audit-sink.js index 23ab203..098eb92 100644 --- a/src/mongodb/audit-sink.js +++ b/src/mongodb/audit-sink.js @@ -55,6 +55,29 @@ const createMongoAuditSink = (options = {}) => { { ...sessionOf(context?.transaction), upsert: true }, ); }, + // A batch in one round trip, each event written once by its id. + async appendMany(events, context) { + const operations = []; + for (const event of events) { + if (typeof event?.id !== 'string') throw new TypeError('An audit event has an id.'); + operations.push({ + updateOne: { + filter: { _id: event.id }, + update: { $setOnInsert: documentOfEvent(event) }, + upsert: true, + }, + }); + } + if (operations.length === 0) return; + if (typeof collection.bulkWrite !== 'function') { + for (const event of events) await this.append(event, context); + return; + } + await collection.bulkWrite(operations, { + ...sessionOf(context?.transaction), + ordered: false, + }); + }, eventsOf, stateOf: async (correlationId) => stateOf(await eventsOf(correlationId)), async ensureIndexes() { diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index 0cd5f6c..e259f56 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -44,6 +44,10 @@ export interface MongoCollectionLike { createIndex(spec: any, options?: any): Promise; /** For `changeStream: true`, on a replica set. */ watch?(pipeline?: any[], options?: any): MongoChangeStreamLike; + /** For `fairness: 'tenant'`. */ + distinct?(field: string, filter?: any): Promise; + /** For the batches of the audit sink. */ + bulkWrite?(operations: any[], options?: any): Promise; } /** The options of {@link createMongoOutbox}. */ @@ -127,6 +131,11 @@ export interface MongoAuditSinkOptions { /** An audit sink in a collection. */ export interface MongoAuditSink extends AuditSink { append(event: AuditEvent, context?: { readonly transaction?: unknown }): Promise; + /** A batch in one bulkWrite (one call per event without it). */ + appendMany( + events: readonly AuditEvent[], + context?: { readonly transaction?: unknown }, + ): Promise; has(id: string): Promise; eventsOf(correlationId: string): Promise; stateOf(correlationId: string): Promise; diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index c9dc67e..8352a15 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -55,10 +55,17 @@ const keyReused = (jobIds, answers) => // the job is enqueued, so that a strict sink that fails refuses the enqueue // rather than reports a job enqueued as failed, enqueued again on a retry; // the event is written once by its id, a duplicate's with it. -const appendQueued = async (queue, { data, request }, transaction) => { - if (queue.audit === undefined) return; +const appendQueued = async (queue, jobs, transaction) => { + if (queue.audit === undefined || jobs.length === 0) return; + const events = []; + for (const { data, request } of jobs) events.push(queuedEventOf(data, request)); try { - await queue.audit.append(queuedEventOf(data, request), { transaction }); + // A batch in one write where the sink can, such as one bulkWrite. + if (events.length > 1 && typeof queue.audit.appendMany === 'function') { + await queue.audit.appendMany(events, { transaction }); + } else { + for (const event of events) await queue.audit.append(event, { transaction }); + } } catch (error) { if (queue.auditStrict) throw error; warnThrown('The audit sink of the mail queue', error, 'MAILER_AUDIT_FAILED'); @@ -78,10 +85,9 @@ const transactionOf = (driver, enqueueOptions) => { return transaction; }; -// What an enqueue gives. A first one is told to the sink in the transaction -// of the store when there is one; a job enqueued again is not, and one +// What an enqueue gives: a job enqueued again is a duplicate, and one // enqueued again for another message is refused. -const afterEnqueue = async (queue, prepared, result, transaction) => { +const answerOf = (queue, prepared, result) => { const { data, request } = prepared; const answer = { id: request.id, correlationId: data.correlationId, messageId: data.messageId }; if (result?.duplicate === true) { @@ -91,7 +97,6 @@ const afterEnqueue = async (queue, prepared, result, transaction) => { queue.log.write('debug', () => ({ jobId: request.id }), 'queue.duplicate'); return { ...answer, duplicate: true }; } - if (transaction !== undefined) await appendQueued(queue, prepared, transaction); queue.log.write( 'debug', () => ({ jobId: request.id, priority: request.priority, template: data.template }), @@ -165,18 +170,24 @@ const createMailQueue = (options = {}) => { enqueueOptions.signal?.throwIfAborted(); const transaction = transactionOf(driver, enqueueOptions); const prepared = await prepare(input); - if (transaction === undefined) await appendQueued(queue, prepared, undefined); - const result = await driver.enqueue(prepared.request, enqueueOptions); - return afterEnqueue(queue, prepared, result, transaction); + if (transaction === undefined) await appendQueued(queue, [prepared], undefined); + const answer = answerOf( + queue, + prepared, + await driver.enqueue(prepared.request, enqueueOptions), + ); + // In the transaction of the store, a first enqueue is told with it. + if (transaction !== undefined && answer.duplicate !== true) { + await appendQueued(queue, [prepared], transaction); + } + return answer; }, async enqueueMany(inputs, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); const transaction = transactionOf(driver, enqueueOptions); const prepared = []; for (const input of inputs) prepared.push(await prepare(input)); - if (transaction === undefined) { - for (const job of prepared) await appendQueued(queue, job, undefined); - } + if (transaction === undefined) await appendQueued(queue, prepared, undefined); let results; if (typeof driver.enqueueMany === 'function') { const requests = []; @@ -190,14 +201,18 @@ const createMailQueue = (options = {}) => { // another message refused once all are. const answers = []; const reused = []; + const firsts = []; for (let index = 0; index < prepared.length; index++) { try { - answers.push(await afterEnqueue(queue, prepared[index], results[index], transaction)); + const answer = answerOf(queue, prepared[index], results[index]); + answers.push(answer); + if (answer.duplicate !== true) firsts.push(prepared[index]); } catch (error) { if (error?.code !== 'queue.key_reused') throw error; reused.push(prepared[index].request.id); } } + if (transaction !== undefined) await appendQueued(queue, firsts, transaction); if (reused.length > 0) throw keyReused(reused, answers); return answers; }, diff --git a/tests/helpers/mongodb/fake-collection.js b/tests/helpers/mongodb/fake-collection.js index f62ff83..603fba7 100644 --- a/tests/helpers/mongodb/fake-collection.js +++ b/tests/helpers/mongodb/fake-collection.js @@ -219,6 +219,17 @@ const fakeCollection = (options = {}) => { }; return cursor; }, + async bulkWrite(operations, opts = {}) { + calls.push(['bulkWrite', operations.length, opts]); + let upsertedCount = 0; + for (const { updateOne } of operations) { + const result = await collection.updateOne(updateOne.filter, updateOne.update, { + upsert: updateOne.upsert, + }); + upsertedCount += result.upsertedCount; + } + return { upsertedCount }; + }, async distinct(field, filter = {}) { calls.push(['distinct', field, filter]); const values = []; diff --git a/tests/unit/audit.test.js b/tests/unit/audit.test.js index 059b183..9fa28ee 100644 --- a/tests/unit/audit.test.js +++ b/tests/unit/audit.test.js @@ -241,3 +241,16 @@ test('Audit: a sink tells whether it holds an event, combined ones if any can', const combined = combineAuditSinks(blind, memory); assert.deepStrictEqual([await combined.has('e1'), await combined.has('e2')], [true, false]); }); + +test('Audit: a batch goes to each sink, in one call where it can', async () => { + const memory = createMemoryAuditSink(); + const appended = []; + const single = { append: (event) => appended.push(event.id) }; + const event = (id) => ({ id, state: 'queued', correlationId: id }); + await combineAuditSinks(memory, single).appendMany([event('e1'), event('e2')]); + assert.deepStrictEqual( + memory.events.map((e) => e.id), + ['e1', 'e2'], + ); + assert.deepStrictEqual(appended, ['e1', 'e2']); +}); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 07003a1..e9e4155 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -573,3 +573,32 @@ test('MongoDB: a change stream that fails is opened again, later each time', asy await settle(); assert.strictEqual(collection.streams.length, 0); }); + +test('MongoDB: the audit sink writes a batch in one bulkWrite, each event once', async () => { + const collection = fakeCollection(); + const sink = createMongoAuditSink({ collection }); + const event = (id) => ({ + id, + state: 'queued', + at: '2026-10-18T09:00:00.000Z', + source: 'producer', + correlationId: id, + attempt: 0, + }); + await sink.appendMany([event('e1'), event('e2')], { transaction: 'session' }); + await sink.appendMany([event('e1'), event('e3')]); + await sink.appendMany([]); + assert.deepStrictEqual( + collection.calls.filter((call) => call[0] === 'bulkWrite'), + [ + ['bulkWrite', 2, { session: 'session', ordered: false }], + ['bulkWrite', 2, { ordered: false }], + ], + ); + assert.strictEqual(collection.documents.size, 3); + // A collection without bulkWrite is written one event at a time. + const plain = fakeCollection(); + plain.bulkWrite = undefined; + await createMongoAuditSink({ collection: plain }).appendMany([event('e1'), event('e2')]); + assert.strictEqual(plain.documents.size, 2); +}); diff --git a/tests/unit/queue.test.js b/tests/unit/queue.test.js index 6a2a6ff..247e53e 100644 --- a/tests/unit/queue.test.js +++ b/tests/unit/queue.test.js @@ -326,3 +326,49 @@ test('Queue: without a transaction, a strict sink that fails refuses the enqueue assert.deepStrictEqual(order, ['audit:queued', 'enqueue', 'audit:queued', 'enqueue']); assert.strictEqual(sink.eventsOf(first.correlationId).length, 1); }); + +test('Queue: enqueueMany tells the sink its batch in one write where it can', async () => { + const batches = []; + const sink = createMemoryAuditSink(); + const batching = { + append: (event) => sink.append(event), + appendMany: (events, context) => { + batches.push([events.length, context.transaction]); + sink.appendMany(events); + }, + }; + const { queue } = setup({ audit: batching }); + const answers = await queue.enqueueMany([ + resetJob({ idempotencyKey: 'a' }), + resetJob({ idempotencyKey: 'b' }), + resetJob({ idempotencyKey: 'c' }), + ]); + assert.deepStrictEqual(batches, [[3, undefined]]); + assert.strictEqual(sink.stateOf(answers[2].correlationId), 'queued'); + // In the transaction of an outbox, the jobs it took are told in it, once. + const told = []; + const transactional = { + id: 'outbox', + transactional: true, + enqueue: async (request) => ({ id: request.id, duplicate: false }), + enqueueMany: async (requests) => + requests.map((request, index) => ({ id: request.id, duplicate: index === 1 })), + consume: () => ({ close: async () => {} }), + }; + await createMailQueue({ + driver: transactional, + audit: { + append: () => {}, + appendMany: (events, context) => told.push([events.length, context.transaction]), + }, + }).enqueueMany( + [ + resetJob({ idempotencyKey: 'x' }), + resetJob({ idempotencyKey: 'y' }), + resetJob({ idempotencyKey: 'z' }), + ], + { transaction: 'session' }, + ); + // The duplicate is not told again. + assert.deepStrictEqual(told, [[2, 'session']]); +}); From f3ec9fc2cfe94b8aaae33d296c819d0e0e8f4482 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:02:21 +0300 Subject: [PATCH 46/83] test(templates): read one tree of includes through every engine The rule of includes was written per engine and kept by none alike: EJS and Mustache read a relative include against the template rendered, Pug read a bare name against the file and added .pug whatever the extensions. One rule is in the guide, and one tree of includes, bare names, /x, ./x and ../x of a partial of a partial and a missing include, is rendered by the eight engines that have includes, MJML through the one inside it. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 11 +++--- tests/unit/templates-engines.test.js | 52 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 2e8e94c..53dc2f1 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -115,14 +115,17 @@ its own. Engines that have layouts of their own (Pug's `extends`, Nunjucks's and Liquid's blocks, Handlebars partial blocks, Eta's `layout()`) keep them. Every include, partial and layout an engine reads goes through the loader of -the renderer: +the renderer, by one rule, the same in every engine: - `/x` is read from the root of the loader; -- `./x` and `../x` against the template it is written in; +- `./x` and `../x` against the template it is written in, a partial of a + partial included, where EJS, Mustache and Pug would tell otherwise; - any other name from the root, then from each directory of `partials`; -- the extensions of the engine are added when the name has none. +- the extensions of the engine are tried when the name has none of them, then + the name as it is. -A name that leads out of the root is not found. +A name that leads out of the root is not found. React Email has no includes: +its components import each other. MJML's are those of the engine inside it. ## Engines diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index d0194da..970ed1d 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -957,6 +957,58 @@ test('Templates: React Email components are templates, each with its subject', a ); }); +test('Templates: every engine reads the same tree of includes by one rule', async () => { + // A bare name from the root, then the partials; /x from the root; ./x and + // ../x against the template written in, a partial of a partial included; + // the extensions of the engine added. React Email has no includes: its + // components import each other; MJML's are those of the engine inside it. + const hbs = createHandlebarsEngine({ handlebars }); + const cases = [ + [createEjsEngine({ ejs }), '.ejs', (name) => `<%- include("${name}") %>`], + [hbs, '.hbs', (name) => `{{> "${name}"}}`], + [ + createMjmlEngine({ mjml2html: (mjml) => ({ html: mjml, errors: [] }), engine: hbs }), + '.mjml.hbs', + (name) => `{{> "${name}"}}`, + '.hbs', + ], + [ + createPugEngine({ pug }), + '.pug', + (name) => `include ${name}\n`, + '.pug', + (text) => `| ${text}\n`, + ], + [createLiquidEngine({ Liquid }), '.liquid', (name) => `{% render "${name}" %}`], + [createNunjucksEngine({ nunjucks }), '.njk', (name) => `{% include "${name}" %}`], + [createEtaEngine({ Eta }), '.eta', (name) => `<%~ include("${name}") %>`], + [createMustacheEngine({ mustache }), '.mustache', (name) => `{{> ${name}}}`], + ]; + for (const [engine, main, include, extension = main, text = (value) => value] of cases) { + const templates = { + [`emails/welcome${main}`]: include('header') + include('/legal') + include('./parts/sign'), + [`partials/header${extension}`]: text('H'), + [`legal${extension}`]: text('L'), + [`emails/parts/sign${extension}`]: text('S') + include('./name') + include('../footer'), + [`emails/parts/name${extension}`]: text('N'), + [`emails/footer${extension}`]: text('F'), + [`emails/name${extension}`]: text('wrong'), + [`name${extension}`]: text('wrong'), + [`x/broken${main}`]: include('./parts/nope'), + }; + const renderer = createTemplateRenderer({ + engine, + loader: createMemoryLoader(templates), + partials: ['partials'], + }); + const output = await renderer.render({ template: `emails/welcome${main}` }); + assert.strictEqual(output.html.replace(/\s+/g, ''), 'HLSNF', engine.name); + const missing = await failureOf(renderer.render({ template: `x/broken${main}` })); + assert.ok(missing instanceof TemplateNotFoundError, engine.name); + assert.match(missing.template, /nope/, engine.name); + } +}); + test('Templates: juice and css-inline inline the CSS and fetch nothing', async () => { const html = '

    x

    '; From ba29da3ced5e6bb9e5b867cbceb539f9a66ee894 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:03:06 +0300 Subject: [PATCH 47/83] perf(bullmq): read the jobs of a batch back many at once With detectDuplicates: 'bulk', enqueueMany read each job back after its addBulk one round trip after the other, so a batch of a thousand waited for a thousand reads in turn. The reads go 32 at once, the results in the order of the requests. Co-Authored-By: Claude Opus 5.5 --- docs/transports/bullmq.md | 2 +- src/bullmq/bullmq-driver.js | 17 ++++++++++++++++- tests/unit/bullmq.test.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index 51a14fe..a0c7922 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -32,7 +32,7 @@ A process that only enqueues passes `{ Queue }`, or a queue of its own | `removeOnComplete`, `removeOnFail` | `dedupeWindowSeconds` (a day) and `retainFailedSeconds` (a week), by time, as on every driver (`jobOptions` wins) | A duplicate is told by reading the job back (`detectDuplicates`, one read per -enqueue; `'bulk'` for `enqueueMany` too). It holds while BullMQ keeps the +enqueue; `'bulk'` for `enqueueMany` too, 32 reads at once). It holds while BullMQ keeps the job, so the retention is by time: one by count shrinks the window of duplicates as the queue gets busy. `deduplication` of BullMQ is not used, as it holds only while the first job is not finished. A completed job keeps only diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index 4188481..123cf5b 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -35,6 +35,8 @@ const RESERVED_WORKER_OPTIONS = new Set(['connection', 'prefix', 'concurrency', // BullMQ's highest priority number, 2^21 - 1. const MAX_PRIORITY = 2_097_151; const DEFAULT_PRIORITIES = Object.freeze({ high: 1, normal: 2, low: 3 }); +// The jobs of a batch read back at once to tell its duplicates. +const DUPLICATE_READS = 32; const checkReserved = (options, reserved, what) => { if (options === undefined) return {}; @@ -259,11 +261,24 @@ const createBullMQDriver = (options = {}) => { if (!request.unique) { results.push({ id: request.id, duplicate: false }); } else if (config.detectDuplicates === 'bulk') { - results.push(await duplicateOf(queue, request)); + results.push(undefined); } else { results.push({ id: request.id, duplicate: undefined }); } } + // The jobs read back, so many at once rather than one round trip each. + for (let start = 0; start < requests.length; start += DUPLICATE_READS) { + const reads = []; + const end = Math.min(start + DUPLICATE_READS, requests.length); + for (let index = start; index < end; index++) { + if (results[index] === undefined) reads.push(duplicateOf(queue, requests[index])); + } + const answers = await Promise.all(reads); + let next = 0; + for (let index = start; index < end; index++) { + if (results[index] === undefined) results[index] = answers[next++]; + } + } return results; }, consume: (handler, consumeOptions = {}) => diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index a7f8306..fa9fc8c 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -495,3 +495,31 @@ test('BullMQ: a job BullMQ gives up itself is told to the audit as of unknown de assert.strictEqual(audit.events.length, 1); await consumer.close(); }); + +test('BullMQ: enqueueMany reads its jobs back many at once to tell duplicates', async () => { + const bullmq = fakeBullMQ(); + const driver = createBullMQDriver({ bullmq, detectDuplicates: 'bulk' }); + const [queue] = bullmq.queues; + let inFlight = 0; + let most = 0; + const getJob = queue.getJob.bind(queue); + queue.getJob = async (id) => { + inFlight++; + most = Math.max(most, inFlight); + await new Promise(setImmediate); + inFlight--; + return getJob(id); + }; + await driver.enqueue(requestOf({ id: 'held' })); + const requests = []; + for (let index = 0; index < 40; index++) requests.push(requestOf({ id: `j${index}` })); + requests.push(requestOf({ id: 'held', enqueuedAt: 2000 })); + requests.push(requestOf({ id: 'plain', unique: false })); + const results = await driver.enqueueMany(requests); + assert.strictEqual(results.length, 42); + assert.deepStrictEqual(results[40], { id: 'held', duplicate: true, existingHash: 'h1' }); + assert.deepStrictEqual(results[41], { id: 'plain', duplicate: false }); + assert.strictEqual(results[0].duplicate, false); + // At most 32 at once, more than one. + assert.ok(most > 1 && most <= 32, String(most)); +}); From 21073786fa7955ec9a462f83a676ad0bfa42e3f2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:04:25 +0300 Subject: [PATCH 48/83] fix(queue): tell what escapes the steps of the worker, and how long each run took An error that escaped the guarded steps of the worker (a classify that threw, a bug of the worker) failed the job with no log and no warning, so its cause showed only as queue.thrown in the store. It is logged (queue.worker.error) and warned (MAILER_QUEUE_WORKER_THREW), its addresses redacted. A failed or retried run is logged with its duration, as a completed one was. Co-Authored-By: Claude Opus 5.5 --- src/queue/handler.js | 9 ++++-- tests/unit/queue-handler.test.js | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/queue/handler.js b/src/queue/handler.js index 02b214c..9dc9d7c 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -173,6 +173,7 @@ const logFailed = (run, code, reason, state) => { code, state, reason, + durationMs: run.config.now() - run.started, }), 'queue.job.failed', ); @@ -218,6 +219,7 @@ const settleFailure = async (run, receipt, code) => { code: outcome.code, delayMilliseconds: outcome.delayMilliseconds, scope: outcome.scope, + durationMs: config.now() - run.started, }), 'queue.job.retrying', ); @@ -597,8 +599,11 @@ const createMailJobHandler = (options) => { return await attempt(run); } catch (error) { // What escaped the steps: a bug of this worker, or a sink or logger - // that threw where it is not guarded. - return { type: 'fail', code: 'queue.thrown', reason: reasonOf(messageOf(error)) }; + // that threw where it is not guarded. Told, not only failed. + const reason = reasonOf(messageOf(error)); + warnThrown('The mail worker', error, 'MAILER_QUEUE_WORKER_THREW'); + config.log.write('error', () => ({ jobId: job.id, reason }), 'queue.worker.error'); + return { type: 'fail', code: 'queue.thrown', reason }; } }; }; diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index fb0a51e..caff3eb 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -656,3 +656,54 @@ test('Queue: a rate limit makes the job’s tenant wait under rateLimitScope ten const alone = await run(handler, jobOf({ idempotencyKey: 'n' })); assert.strictEqual(alone.scope, 'job'); }); + +test('Queue: what escapes the steps is told, and each outcome logged with its duration', async (t) => { + const warnings = []; + const listener = (warning) => warnings.push(warning.code); + process.on('warning', listener); + t.after(() => process.off('warning', listener)); + const records = []; + const logger = { + error: (record, message) => records.push([message, record]), + warn: (record, message) => records.push([message, record]), + info: (record, message) => records.push([message, record]), + }; + let clock = NOW; + const failing = fakeTransport(() => failure('server-error', { retryable: true })); + const { handler: retrying } = handlerOf({ + transport: failing, + logger, + now: () => (clock += 250), + }); + await run(retrying, jobOf()); + const retried = records.find(([message]) => message === 'queue.job.retrying'); + assert.ok(retried[1].durationMs > 0); + const { handler: refusing } = handlerOf({ + transport: fakeTransport(() => failure('rejected')), + logger, + now: () => (clock += 250), + }); + await run(refusing, jobOf({ idempotencyKey: 'r' })); + const failed = records.find(([message]) => message === 'queue.job.failed'); + assert.ok(failed[1].durationMs > 0); + const broken = await run( + createMailJobHandler({ + transport: fakeTransport(() => failure('server-error', { retryable: true })), + renderer: templates, + logger, + now: () => clock, + classify: () => { + throw new Error('classify broke'); + }, + }), + jobOf({ idempotencyKey: 'b' }), + ); + assert.deepStrictEqual([broken.type, broken.code], ['fail', 'queue.thrown']); + assert.ok( + records.some( + ([message, record]) => message === 'queue.worker.error' && record.reason === 'classify broke', + ), + ); + await new Promise(setImmediate); + assert.ok(warnings.includes('MAILER_QUEUE_WORKER_THREW')); +}); From a48dd98fc04161855ea5b8156a0930bb458fa4ec Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:05:08 +0300 Subject: [PATCH 49/83] perf(templates): compile an EJS include once, not on every render The EJS engine compiled with `cache: false`, and its includer gave the source of an include, so EJS compiled every include of a template on every render: 2000 renders of a template with three includes took 125 ms, and take 50 ms now. With the renderer's cache on, EJS keeps a compiled include in ejs.cache by its file name, which is its name in the loader and the number of the renderer's includes, so that renderers share none; the engine keeps as many as cache.maxEntries, forgetting each it drops, and forgets them all at clear(), which calls the new `clear(includes)` of an engine. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 17 ++++++ src/templates/cache.js | 13 ++++- src/templates/ejs.js | 54 ++++++++++++++++-- src/templates/includes.js | 1 + src/templates/index.d.ts | 7 +++ src/templates/mjml.js | 1 + src/templates/renderer.js | 4 +- tests/unit/templates-engines.test.js | 85 ++++++++++++++++++++++++++++ 8 files changed, 174 insertions(+), 8 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 53dc2f1..1dcabe9 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -243,6 +243,23 @@ wait for the one compile. A template that is missing or fails is not kept, so one deployed later is found. `clear()` forgets everything; `cache: false` reads and compiles on every render, for development. +EJS compiles an include as it renders, so it keeps the compiled include in +`ejs.cache`, which the whole process shares, by file name. The engine names +each one by its template and its renderer (`emails/footer.ejs#3`), so that two +renderers share none, keeps as many as `cache.maxEntries`, and forgets them at +`clear()`. `ejs.cache` keeps whatever else the process compiles with +`cache: true` without a bound; give it one with an LRU, as EJS documents: + +```js +const { LRUCache } = require('lru-cache'); +ejs.cache = new LRUCache({ max: 500 }); +``` + +On Node 24 (Apple M3 Max), 2000 renders of a template with three includes, one of +them nested, took 125 ms when every include was compiled on each render, and +50 ms with them kept; EJS still checks the disk for each relative include it +is asked for, which takes most of what is left. + ## Bounds A render is bounded, so that a template, or a context, that runs away fails diff --git a/src/templates/cache.js b/src/templates/cache.js index 60ebd20..2f97ddb 100644 --- a/src/templates/cache.js +++ b/src/templates/cache.js @@ -8,9 +8,11 @@ const { timerMs } = require('../internal/sleep.js'); * Makes a cache of at most `maxEntries` entries. With `maxEntries` 0 it keeps * nothing. * + * @param maxEntries The most it keeps. + * @param onDrop Told the key of each entry the bound drops, if given. * @internal */ -const createCache = (maxEntries) => { +const createCache = (maxEntries, onDrop) => { const entries = new Map(); return { get(key) { @@ -24,12 +26,19 @@ const createCache = (maxEntries) => { if (maxEntries === 0) return; entries.delete(key); entries.set(key, value); - if (entries.size > maxEntries) entries.delete(entries.keys().next().value); + if (entries.size > maxEntries) { + const oldest = entries.keys().next().value; + entries.delete(oldest); + onDrop?.(oldest); + } }, // Forgets an entry, unless another value took its place meanwhile. forget(key, value) { if (entries.get(key) === value) entries.delete(key); }, + keys() { + return entries.keys(); + }, clear() { entries.clear(); }, diff --git a/src/templates/ejs.js b/src/templates/ejs.js index 6d21c7c..6679119 100644 --- a/src/templates/ejs.js +++ b/src/templates/ejs.js @@ -4,6 +4,8 @@ // its options. @nestjs-modules/mailer let an async template's promise reject // unheard, which hung the send. +const { createCache } = require('./cache.js'); +const { perIncludes } = require('./environment.js'); const { checkEngineExtensions, checkEngineOptions, @@ -18,6 +20,26 @@ const UNESCAPED = new Map([ ['escapeFunction', () => true], ]); +// EJS keeps a compiled include in ejs.cache, which the process shares, by +// its file name: with the cache on, that is its name in the loader and, +// after a `#`, which no name holds, the number of the renderer's includes, +// so that two renderers, or one before and after clear(), share none. +const SEPARATOR = '#'; +let scopes = 0; + +const nameOf = (filename) => { + const separator = filename.lastIndexOf(SEPARATOR); + return separator === -1 ? filename : filename.slice(0, separator); +}; + +// Forgets an entry of ejs.cache: `remove` of EJS's own, `delete` of an +// lru-cache put in its place. +const forget = (ejs, key) => { + const { cache } = ejs; + if (typeof cache?.remove === 'function') cache.remove(key); + else if (typeof cache?.delete === 'function') cache.delete(key); +}; + /** * Makes the EJS engine. * @@ -35,6 +57,20 @@ const createEjsEngine = (options = {}) => { const given = checkEngineOptions(options.options, RESERVED, 'ejs'); checkEscaping(options, given, UNESCAPED, 'ejs'); const extensions = checkEngineExtensions(options.extensions, ['.ejs']); + // The file names a renderer's includes gave ejs.cache, as many as its + // cache keeps: one dropped is forgotten by ejs.cache too. + const scopeOf = perIncludes((includes) => { + const suffix = `${SEPARATOR}${++scopes}`; + const keys = createCache(includes.maxEntries, (key) => forget(ejs, key)); + return { suffix, keys }; + }); + const fileOf = (includes, name) => { + if (!includes.cache) return name; + const scope = scopeOf(includes); + const key = name + scope.suffix; + scope.keys.set(key, true); + return key; + }; return { name: 'ejs', extensions, @@ -42,21 +78,29 @@ const createEjsEngine = (options = {}) => { const { includes } = context; const hooks = { // EJS calls the includer as a method of the options of the template - // the include is written in, whose file name is its name in the + // the include is written in, whose file name holds its name in the // loader: a relative include is read against it. includer(include) { - const found = includes.lookupSync(include, this.filename, extensions); - return { filename: found.name, template: found.source }; + const found = includes.lookupSync(include, nameOf(this.filename), extensions); + return { filename: fileOf(includes, found.name), template: found.source }; }, }; + // With the cache on, EJS compiles an include once, not on every render. return ejs.compile(source, { ...given, - filename: context.name, + filename: fileOf(includes, context.name), includer: hooks.includer, - cache: false, + cache: includes.cache, client: false, }); }, + // The renderer's clear(): what its includes gave ejs.cache is forgotten. + clear(includes) { + if (!includes.cache) return; + const scope = scopeOf(includes); + for (const key of scope.keys.keys()) forget(ejs, key); + scope.keys.clear(); + }, }; }; diff --git a/src/templates/includes.js b/src/templates/includes.js index 5eac7af..e67f35f 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -209,6 +209,7 @@ const createIncludes = (loader, options) => { const includes = Object.freeze({ partials, cache: options.cache, + maxEntries: options.maxEntries, join: joinName, read, readSync, diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index d4c1ea0..0b10f28 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -122,6 +122,8 @@ export interface TemplateIncludes { readonly partials: readonly string[]; /** Whether what is read and compiled may be kept. */ readonly cache: boolean; + /** The most entries the renderer keeps in each of its caches, for an engine to keep as many. */ + readonly maxEntries: number; /** * The name `include` stands for in the template `from`: `./x` and `../x` * against its directory, `/x` from the root, any other as it is; undefined @@ -191,6 +193,11 @@ export interface TemplateEngine { finish?(html: string, context: TemplateCompileContext): string | Promise; /** The templates of an engine whose templates are code (React Email), when the renderer has no loader. */ readonly loader?: TemplateLoader; + /** + * Forgets what the engine keeps for these includes outside them, such as + * EJS's entries in `ejs.cache`: called by the renderer's `clear()`. + */ + clear?(includes: TemplateIncludes): void; } /** Inlines the CSS of an HTML body. */ diff --git a/src/templates/mjml.js b/src/templates/mjml.js index a950bcb..2e278ab 100644 --- a/src/templates/mjml.js +++ b/src/templates/mjml.js @@ -91,6 +91,7 @@ const createMjmlEngine = (options = {}) => { return () => html; }, ...(inner === undefined ? {} : { finish: toHtml }), + ...(typeof inner?.clear === 'function' ? { clear: (of) => inner.clear(of) } : {}), }; }; diff --git a/src/templates/renderer.js b/src/templates/renderer.js index de9cd57..7181395 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -311,7 +311,9 @@ const createTemplateRenderer = (options = {}) => { } }, clear() { - // Engines key what they keep by the includes, so they make it anew too. + // Engines key what they keep by the includes, so they make it anew too; + // one that keeps it elsewhere, EJS in ejs.cache, forgets it. + for (const engine of config.engines) engine.clear?.(state.files.includes); state.files = makeIncludes(); compiled.clear(); resolved.clear(); diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 970ed1d..d995a59 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -118,6 +118,91 @@ test('Templates: an EJS include is read against the template it is written in', ); }); +test('Templates: an EJS include is compiled once, in ejs.cache, for its renderer alone', async (t) => { + // Before: \`cache: false\`, so EJS compiled every include of a template + // on every render. + const kept = new Map(); + const original = ejs.cache; + ejs.cache = { + get: (key) => kept.get(key), + set: (key, value) => kept.set(key, value), + remove: (key) => kept.delete(key), + reset: () => kept.clear(), + }; + t.after(() => { + ejs.cache = original; + }); + const compile = t.mock.method(ejs, 'compile'); + const engine = createEjsEngine({ ejs }); + const renderer = createTemplateRenderer({ + engine, + loader: createMemoryLoader({ + 'a.ejs': '<%- include("b") %><%- include("./c") %>', + 'b.ejs': 'b', + 'c.ejs': 'c<%= n %>', + }), + }); + for (let n = 0; n < 3; n++) { + assert.strictEqual((await renderer.render({ template: 'a', context: { n } })).html, `bc${n}`); + } + assert.strictEqual(compile.mock.callCount(), 3, 'a, b and c, once each'); + assert.strictEqual(kept.size, 2); + // Another renderer, its own templates of the same names, its own entries. + const other = createTemplateRenderer({ + engine, + loader: createMemoryLoader({ 'a.ejs': '<%- include("b") %>', 'b.ejs': 'other b' }), + }); + assert.strictEqual((await other.render({ template: 'a' })).html, 'other b'); + assert.strictEqual(kept.size, 3); + // clear() forgets what the renderer gave ejs.cache, and compiles anew. + renderer.clear(); + assert.strictEqual(kept.size, 1); + await renderer.render({ template: 'a', context: { n: 0 } }); + assert.strictEqual( + compile.mock.callCount(), + 8, + "the other renderer's two, and these three again", + ); + // As many as the cache of the renderer keeps; an lru-cache deletes. + const lru = new Map(); + ejs.cache = { + get: (key) => lru.get(key), + set: (key, value) => lru.set(key, value), + delete: (key) => lru.delete(key), + }; + const small = createTemplateRenderer({ + engine, + loader: createMemoryLoader({ + 'a.ejs': '<%- include("b") %><%- include("c") %>', + 'b.ejs': 'b', + 'c.ejs': 'c', + }), + cache: { maxEntries: 1 }, + }); + await small.render({ template: 'a' }); + assert.ok(lru.size <= 1); + small.clear(); + assert.strictEqual(lru.size, 0); + // Without a cache, EJS keeps nothing. + const uncached = createTemplateRenderer({ + engine: createMjmlEngine({ mjml2html: (html) => ({ html, errors: [] }), engine }), + loader: createMemoryLoader({ 'a.mjml.ejs': '<%- include("b") %>', 'b.ejs': 'b' }), + cache: false, + }); + assert.strictEqual((await uncached.render({ template: 'a' })).html, 'b'); + uncached.clear(); + assert.strictEqual(lru.size, 0); + // MJML clears the engine inside it. + const mjml = createTemplateRenderer({ + engine: createMjmlEngine({ mjml2html: (html) => ({ html, errors: [] }), engine }), + loader: createMemoryLoader({ 'a.mjml.ejs': '<%- include("b") %>', 'b.ejs': 'b' }), + }); + await mjml.render({ template: 'a' }); + assert.strictEqual(lru.size, 1); + mjml.clear(); + assert.strictEqual(lru.size, 0); +}); + test('Templates: an async EJS template that throws rejects, rather than hanging the send', async () => { // Before, in @nestjs-modules/mailer: `rendered.then(render)` had no catch. const engine = createEjsEngine({ ejs, options: { async: true } }); From a36ab86a0dc62fe16e406aa1a1d22a8e86fab9cb Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:06:42 +0300 Subject: [PATCH 50/83] feat(templates): name the locales templates are written in A locale a user chose was looked up with each of its subtags, and kept an entry of the resolved cache of its own, so made-up tags read the loader and pushed templates out of the cache. A renderer takes `locales`: a locale asked for is looked up as the first of them its subtags reach, and any other as none, keyed so in the cache. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 5 ++++ src/templates/index.d.ts | 7 +++++ src/templates/renderer.js | 25 ++++++++++++++++- tests/types/templates.test-d.ts | 1 + tests/unit/templates-renderer.test.js | 40 +++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 1dcabe9..feb421c 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -87,6 +87,11 @@ are taken at the locale the HTML was found at, so that a Ukrainian body never goes with an English subject. `variantName(template, { locale, part })` names the variants another way. +When the locale comes from a user, name the locales your templates are written +in: with `locales: ['en', 'uk']`, `uk-UA` is looked up as `uk` and any other +locale as none, so that a made-up tag reads no template and keeps no entry of +the cache of its own. + A name with the extension of an engine (`welcome.hbs`) is that engine's alone; without one, the engines are tried in their order. diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 0b10f28..621dd3a 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -251,6 +251,13 @@ export interface TemplateRendererOptions { ) => TemplateContext | undefined | Promise); /** The locale tried after the requested ones, before the templates without one. */ readonly defaultLocale?: string; + /** + * The locales templates are written in. A locale asked for is then looked + * up as the first of them its subtags reach (`uk-UA` as `uk`), and any + * other as none, so that a locale a user chose reads no template and keeps + * no entry of the cache of its own. Any by default. + */ + readonly locales?: readonly string[]; /** The name of a variant before its extension. @default `${template}[.${locale}][.text|.subject]` */ readonly variantName?: ( template: string, diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 7181395..0e678a6 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -106,6 +106,28 @@ const checkLocale = (locale, what) => { return Intl.getCanonicalLocales(locale)[0]; }; +// The locales templates are written in, when the application names them: a +// locale asked for is looked up as the first of them its subtags reach, so +// that a locale a user chose keeps no entry of its own in the cache. +const checkLocales = (locales) => { + if (locales === undefined) return undefined; + if (!Array.isArray(locales)) throw new TypeError('locales must be an array of BCP 47 tags.'); + const set = new Set(); + for (const locale of locales) set.add(checkLocale(locale, 'A locale of locales')); + return set; +}; + +const allowedLocale = (config, locale) => { + if (config.locales === undefined || locale === undefined) return locale; + let current = locale; + for (;;) { + if (config.locales.has(current)) return current; + const dash = current.lastIndexOf('-'); + if (dash === -1) return undefined; + current = current.slice(0, dash); + } +}; + const createRendererConfig = (options) => { const { engines, suffixes } = enginesOf(options); const loader = options.loader ?? (engines.length === 1 ? engines[0].loader : undefined); @@ -141,6 +163,7 @@ const createRendererConfig = (options) => { layout: options.layout, globals, defaultLocale: checkLocale(options.defaultLocale, 'defaultLocale'), + locales: checkLocales(options.locales), variantName: checkOptionalFunction(options.variantName, 'variantName') ?? defaultVariantName, frontMatter: options.frontMatter ?? true, inlineCss: checkOptionalFunction(options.inlineCss, 'inlineCss'), @@ -504,7 +527,7 @@ const renderTemplate = async (state, checked, input, bounds) => { const { config } = state; let resolution; try { - resolution = await resolvedFor(state, checked.template, checked.locale); + resolution = await resolvedFor(state, checked.template, allowedLocale(config, checked.locale)); } catch (error) { throw inPhase(error, 'resolve'); } diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index 9375cb0..62ba193 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -89,6 +89,7 @@ const templates = createTemplateRenderer({ layout: 'layouts/main', globals: (input) => ({ locale: input.locale }), defaultLocale: 'en', + locales: ['en', 'uk'], inlineCss: cssInline, cache: { maxEntries: 100 }, maxIncludes: 100, diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index 1b86550..ab9ed0c 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -110,6 +110,46 @@ test('Templates: locales fall back by subtag, then the default, then none', asyn await assert.rejects(plain.render({ template: 'welcome', locale: 'not a tag' }), RangeError); }); +test('Templates: with locales, a locale asked for is one of them or none', async () => { + // Before: every locale a user chose was looked up, and kept an entry of the + // cache of its own, so made-up tags pushed the templates out of it. + const reads = []; + const loader = createMemoryLoader({ + 'welcome.tpl': '

    Hi

    ', + 'welcome.uk.tpl': '

    Привіт

    ', + }); + const counted = { + load: (name) => { + reads.push(name); + return loader.load(name); + }, + }; + const renderer = createTemplateRenderer({ + engine: fakeEngine(), + loader: counted, + locales: ['uk', 'EN'], + defaultLocale: 'en', + }); + const uk = await renderer.render({ template: 'welcome', locale: 'uk-UA-x-anything' }); + assert.deepStrictEqual([uk.template, uk.locale], ['welcome.uk.tpl', 'uk']); + const before = reads.length; + for (let index = 0; index < 20; index++) { + const other = await renderer.render({ template: 'welcome', locale: `de-x-tag${index}` }); + assert.deepStrictEqual([other.template, other.locale], ['welcome.tpl', undefined]); + } + // The twenty made-up tags were one lookup: the template without a locale. + assert.ok(reads.length - before <= 4, String(reads.length - before)); + assert.ok(!reads.some((name) => name.includes('tag'))); + assert.throws( + () => createTemplateRenderer({ engine: fakeEngine(), loader, locales: 'uk' }), + /locales must be an array/, + ); + assert.throws( + () => createTemplateRenderer({ engine: fakeEngine(), loader, locales: ['not a tag'] }), + RangeError, + ); +}); + test('Templates: a missing template names what was tried, and is not retryable', async () => { const { renderer } = rendererOf({}, { defaultLocale: 'en' }); const error = await renderer.render({ template: 'nope', locale: 'uk' }).catch((e) => e); From 8620476eb599065720e162efbaab831445784301 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:09:01 +0300 Subject: [PATCH 51/83] feat(queue): read what the outbox and the memory queue hold The outbox had get(id) alone, so its backlog, the lag of its workers and its failed jobs could not be seen, and a failed job went unseen until its retention dropped it. stats() gives the jobs by state (ready, delayed, active, completed, failed), how long the oldest job due has waited and the pauses in force; failed({ limit, after }) lists the failed jobs a page at a time by id; retryFailed(id) sends one again with its attempts anew, and refuses one whose data retainData dropped. The memory queue has the same; for BullMQ the docs point to its own counts and Bull Board. The checks and readers of the outbox move out of createMongoOutbox, past 80 lines. ./queue goes from 18.3 to 18.8 KB min+gzip and ./mongodb from 6.6 to 7.2: their budgets go from 18.5 to 19 and from 7 to 7.5 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- docs/guide/outbox.md | 7 +- docs/guide/queue.md | 3 +- docs/transports/bullmq.md | 4 + scripts/size-budgets.json | 4 +- src/mongodb/index.d.ts | 17 ++- src/mongodb/outbox.js | 158 ++++++++++++++++++----- src/queue/index.d.ts | 26 ++++ src/queue/memory-queue.js | 92 ++++++++++++- tests/helpers/mongodb/fake-collection.js | 13 +- tests/mongodb/outbox.mongodb.test.js | 20 +++ tests/types/mongodb.test-d.ts | 2 + tests/types/queue.test-d.ts | 4 + tests/unit/mongodb.test.js | 38 ++++++ tests/unit/queue-memory.test.js | 42 ++++++ 15 files changed, 391 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04cd50e..e67be69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,7 +87,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. one lane per change after which a job may be ready, and is opened again when it fails. `fairness: 'tenant'` (memory queue, outbox) takes a high job of any tenant first, then the tenants in turn, so one tenant's backlog holds no other's mail; `rateLimitScope: 'tenant'` pauses - one tenant's jobs alone, and `resume()` ends a pause early. + one tenant's jobs alone, and `resume()` ends a pause early. `stats()` (jobs by state, the age of + the oldest job due, the pauses), `failed({ limit, after })` and `retryFailed(id)` read the outbox + and the memory queue, so a failed job does not go unseen until its retention drops it. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index 2092e7b..bac6ccc 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -129,7 +129,12 @@ its backups. } ``` -`outbox.get(id)` reads one. The data of a job is stored as it is, a token of a +`outbox.get(id)` reads one. `outbox.stats()` gives its jobs by state (`ready`, +`delayed`, `active`, `completed`, `failed`), how long the oldest job due has +waited (`oldestReadyMilliseconds`, the lag of the workers, to alert on) and its +pauses; `outbox.failed({ limit, after })` lists the failed jobs a page at a +time, and `outbox.retryFailed(id)` sends one again, its attempts anew, before +`retainFailedSeconds` drops it. The data of a job is stored as it is, a token of a reset link among it, unless the queue seals it with a [codec](/guide/queue#sealed-jobs): restrict who reads the collection. diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 652df87..2bd072f 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -261,7 +261,8 @@ per process unless given a shared store (see ## The memory queue and the polling driver `createMemoryQueue()` is a queue in memory for tests and a single process, -with `list()`, `get(id)`, `drain()` and `wake()`, and `now` and `sleep` for a +with `list()`, `get(id)`, `drain()`, `wake()`, and `stats()`, `failed()` and +`retryFailed(id)` as the outbox has them, and `now` and `sleep` for a clock of the tests. It is the polling driver over a source in memory: `createPollingDriver(source)` makes a driver of any `QueueSource`, a store that lends its jobs (`reserve`, `extend`, `ack`, `retry`, `fail`, `release`), such diff --git a/docs/transports/bullmq.md b/docs/transports/bullmq.md index a0c7922..516179b 100644 --- a/docs/transports/bullmq.md +++ b/docs/transports/bullmq.md @@ -54,6 +54,10 @@ A job BullMQ fails itself, which the handler never settled (stalled more than sink as `delivery_unknown` (code `queue.lost`): its run may have sent it before it stalled. +Its jobs by state, the failed ones and a retry of one by hand are BullMQ's own +(`queue.getJobCounts()`, `queue.getFailed()`, `job.retry()`), and a dashboard +such as Bull Board reads them. + ## The workers it makes `queue.createWorker()` makes a BullMQ `Worker` with your `workerOptions` diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 090eb59..c09f50b 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 18.5, + "./queue": 19, "./bullmq": 4.5, - "./mongodb": 7 + "./mongodb": 7.5 } diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index e259f56..d56e9ca 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -13,7 +13,7 @@ import type { QueueSource, } from '../../index.js'; import type { IdempotencyStore } from '../idempotency/index.js'; -import type { PollingDriverOptions } from '../queue/index.js'; +import type { MailQueueStats, PollingDriverOptions } from '../queue/index.js'; /** What a write of a collection gives. */ export interface MongoUpdateResultLike { @@ -48,6 +48,8 @@ export interface MongoCollectionLike { distinct?(field: string, filter?: any): Promise; /** For the batches of the audit sink. */ bulkWrite?(operations: any[], options?: any): Promise; + /** For `stats()`. */ + countDocuments?(filter?: any): Promise; } /** The options of {@link createMongoOutbox}. */ @@ -101,6 +103,19 @@ export interface MongoOutbox extends QueueDriver { get(id: string): Promise; /** Ends a pause of the queue, or of one tenant's jobs, before its time. */ resume(scope?: { readonly tenantId?: string }): Promise; + /** Its jobs by state, the lag of its workers and its pauses: for a dashboard or an alert. */ + stats(): Promise; + /** The failed jobs, by id, a page of `limit` (100) after `after`. */ + failed(options?: { + readonly limit?: number; + readonly after?: string; + }): Promise; + /** + * A failed job back to ready, its attempts anew: false when it is not a + * failed job. + * @throws {TypeError} If its data was dropped (`retainData`). + */ + retryFailed(id: string): Promise; } /** diff --git a/src/mongodb/outbox.js b/src/mongodb/outbox.js index 32d9f64..76aad1e 100644 --- a/src/mongodb/outbox.js +++ b/src/mongodb/outbox.js @@ -9,7 +9,7 @@ const { randomUUID } = require('node:crypto'); const { createPollingDriver } = require('../internal/polling-driver.js'); const { createLog } = require('../internal/log.js'); -const { checkOptionalFunction } = require('../internal/options.js'); +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { sleep } = require('../internal/sleep.js'); const { checkNonNegativeInteger, @@ -74,6 +74,44 @@ const jobOf = (document) => { return job; }; +// A job as the outbox holds it, for its readers. +const viewOf = (document) => ({ + ...jobOf(document), + state: document.state, + priority: document.priority, + failure: document.failure, + result: document.result, +}); + +// What the outbox holds: its jobs by state, how long the oldest job due has +// waited, and its pauses. +const statsOf = async (collection, source, nowMs) => { + const now = new Date(nowMs); + const counts = {}; + const filters = { + ready: { state: 'ready', availableAt: { $lte: now } }, + delayed: { state: 'ready', availableAt: { $gt: now } }, + active: { state: 'active' }, + completed: { state: 'completed' }, + failed: { state: 'failed' }, + }; + for (const state of Object.keys(filters)) { + counts[state] = await collection.countDocuments(filters[state]); + } + const oldest = await collection.findOne(filters.ready, { + sort: { availableAt: 1 }, + projection: { availableAt: 1 }, + }); + const stats = { ...counts }; + if (oldest?.availableAt instanceof Date) { + stats.oldestReadyMilliseconds = nowMs - oldest.availableAt.getTime(); + } + const pauses = await source.pausesAt(nowMs); + if (pauses.until > nowMs) stats.pausedUntil = pauses.until; + if (pauses.tenants.length > 0) stats.pausedTenants = pauses.tenants; + return stats; +}; + // The filter of a write under a lease: the job is still lent to it. const leased = (lease) => ({ _id: lease.job.id, token: lease.token, state: 'active' }); @@ -316,6 +354,17 @@ class MongoOutboxSource { } } + // The pauses in force, read now: the queue's, and the tenants'. + async pausesAt(now) { + this.#pauses.readAt = -Infinity; + const pauses = await this.#readPauses(now); + const tenants = []; + for (const [tenantId, until] of pauses.tenants) { + if (until > now) tenants.push({ tenantId, until }); + } + return { until: pauses.until, tenants }; + } + async resume(scope = {}) { const { tenantId } = scope; await this.#collection.deleteOne({ _id: pauseIdOf(tenantId) }); @@ -398,22 +447,8 @@ const subscribe = (collection, listener, config) => { }; }; -/** - * Makes a mail queue in a collection of MongoDB. - * - * @param options `collection`, `db.collection('mail_outbox')`; - * `changeStream` (false), to wake the workers on a change (a replica set); - * `retainData` ('failed'), the finished jobs whose data is kept; `fairness` - * ('none'), or 'tenant' for the tenants' jobs in turn; - * `dedupeWindowSeconds` (a day), the time a completed job is kept, and so - * the window of its duplicates; `retainFailedSeconds` (a week), that of a - * failed one; `clockSkewToleranceMilliseconds` (5000), how - * far the clocks of the workers may be apart; `now`, `logger`, and the - * options of the polling driver. - * @returns The driver, with `source`, `ensureIndexes()` and `get(id)`. - * @throws {TypeError} If an option is invalid. - */ -const createMongoOutbox = (options = {}) => { +// The options of an outbox, checked. +const createOutboxConfig = (options) => { const fairness = options.fairness ?? 'none'; if (!FAIRNESS.has(fairness)) throw new TypeError("fairness must be 'none' or 'tenant'."); const collection = checkCollection(options.collection, 'createMongoOutbox'); @@ -435,7 +470,8 @@ const createMongoOutbox = (options = {}) => { 'leaseMilliseconds must be more than 3 times clockSkewToleranceMilliseconds.', ); } - const config = { + return { + collection, now: checkOptionalFunction(options.now, 'now') ?? Date.now, sleep: checkOptionalFunction(options.sleep, 'sleep') ?? sleep, // The polling driver checks it; the pause is read again as often. @@ -449,6 +485,77 @@ const createMongoOutbox = (options = {}) => { ...retentionOf(options), log: createLog(options.logger, { component: 'queue', queue: 'mongodb' }), }; +}; + +// What the outbox holds, for its readers: a job, the counts, the failed jobs +// and a retry of one by hand. +const readersOf = (collection, source, config) => ({ + async get(id) { + const document = await collection.findOne({ _id: id }); + // The pauses of the queue are no jobs. + if (document === null || document === undefined || id.startsWith(PAUSE_ID)) return undefined; + return viewOf(document); + }, + stats: () => statsOf(collection, source, config.now()), + async failed(failedOptions = {}) { + const limit = checkPositiveInteger(failedOptions.limit ?? 100, 'limit'); + const filter = + failedOptions.after === undefined + ? { state: 'failed' } + : { state: 'failed', _id: { $gt: failedOptions.after } }; + const documents = await collection + // oxlint-disable-next-line no-restricted-properties -- a MongoDB collection, not an array + .find(filter) + .sort({ _id: 1 }) + .limit(limit) + .toArray(); + const jobs = []; + for (const document of documents) jobs.push(viewOf(document)); + return jobs; + }, + async retryFailed(id) { + const held = await collection.findOne({ _id: id }, { projection: { state: 1, data: 1 } }); + if (held?.state !== 'failed') return false; + if (held.data === undefined) { + throw new TypeError(`The job ${id} keeps no data to send again: retainData dropped it.`); + } + const now = new Date(config.now()); + const result = await collection.updateOne( + { _id: id, state: 'failed' }, + { + $set: { + state: 'ready', + availableAt: now, + attempts: 0, + redeliveries: 0, + redelivered: false, + updatedAt: now, + }, + $unset: { failure: '', finishedAt: '', expireAt: '' }, + }, + ); + return (result?.matchedCount ?? 0) > 0; + }, +}); + +/** + * Makes a mail queue in a collection of MongoDB. + * + * @param options `collection`, `db.collection('mail_outbox')`; + * `changeStream` (false), to wake the workers on a change (a replica set); + * `retainData` ('failed'), the finished jobs whose data is kept; `fairness` + * ('none'), or 'tenant' for the tenants' jobs in turn; + * `dedupeWindowSeconds` (a day), the time a completed job is kept, and so + * the window of its duplicates; `retainFailedSeconds` (a week), that of a + * failed one; `clockSkewToleranceMilliseconds` (5000), how + * far the clocks of the workers may be apart; `now`, `logger`, and the + * options of the polling driver. + * @returns The driver, with `source`, `ensureIndexes()` and `get(id)`. + * @throws {TypeError} If an option is invalid. + */ +const createMongoOutbox = (options = {}) => { + const config = createOutboxConfig(options); + const { collection } = config; const source = new MongoOutboxSource(collection, config); const driver = createPollingDriver(source, { ...options, now: config.now }); return { @@ -478,20 +585,7 @@ const createMongoOutbox = (options = {}) => { { name: 'mailer_expire', expireAfterSeconds: 0 }, ); }, - async get(id) { - const document = await collection.findOne({ _id: id }); - // The pause of the queue is no job. - // The pauses of the queue are no jobs. - if (document === null || document === undefined || id.startsWith(PAUSE_ID)) return undefined; - const job = jobOf(document); - return { - ...job, - state: document.state, - priority: document.priority, - failure: document.failure, - result: document.result, - }; - }, + ...readersOf(collection, source, config), }; }; diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 65d96dd..b16ae10 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -512,6 +512,23 @@ export interface MemoryQueueOptions extends PollingDriverOptions { */ export type MailQueueRetainData = 'all' | 'failed' | 'none'; +/** What a store of jobs holds. */ +export interface MailQueueStats { + /** Due now. */ + readonly ready: number; + /** Not due yet: scheduled, or waiting to be retried. */ + readonly delayed: number; + readonly active: number; + readonly completed: number; + readonly failed: number; + /** How long the oldest job due has waited: the lag of the workers. */ + readonly oldestReadyMilliseconds?: number; + /** Until when the whole queue is paused. */ + readonly pausedUntil?: number; + /** The tenants whose jobs are paused, and until when. */ + readonly pausedTenants?: readonly { readonly tenantId: string; readonly until: number }[]; +} + /** The queue in memory. */ export interface MemoryQueue extends QueueDriver { /** The source the polling driver reads: what a store's outbox implements, in memory. */ @@ -524,6 +541,15 @@ export interface MemoryQueue extends QueueDriver { wake(): void; /** Ends a pause of the queue, or of one tenant's jobs, before its time. */ resume(scope?: { readonly tenantId?: string }): Promise; + stats(): MailQueueStats; + /** The failed jobs, by id, a page of `limit` (100) after `after`. */ + failed(options?: { readonly limit?: number; readonly after?: string }): readonly MemoryQueueJob[]; + /** + * A failed job back to ready, its attempts anew: false when it is not a + * failed job. + * @throws {TypeError} If its data was dropped (`retainData`). + */ + retryFailed(id: string): boolean; } /** diff --git a/src/queue/memory-queue.js b/src/queue/memory-queue.js index e97d926..2d9def4 100644 --- a/src/queue/memory-queue.js +++ b/src/queue/memory-queue.js @@ -38,6 +38,8 @@ const nodeOf = (entry) => ({ leaseUntil: entry.leaseUntil, }); +const isFinished = (entry) => entry.state === 'completed' || entry.state === 'failed'; + const isCurrent = (node, state) => node.version === node.entry.version && node.entry.state === state; @@ -115,14 +117,27 @@ class MemorySource { this.#notify(); } + // The finished job at the head of a list, the stale places of jobs retried + // since dropped. + #finishedHead(list) { + for (;;) { + const id = headOf(list); + if (id === undefined) return undefined; + const entry = this.jobs.get(id); + if (entry !== undefined && isFinished(entry)) return entry; + dropHead(list); + } + } + // Drops the finished jobs whose time is up, then the oldest beyond the // count kept, the memory of their keys with them. #sweep(now) { this.#expireList(this.#completed, now); this.#expireList(this.#failed, now); while (sizeOf(this.#completed) + sizeOf(this.#failed) > this.#config.retainFinished) { - const completed = this.jobs.get(headOf(this.#completed)); - const failed = this.jobs.get(headOf(this.#failed)); + const completed = this.#finishedHead(this.#completed); + const failed = this.#finishedHead(this.#failed); + if (completed === undefined && failed === undefined) return; const older = failed === undefined || (completed !== undefined && completed.finishedSeq < failed.finishedSeq) @@ -144,12 +159,44 @@ class MemorySource { const id = headOf(list); if (id === undefined) return; const entry = this.jobs.get(id); + // A job retried by hand since is open again: its place here is stale. + if (entry !== undefined && !isFinished(entry)) { + dropHead(list); + continue; + } if (entry !== undefined && entry.expireAt > now) return; this.jobs.delete(id); dropHead(list); } } + // A failed job back to ready, its attempts anew: false when it is not one. + retryFailed(id) { + const entry = this.jobs.get(id); + if (entry === undefined || entry.state !== 'failed') return false; + if (entry.data === undefined) { + throw new TypeError(`The job ${id} keeps no data to send again: retainData dropped it.`); + } + entry.attempt = 0; + entry.redeliveries = 0; + entry.failure = undefined; + entry.availableAt = this.#config.now(); + entry.expireAt = undefined; + this.#open += 1; + this.#wait(entry); + this.#notify(); + return true; + } + + // The pauses in force: the queue's, and the tenants'. + pausesAt(now) { + const tenants = []; + for (const [tenantId, until] of this.#tenantPauses) { + if (until > now) tenants.push({ tenantId, until }); + } + return { until: this.#pausedUntil, tenants }; + } + #holds(lease) { const entry = this.jobs.get(lease?.job?.id); const held = entry !== undefined && entry.state === 'active' && entry.token === lease.token; @@ -478,8 +525,8 @@ const viewOf = (entry, now) => { * (10000), `dedupeWindowSeconds` (a day), `retainFailedSeconds` (a week), * `retainData` ('failed'), `fairness` ('none', or 'tenant' for the tenants' * jobs in turn), `logger`. - * @returns The driver, with `source`, `list()`, `get(id)`, `drain()` and - * `wake()`. + * @returns The driver, with `source`, `list()`, `get(id)`, `drain()`, + * `wake()`, `resume()`, `stats()`, `failed()` and `retryFailed(id)`. * @throws {TypeError} If an option is invalid. */ const createMemoryQueue = (options = {}) => { @@ -513,6 +560,43 @@ const createMemoryQueue = (options = {}) => { }, drain: () => source.drain(), resume: (scope) => source.resume(scope), + stats() { + source.prune(); + const now = config.now(); + const stats = { ready: 0, delayed: 0, active: 0, completed: 0, failed: 0 }; + let oldest; + for (const entry of source.jobs.values()) { + if (entry.state !== 'waiting') { + stats[entry.state] += 1; + } else if (entry.availableAt > now) { + stats.delayed += 1; + } else { + stats.ready += 1; + if (oldest === undefined || entry.availableAt < oldest) oldest = entry.availableAt; + } + } + if (oldest !== undefined) stats.oldestReadyMilliseconds = now - oldest; + const pauses = source.pausesAt(now); + if (pauses.until > now) stats.pausedUntil = pauses.until; + if (pauses.tenants.length > 0) stats.pausedTenants = pauses.tenants; + return stats; + }, + failed(failedOptions = {}) { + const limit = checkPositiveInteger(failedOptions.limit ?? 100, 'limit'); + source.prune(); + const ids = []; + for (const entry of source.jobs.values()) { + if (entry.state !== 'failed') continue; + if (failedOptions.after === undefined || entry.id > failedOptions.after) ids.push(entry.id); + } + const jobs = []; + for (const id of ids.toSorted()) { + if (jobs.length === limit) break; + jobs.push(viewOf(source.jobs.get(id), config.now())); + } + return jobs; + }, + retryFailed: (id) => source.retryFailed(id), wake: () => source.wake(), }; }; diff --git a/tests/helpers/mongodb/fake-collection.js b/tests/helpers/mongodb/fake-collection.js index 603fba7..0b1e085 100644 --- a/tests/helpers/mongodb/fake-collection.js +++ b/tests/helpers/mongodb/fake-collection.js @@ -205,20 +205,31 @@ const fakeCollection = (options = {}) => { find(filter, opts = {}) { calls.push(['find', filter, opts]); let sort; + let limit = Infinity; const cursor = { sort(spec) { sort = spec; return cursor; }, + limit(count) { + limit = count; + return cursor; + }, async toArray() { return sortBy( [...documents.values()].filter((document) => matches(document, filter)), sort, - ).map(clone); + ) + .slice(0, limit) + .map(clone); }, }; return cursor; }, + async countDocuments(filter = {}) { + calls.push(['countDocuments', filter]); + return [...documents.values()].filter((document) => matches(document, filter)).length; + }, async bulkWrite(operations, opts = {}) { calls.push(['bulkWrite', operations.length, opts]); let upsertedCount = 0; diff --git a/tests/mongodb/outbox.mongodb.test.js b/tests/mongodb/outbox.mongodb.test.js index 79e50ca..dee3e1d 100644 --- a/tests/mongodb/outbox.mongodb.test.js +++ b/tests/mongodb/outbox.mongodb.test.js @@ -296,3 +296,23 @@ test( assert.equal(transport.calls.length, 1); }, ); + +test( + 'MongoDB (replica set): the outbox tells what it holds, and retries a failed job', + { skip }, + async (t) => { + const { outbox, queue } = await setup(t); + const { id } = await queue.enqueue(job()); + await queue.enqueue(job({ scheduledAt: new Date(Date.now() + 60_000) })); + const { lease } = await outbox.source.reserve(1000); + await outbox.source.fail(lease, { code: 'x.rejected', reason: 'no' }); + const stats = await outbox.stats(); + assert.deepEqual([stats.ready, stats.delayed, stats.failed], [0, 1, 1]); + assert.deepEqual( + (await outbox.failed()).map((failed) => failed.id), + [id], + ); + assert.equal(await outbox.retryFailed(id), true); + assert.equal((await outbox.stats()).ready, 1); + }, +); diff --git a/tests/types/mongodb.test-d.ts b/tests/types/mongodb.test-d.ts index 39a5990..d0ee19b 100644 --- a/tests/types/mongodb.test-d.ts +++ b/tests/types/mongodb.test-d.ts @@ -24,6 +24,8 @@ expectAssignable(typed); const typedOutbox = createMongoOutbox({ collection: db.collection('x') }); expectType(typedOutbox.transactional); expectType>(typedOutbox.resume()); +expectType>(typedOutbox.retryFailed('j')); +expectType((await typedOutbox.stats()).oldestReadyMilliseconds); createMongoOutbox({ collection: db.collection('x'), fairness: 'tenant' }); const outbox = createMongoOutbox({ collection: db.collection('mail_outbox'), diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index fac4bdb..2c60ebf 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -24,6 +24,7 @@ import { type MailQueue, type MailWorker, type MemoryQueue, + type MemoryQueueJob, } from '../../src/queue/index.js'; import { createTemplateRenderer, type TemplateEngine } from '../../src/templates/index.js'; @@ -34,6 +35,9 @@ expectType(driver); expectType(driver.source); createMemoryQueue({ retainData: 'none', fairness: 'tenant' }); expectType>(driver.resume({ tenantId: 'acme' })); +expectType(driver.stats().ready); +expectType(driver.retryFailed('j')); +expectType(driver.failed({ limit: 10 })); expectError(createMemoryQueue({ fairness: 'fifo' })); expectError(createMemoryQueue({ retainData: 'some' })); expectAssignable(driver); diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index e9e4155..4796c15 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -602,3 +602,41 @@ test('MongoDB: the audit sink writes a batch in one bulkWrite, each event once', await createMongoAuditSink({ collection: plain }).appendMany([event('e1'), event('e2')]); assert.strictEqual(plain.documents.size, 2); }); + +test('MongoDB: the outbox tells what it holds, lists its failed jobs and retries one', async () => { + const { source, outbox, advance } = outboxOf({ start: 10_000 }); + await source.enqueue(requestOf('r1', { enqueuedAt: 4000 })); + await source.enqueue(requestOf('later', { availableAt: 99_999 })); + for (const id of ['f2', 'f1', 'f3']) { + await source.enqueue(requestOf(id, { enqueuedAt: 9000, priority: 'high' })); + } + for (let index = 0; index < 3; index++) { + await source.fail((await source.reserve(1000)).lease, { code: 'x.rejected', reason: 'no' }); + } + await source.pause(20_000, { tenantId: 'acme' }); + const stats = await outbox.stats(); + assert.deepStrictEqual( + [stats.ready, stats.delayed, stats.active, stats.completed, stats.failed], + [1, 1, 0, 0, 3], + ); + assert.strictEqual(stats.oldestReadyMilliseconds, 6000); + assert.deepStrictEqual(stats.pausedTenants, [{ tenantId: 'acme', until: 20_000 }]); + assert.deepStrictEqual( + (await outbox.failed({ limit: 2 })).map((job) => job.id), + ['f1', 'f2'], + ); + assert.deepStrictEqual( + (await outbox.failed({ after: 'f2' })).map((job) => job.id), + ['f3'], + ); + assert.strictEqual(await outbox.retryFailed('f1'), true); + assert.strictEqual(await outbox.retryFailed('f1'), false); + const again = await outbox.get('f1'); + assert.deepStrictEqual([again.state, again.attempt, again.failure], ['ready', 0, undefined]); + advance(1); + assert.strictEqual((await source.reserve(1000)).lease.job.id, 'f1'); + const none = outboxOf({ extra: { retainData: 'none' } }); + await none.source.enqueue(requestOf('x')); + await none.source.fail((await none.source.reserve(1000)).lease, { code: 'x', reason: 'y' }); + await assert.rejects(none.outbox.retryFailed('x'), /keeps no data/); +}); diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index f179b0f..4d20104 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -640,3 +640,45 @@ test('Queue: a retry of a tenant’s scope pauses that tenant’s jobs, not the await consumer.close(); assert.deepStrictEqual(runs.toSorted(), ['a1', 'a1', 'a2', 'b1']); }); + +test('Queue: the memory queue tells what it holds, lists its failed jobs and retries one', async () => { + const clock = manualClock(10_000); + const queue = createMemoryQueue({ now: clock.now }); + await queue.enqueue(requestOf('r1', { enqueuedAt: 4000 })); + await queue.enqueue(requestOf('later', { availableAt: 99_999 })); + for (const id of ['f2', 'f1', 'f3']) { + await queue.enqueue(requestOf(id, { enqueuedAt: 9000, priority: 'high' })); + } + for (let index = 0; index < 3; index++) { + const { lease } = await queue.source.reserve(1000); + await queue.source.fail(lease, { code: 'x.rejected', reason: 'no' }); + } + await queue.source.pause(20_000, { tenantId: 'acme' }); + const stats = queue.stats(); + assert.deepStrictEqual( + [stats.ready, stats.delayed, stats.active, stats.completed, stats.failed], + [1, 1, 0, 0, 3], + ); + assert.strictEqual(stats.oldestReadyMilliseconds, 6000); + assert.deepStrictEqual(stats.pausedTenants, [{ tenantId: 'acme', until: 20_000 }]); + // In the order of their ids, a page at a time. + assert.deepStrictEqual( + queue.failed({ limit: 2 }).map((job) => job.id), + ['f1', 'f2'], + ); + assert.deepStrictEqual( + queue.failed({ after: 'f2' }).map((job) => job.id), + ['f3'], + ); + assert.strictEqual(queue.retryFailed('f1'), true); + assert.strictEqual(queue.retryFailed('f1'), false); + assert.strictEqual(queue.retryFailed('r1'), false); + const again = queue.get('f1'); + assert.deepStrictEqual([again.state, again.attempt, again.failure], ['waiting', 0, undefined]); + // A job whose data was dropped cannot go again. + const none = createMemoryQueue({ now: clock.now, retainData: 'none' }); + await none.enqueue(requestOf('x')); + await none.source.fail((await none.source.reserve(1000)).lease, { code: 'x', reason: 'y' }); + assert.throws(() => none.retryFailed('x'), /keeps no data/); + assert.throws(() => queue.failed({ limit: 0 }), /limit/); +}); From f63b489ae6757c4314007b262b6f392e0e008386 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:09:43 +0300 Subject: [PATCH 52/83] fix(templates): bound what each engine keeps by the cache of the renderer cache.maxEntries bounded the renderer's caches alone: Handlebars kept every partial it registered in one environment, Nunjucks every template it read in the cache of its loader and every failed lookup up to 1000, Eta every template it included, and Liquid an LRU of 1024, so a renderer of templates a tenant writes grew with them. A Handlebars environment is made anew once it holds as many partials (they are registered as a template compiles, so it grows by none after); Nunjucks's loader cache, its failures and Eta's caches are LRUs of maxEntries, and Liquid's LRU holds as many. Co-Authored-By: Claude Opus 5.5 --- src/templates/cache.js | 3 ++ src/templates/environment.js | 27 ++++++++-- src/templates/eta.js | 18 +++++++ src/templates/handlebars.js | 19 ++++--- src/templates/liquid.js | 3 +- src/templates/nunjucks.js | 9 ++-- tests/unit/templates-engines.test.js | 75 ++++++++++++++++++++++++++++ 7 files changed, 139 insertions(+), 15 deletions(-) diff --git a/src/templates/cache.js b/src/templates/cache.js index 2f97ddb..9c91a73 100644 --- a/src/templates/cache.js +++ b/src/templates/cache.js @@ -32,6 +32,9 @@ const createCache = (maxEntries, onDrop) => { onDrop?.(oldest); } }, + delete(key) { + entries.delete(key); + }, // Forgets an entry, unless another value took its place meanwhile. forget(key, value) { if (entries.get(key) === value) entries.delete(key); diff --git a/src/templates/environment.js b/src/templates/environment.js index 347df5e..79bd5c9 100644 --- a/src/templates/environment.js +++ b/src/templates/environment.js @@ -11,14 +11,20 @@ const { TemplateNotFoundError } = require('./errors.js'); * each render without a cache. * * @param make Makes it, given the includes. + * @param sizeOf How many entries it keeps, if it keeps them as it compiles: + * once that is `maxEntries` of the includes, the next template is compiled in + * a new one, the old one kept by the templates compiled in it alone. * @returns `(includes) => instance`. * @internal */ -const perIncludes = (make) => { +const perIncludes = (make, sizeOf) => { const instances = new WeakMap(); return (includes) => { let instance = instances.get(includes); - if (instance === undefined) { + if ( + instance === undefined || + (sizeOf !== undefined && sizeOf(instance) >= includes.maxEntries) + ) { instance = make(includes); instances.set(includes, instance); } @@ -26,6 +32,21 @@ const perIncludes = (make) => { }; }; +/** + * A cache of an engine, an object it reads and writes by key, bounded by the + * cache of the renderer: Nunjucks's `loader.cache`. + * + * @internal + */ +const boundedRecord = (cache) => + new Proxy(Object.create(null), { + get: (target, key) => cache.get(key), + set: (target, key, value) => { + cache.set(key, value); + return true; + }, + }); + /** * A template an engine read by its name, which is not there. * @@ -65,4 +86,4 @@ const checkExtensions = (extensions, what) => { return Object.freeze([...extensions]); }; -module.exports = { checkExtensions, hasExtension, missingTemplate, perIncludes }; +module.exports = { boundedRecord, checkExtensions, hasExtension, missingTemplate, perIncludes }; diff --git a/src/templates/eta.js b/src/templates/eta.js index ecf99a1..9a4329e 100644 --- a/src/templates/eta.js +++ b/src/templates/eta.js @@ -3,6 +3,7 @@ // so `include()` and `layout()` read nothing from disk, a name relative to the // template it is written in; output is escaped, as Eta does by default. +const { createCache } = require('./cache.js'); const { missingTemplate, perIncludes } = require('./environment.js'); const { checkEngineExtensions, @@ -18,6 +19,21 @@ const UNESCAPED = new Map([ ['escapeFunction', () => true], ]); +// A Cacher of Eta, which keeps every template it includes, as it renders, +// bounded as the caches of the renderer are. +const cacherOf = (maxEntries) => { + const cache = createCache(maxEntries); + return { + define: (key, value) => cache.set(key, value), + get: (key) => cache.get(key), + remove: (key) => cache.delete(key), + reset: () => cache.clear(), + load(record) { + for (const key of Object.keys(record)) cache.set(key, record[key]); + }, + }; +}; + /** * Makes the Eta engine. * @@ -40,6 +56,8 @@ const createEtaEngine = (options = {}) => { const extensions = checkEngineExtensions(options.extensions, ['.eta']); const instanceOf = perIncludes((includes) => { const eta = new Eta({ ...given, cache: includes.cache, cacheFilepaths: false }); + eta.templatesSync = cacherOf(includes.maxEntries); + eta.templatesAsync = cacherOf(includes.maxEntries); eta.resolvePath = (template, resolveOptions) => includes.lookupSync(template, resolveOptions?.filepath, extensions).name; eta.readFile = (name) => { diff --git a/src/templates/handlebars.js b/src/templates/handlebars.js index 30534c8..3169795 100644 --- a/src/templates/handlebars.js +++ b/src/templates/handlebars.js @@ -92,13 +92,18 @@ const createHandlebarsEngine = (options = {}) => { checkEscaping(options, given, UNESCAPED, 'handlebars'); const configure = checkOptionalFunction(options.configure, 'configure'); const extensions = checkEngineExtensions(options.extensions, ['.hbs', '.handlebars']); - // The environment of each renderer. - const environmentOf = perIncludes(() => { - const env = handlebars.create(); - for (const name of Object.keys(helpers)) env.registerHelper(name, helpers[name]); - configure?.(env); - return { env, partials: new Map() }; - }); + // The environment of each renderer, made anew once it holds as many + // partials as the cache of the renderer keeps: partials are registered as a + // template is compiled, so an environment grows by none after. + const environmentOf = perIncludes( + () => { + const env = handlebars.create(); + for (const name of Object.keys(helpers)) env.registerHelper(name, helpers[name]); + configure?.(env); + return { env, partials: new Map() }; + }, + (environment) => environment.partials.size, + ); // Compiles and registers a partial once per environment however many // templates name it at once, under the name of its template, and gives the // partials it names, which it does not wait for: a partial that names diff --git a/src/templates/liquid.js b/src/templates/liquid.js index fde177a..c11a5a2 100644 --- a/src/templates/liquid.js +++ b/src/templates/liquid.js @@ -145,7 +145,8 @@ const createLiquidEngine = (options = {}) => { extname: extensions[0], relativeReference: true, fs: fileSystemOf(includes, roots, extensions[0]), - cache: includes.cache, + // An LRU of as many templates as the renderer's caches keep. + cache: includes.cache ? includes.maxEntries : false, }); for (const name of Object.keys(filters)) liquid.registerFilter(name, filters[name]); configure?.(liquid); diff --git a/src/templates/nunjucks.js b/src/templates/nunjucks.js index 6863757..cbb0491 100644 --- a/src/templates/nunjucks.js +++ b/src/templates/nunjucks.js @@ -13,7 +13,7 @@ const { checkModule, } = require('./options.js'); const { checkOptionalFunction } = require('../internal/options.js'); -const { perIncludes } = require('./environment.js'); +const { boundedRecord, perIncludes } = require('./environment.js'); const isRelative = (name) => name.startsWith('./') || name.startsWith('../'); @@ -23,8 +23,6 @@ const UNESCAPED = new Map([['autoescape', (value) => value != null && !value]]); // What Nunjucks says of a template no loader has. const NOT_FOUND = 'template not found: '; -// The failed lookups an environment keeps, to tell which names were tried. -const MAX_FAILURES = 1000; // The template Nunjucks found missing, by its error, whose message ends a // line with `template not found: `: the error is made anew, without @@ -72,7 +70,7 @@ const createNunjucksEngine = (options = {}) => { // The lookups that found nothing, by name: Nunjucks is told there is no // such template, which it fails on itself unless `ignore missing`, and // its error names the template, so each render finds its own. - const failures = createCache(MAX_FAILURES); + const failures = createCache(includes.maxEntries); const loader = { async: true, isRelative, @@ -100,6 +98,9 @@ const createNunjucksEngine = (options = {}) => { }; // `dev` keeps the error Nunjucks wraps as its cause, such as a loader's. const env = new nunjucks.Environment([loader], { autoescape: true, dev: true, ...given }); + // Nunjucks keeps every template it reads in the cache of its loader, as + // it renders: bounded, as the renderer's are. + loader.cache = boundedRecord(createCache(includes.cache ? includes.maxEntries : 0)); for (const name of Object.keys(filters)) env.addFilter(name, filters[name]); configure?.(env); return { env, failures }; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index d995a59..9e99cdc 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -1094,6 +1094,81 @@ test('Templates: every engine reads the same tree of includes by one rule', asyn } }); +test('Templates: what each engine keeps is bounded by the cache of the renderer', async () => { + // Before: Handlebars kept every partial in one environment, Nunjucks and + // Eta every template they included, and Liquid 1024, whatever + // cache.maxEntries said. + const cache = { maxEntries: 2 }; + const includes = ['i1', 'i2', 'i3', 'i4']; + // Handlebars: a new environment once one holds as many partials. + const environments = []; + const hbs = createHandlebarsEngine({ handlebars, configure: (env) => environments.push(env) }); + const partials = createTemplateRenderer({ + engine: hbs, + loader: createMemoryLoader({ + 'a.hbs': '{{> p1}}{{> p2}}', + 'b.hbs': '{{> p3}}', + 'p1.hbs': '1', + 'p2.hbs': '2', + 'p3.hbs': '3', + }), + cache, + }); + assert.strictEqual((await partials.render({ template: 'a' })).html, '12'); + assert.strictEqual((await partials.render({ template: 'b' })).html, '3'); + assert.strictEqual(environments.length, 2); + // Nunjucks: the cache of its loader. + let env; + const njk = createNunjucksEngine({ nunjucks, configure: (made) => (env = made) }); + // An async loader needs asyncEach around an include. + const sources = { 'a.njk': '{% asyncEach n in names %}{% include n + ".njk" %}{% endeach %}' }; + for (const name of includes) sources[`${name}.njk`] = name; + const looped = createTemplateRenderer({ + engine: njk, + loader: createMemoryLoader(sources), + cache, + }); + assert.strictEqual( + (await looped.render({ template: 'a', context: { names: includes } })).html, + 'i1i2i3i4', + ); + assert.strictEqual(env.loaders[0].cache['i1.njk'], undefined); + assert.ok(env.loaders[0].cache['i4.njk'] !== undefined); + // Eta: what it includes, compiled again once dropped. + let compiles = 0; + class CountingEta extends Eta { + constructor(config) { + super(config); + const compile = this.compile; + this.compile = (...args) => { + compiles++; + return compile.apply(this, args); + }; + } + } + const eta = createTemplateRenderer({ + engine: createEtaEngine({ Eta: CountingEta }), + loader: createMemoryLoader({ + 'a.eta': '<%~ include("i1") %><%~ include("i2") %><%~ include("i3") %>', + 'i1.eta': '1', + 'i2.eta': '2', + 'i3.eta': '3', + }), + cache, + }); + await eta.render({ template: 'a' }); + const first = compiles; + assert.strictEqual((await eta.render({ template: 'a' })).html, '123'); + assert.ok(compiles > first); + // Liquid: an LRU of as many. + let liquid; + const liquidEngine = createLiquidEngine({ Liquid, configure: (made) => (liquid = made) }); + await renderWith(liquidEngine, { 'a.liquid': 'x' }, { template: 'a' }, { cache }); + assert.strictEqual(liquid.options.cache.limit, 2); + await renderWith(liquidEngine, { 'a.liquid': 'x' }, { template: 'a' }, { cache: false }); + assert.strictEqual(liquid.options.cache, undefined); +}); + test('Templates: juice and css-inline inline the CSS and fetch nothing', async () => { const html = '

    x

    '; From 2105f0d686f14775d73eaacb539e8419b77ad25c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:09:47 +0300 Subject: [PATCH 53/83] perf(templates): parse a Mustache template once, and a partial once per template read The Mustache engine rendered with writer.render, which parsed the template again on each render through a cache keyed by its whole source, and every partial the same way, so the cache held a copy of each source and every render a concatenation of each. The engine renders the tokens it parsed when it compiled, with Mustache's Context, and keeps the tokens of a partial by the template the includes read, in a WeakMap, so Mustache keeps nothing; `mustache.Context` is now needed. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 3 +- src/templates/index.d.ts | 4 +- src/templates/mustache.js | 60 ++++++++++++++++++---------- tests/types/templates.test-d.ts | 2 +- tests/unit/templates-engines.test.js | 22 +++++++++- 5 files changed, 65 insertions(+), 26 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index feb421c..ce07d1f 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -173,7 +173,8 @@ Some engines behave a little differently here than they do on their own: - **Mustache** fails on a partial that is not there, where it renders nothing, and reads `./x` against the partial it is written in, which Mustache does not tell. Its writer, and the cache of what it parsed, are per renderer and - bounded. + bounded: a template is parsed once, as it is compiled, and a partial once + per template read, where Mustache keeps what it parses by the whole source. - **MJML** renders after the engine inside it (`.mjml.hbs`), once the layout is applied, so the layout of an MJML template is MJML too. A static `.mjml` is rendered once. What MJML finds wrong under `validationLevel: 'soft'` goes to diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 621dd3a..06b7183 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -670,8 +670,10 @@ export declare function createEtaEngine(options: EtaEngineOptions): TemplateEngi /** The members of `require('mustache')` the engine calls. */ export interface MustacheModule { - /** A writer of its own, whose cache the engine bounds; mustache.render shares one for the process. */ + /** A writer of its own, which keeps nothing; mustache.render shares one for the process, which keeps every source it parses. */ readonly Writer: new () => any; + /** The context of a render, as the engine renders the tokens it parsed. */ + readonly Context: new (view: object) => any; } /** The options of {@link createMustacheEngine}. */ export interface MustacheEngineOptions { diff --git a/src/templates/mustache.js b/src/templates/mustache.js index d31d823..6727ffc 100644 --- a/src/templates/mustache.js +++ b/src/templates/mustache.js @@ -1,17 +1,15 @@ // Not ported: the Mustache engine of ./templates, over the `mustache` the -// application passes in. Each renderer has a writer of its own, whose cache -// of parsed templates is bounded, where `mustache.render` shares one without a -// bound for the process; a partial that is not there fails the render, where +// application passes in. Each renderer has a writer of its own, where +// `mustache.render` shares one for the process, whose cache keeps what it +// parses by the whole source, without a bound: a template is parsed once as +// it is compiled, and a partial once per template the includes read, kept as +// long as they keep it. A partial that is not there fails the render, where // Mustache renders it as nothing; and a partial is looked up from the // template it is written in, where Mustache tells its partials nothing of it. -const { createCache } = require('./cache.js'); const { perIncludes } = require('./environment.js'); const { checkEngineExtensions, checkModule } = require('./options.js'); -// The parsed templates a writer keeps. -const MAX_PARSED = 1000; - // The fields of a partial's token in Mustache 4: its name, and the // indentation of a standalone partial. const NAME = 1; @@ -46,28 +44,47 @@ const createMustacheEngine = (options = {}) => { 'Writer', 'createMustacheEngine needs mustache: the module of the mustache package.', ); + checkModule(mustache.Context, undefined, 'mustache has no Context.'); const tags = checkTags(options.tags); const extensions = checkEngineExtensions(options.extensions, ['.mustache']); const writerOf = perIncludes((includes) => { const writer = new mustache.Writer(); + // Mustache keeps nothing: the engine keeps the tokens. + writer.templateCache = undefined; + // The tokens of each partial, by the template the includes read, which + // the WeakMap drops with it, and by its indentation, as Mustache indents + // a standalone partial before it parses it. + const parsed = new WeakMap(); + const tokensOf = (found, indentation, parse) => { + let byIndentation = parsed.get(found); + if (byIndentation === undefined) { + byIndentation = new Map(); + parsed.set(found, byIndentation); + } + let tokens = byIndentation.get(indentation); + if (tokens === undefined) { + tokens = parse(); + byIndentation.set(indentation, tokens); + } + return tokens; + }; // Mustache's renderPartial, but for the partials it is given, which are // the template they are written in: each partial renders its own with // the partials of its name. writer.renderPartial = function renderPartial(token, context, partials, config) { const found = includes.lookupSync(token[NAME], partials.from, extensions); - let source = found.source; - if (token[TAG_INDEX] === 0 && token[INDENTATION]) { - source = this.indentPartial(source, token[INDENTATION], token[LINE_HAS_NON_SPACE]); - } - const tokens = this.parse(source, this.getConfigTags(config)); + const standalone = token[TAG_INDEX] === 0 && Boolean(token[INDENTATION]); + const source = standalone + ? this.indentPartial(found.source, token[INDENTATION], token[LINE_HAS_NON_SPACE]) + : found.source; + const indentation = standalone + ? `${token[INDENTATION]}\u0000${token[LINE_HAS_NON_SPACE]}` + : ''; + const tokens = tokensOf(found, indentation, () => + this.parse(source, this.getConfigTags(config)), + ); return this.renderTokens(tokens, context, { from: found.name }, source, config); }; - const parsed = createCache(includes.cache ? MAX_PARSED : 0); - // Mustache calls set and get; the renderer forgets a writer, not its cache. - writer.templateCache = { - set: (key, value) => parsed.set(key, value), - get: (key) => parsed.get(key), - }; return writer; }); return { @@ -75,10 +92,11 @@ const createMustacheEngine = (options = {}) => { extensions, compile(source, context) { const writer = writerOf(context.includes); - // Parsed now, so that a syntax error is one of the compile. - writer.parse(source, tags); + // Parsed now, so that a syntax error is one of the compile, and once. + const tokens = writer.parse(source, tags); const partials = { from: context.name }; - return (scope) => writer.render(source, scope, partials, tags); + return (scope) => + writer.renderTokens(tokens, new mustache.Context(scope), partials, source, tags); }, }; }; diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index 62ba193..e5ba948 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -44,7 +44,7 @@ expectType>(juiced('

    x

    ')); declare const ejs: { compile(source: string, options?: object): (data?: object) => string }; declare const pug: { compile(source: string, options?: object): (locals?: object) => string }; -declare const mustache: { Writer: new () => object }; +declare const mustache: { Writer: new () => object; Context: new (view: object) => object }; declare const nunjucks: { Environment: new (loaders: unknown, options?: object) => object; Template: new ( diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 9e99cdc..6c263eb 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -1096,8 +1096,8 @@ test('Templates: every engine reads the same tree of includes by one rule', asyn test('Templates: what each engine keeps is bounded by the cache of the renderer', async () => { // Before: Handlebars kept every partial in one environment, Nunjucks and - // Eta every template they included, and Liquid 1024, whatever - // cache.maxEntries said. + // Eta every template they included, Liquid 1024, and Mustache 1000 parsed + // templates by their whole source, whatever cache.maxEntries said. const cache = { maxEntries: 2 }; const includes = ['i1', 'i2', 'i3', 'i4']; // Handlebars: a new environment once one holds as many partials. @@ -1169,6 +1169,24 @@ test('Templates: what each engine keeps is bounded by the cache of the renderer' assert.strictEqual(liquid.options.cache, undefined); }); +test('Templates: Mustache parses a template once, and a partial once per template read', async (t) => { + // Before: every render parsed the template and its partials again through + // a cache keyed by their whole source. + const parse = t.mock.method(mustache.Writer.prototype, 'parse'); + const renderer = createTemplateRenderer({ + engine: createMustacheEngine({ mustache }), + loader: createMemoryLoader({ 'a.mustache': '{{n}}{{> b}}', 'b.mustache': '[{{n}}]' }), + }); + for (let n = 0; n < 3; n++) { + assert.strictEqual( + (await renderer.render({ template: 'a', context: { n } })).html, + `${n}[${n}]`, + ); + } + assert.strictEqual(parse.mock.callCount(), 2); + assert.throws(() => createMustacheEngine({ mustache: { Writer: class {} } }), /no Context/); +}); + test('Templates: juice and css-inline inline the CSS and fetch nothing', async () => { const html = '

    x

    '; From 4cfa5aab55d7e10f98b134c1c763119e5a66eb6c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:10:14 +0300 Subject: [PATCH 54/83] feat(mongodb): find the events of a message by the provider's id of it A webhook's event without the correlationId of its message (an SMTP relay, a provider that drops metadata) is filed under the provider's id of the message, apart from the worker's events, and the MongoDB sink had no way to bring them together. eventsOfProviderMessage(id) finds the events filed under it and those of the message whose accepted event carries it, by a sparse index on providerMessageId that ensureIndexes() makes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++- docs/guide/audit.md | 8 ++++++++ src/mongodb/audit-sink.js | 29 ++++++++++++++++++++++++++++- src/mongodb/index.d.ts | 8 +++++++- tests/unit/mongodb.test.js | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e67be69..a8455d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. of an attempt whose run before died `delivery_unknown` rather than a second send, unless the audit sink tells (`has`, new on the memory, MongoDB and combined sinks) that it died before its submit. `AuditSink.appendMany` (memory, MongoDB in one `bulkWrite`, combined) writes the - `queued` events of an `enqueueMany` in one call. The fields of a job of version 1 are fixed: + `queued` events of an `enqueueMany` in one call. The MongoDB sink's + `eventsOfProviderMessage(id)` finds the events of a message by the provider's id of it, where a + webhook's event without `correlationId` is filed. The fields of a job of version 1 are fixed: a later version, or a field this one does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`, given back with its attempts not spent, for a week or until it expires) rather than fail its hash during a rolling deploy. diff --git a/docs/guide/audit.md b/docs/guide/audit.md index f7437ae..efe2525 100644 --- a/docs/guide/audit.md +++ b/docs/guide/audit.md @@ -89,6 +89,14 @@ await commit(); unsubscribe are engagement, not states. The queue writes `correlationId` into the metadata of each message, which the providers give back with its events. +A webhook's event names its message by the `correlationId` the message's +metadata gave back, which every HTTP provider of `./webhooks` returns; without +it (an SMTP relay, a provider that drops metadata) it is filed under the +provider's id of the message. The MongoDB sink's +`eventsOfProviderMessage(providerMessageId)` finds both: the events filed under +that id and those of the message whose `accepted` event carries it (an index of +`ensureIndexes()`). + ## Messages to reconcile A message whose last state is `submitted` long after its send, or diff --git a/src/mongodb/audit-sink.js b/src/mongodb/audit-sink.js index 098eb92..50c3115 100644 --- a/src/mongodb/audit-sink.js +++ b/src/mongodb/audit-sink.js @@ -17,12 +17,33 @@ const eventOfDocument = (document) => { return { id: _id, ...rest, at: at instanceof Date ? at.toISOString() : at }; }; +// The events of a message named by the provider's id of it: a webhook's +// event with no correlationId is filed under that id, and the worker's +// accepted event carries it, so the two meet here. +const eventsOfProviderMessage = async (collection, providerMessageId) => { + const named = await collection + // oxlint-disable-next-line no-restricted-properties -- a MongoDB collection, not an array + .find({ providerMessageId }) + .toArray(); + const correlationIds = new Set([providerMessageId]); + for (const document of named) correlationIds.add(document.correlationId); + const documents = await collection + // oxlint-disable-next-line no-restricted-properties -- a MongoDB collection, not an array + .find({ correlationId: { $in: [...correlationIds] } }) + .sort({ at: 1, attempt: 1, rank: 1 }) + .toArray(); + const events = []; + for (const document of documents) events.push(eventOfDocument(document)); + return events; +}; + /** * Makes an audit sink in a collection. * * @param options `collection`, `db.collection('mail_audit')`. * @returns The sink, with `has(id)`, `eventsOf(correlationId)`, - * `stateOf(correlationId)` and `ensureIndexes()`. + * `eventsOfProviderMessage(providerMessageId)`, `stateOf(correlationId)` and + * `ensureIndexes()`. * @throws {TypeError} If it is given no collection. */ const createMongoAuditSink = (options = {}) => { @@ -43,6 +64,8 @@ const createMongoAuditSink = (options = {}) => { return events; }; return { + eventsOfProviderMessage: (providerMessageId) => + eventsOfProviderMessage(collection, providerMessageId), async has(id) { const found = await collection.findOne({ _id: id }, { projection: { _id: 1 } }); return found !== null && found !== undefined; @@ -86,6 +109,10 @@ const createMongoAuditSink = (options = {}) => { { name: 'mailer_correlation' }, ); await collection.createIndex({ jobId: 1 }, { name: 'mailer_job', sparse: true }); + await collection.createIndex( + { providerMessageId: 1 }, + { name: 'mailer_provider_message', sparse: true }, + ); }, }; }; diff --git a/src/mongodb/index.d.ts b/src/mongodb/index.d.ts index d56e9ca..c337464 100644 --- a/src/mongodb/index.d.ts +++ b/src/mongodb/index.d.ts @@ -153,8 +153,14 @@ export interface MongoAuditSink extends AuditSink { ): Promise; has(id: string): Promise; eventsOf(correlationId: string): Promise; + /** + * The events of the message a provider's id names: those filed under it (a + * webhook's event with no correlationId) and those of the message whose + * accepted event carries it. + */ + eventsOfProviderMessage(providerMessageId: string): Promise; stateOf(correlationId: string): Promise; - /** Makes the indexes of the events: by correlation id and time, by job. */ + /** Makes the indexes of the events: by correlation id and time, by job, by the provider's id. */ ensureIndexes(): Promise; } diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 4796c15..88ee4fb 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -324,6 +324,7 @@ test('MongoDB: the audit sink writes an event once by its id', async () => { assert.deepStrictEqual(collection.indexes, [ [{ correlationId: 1, at: 1, attempt: 1, rank: 1 }, { name: 'mailer_correlation' }], [{ jobId: 1 }, { name: 'mailer_job', sparse: true }], + [{ providerMessageId: 1 }, { name: 'mailer_provider_message', sparse: true }], ]); // A sink keeps the memory sink's contract. const memory = createMemoryAuditSink(); @@ -640,3 +641,34 @@ test('MongoDB: the outbox tells what it holds, lists its failed jobs and retries await none.source.fail((await none.source.reserve(1000)).lease, { code: 'x', reason: 'y' }); await assert.rejects(none.outbox.retryFailed('x'), /keeps no data/); }); + +test('MongoDB: the events of a message are found by the provider’s id of it', async () => { + const { auditEventOfDeliveryEvent } = require('../../src/audit/index.js'); + const collection = fakeCollection(); + const sink = createMongoAuditSink({ collection }); + const base = { source: 'worker', correlationId: 'order-42', attempt: 1 }; + await sink.append({ ...base, id: 'e1', state: 'submitted', at: '2026-10-18T09:00:00.000Z' }); + await sink.append({ + ...base, + id: 'e2', + state: 'accepted', + at: '2026-10-18T09:00:01.000Z', + providerMessageId: 'ses-1', + }); + // A webhook's event with no correlationId is filed under the provider's id. + const delivered = auditEventOfDeliveryEvent({ + id: 'w1', + provider: 'ses', + type: 'delivered', + messageId: 'ses-1', + occurredAt: new Date('2026-10-18T09:00:05.000Z'), + }); + assert.strictEqual(delivered.correlationId, 'ses-1'); + await sink.append(delivered); + const events = await sink.eventsOfProviderMessage('ses-1'); + assert.deepStrictEqual( + events.map((event) => event.state), + ['submitted', 'accepted', 'delivered'], + ); + assert.deepStrictEqual(await sink.eventsOfProviderMessage('unknown'), []); +}); From ec9ace6b483a845bfd69167177395c74dbc1c354 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:11:35 +0300 Subject: [PATCH 55/83] feat(templates): tell a logger each compile and render, a hit or a miss Nothing told how long a template took to compile or to render, or whether the cache had it. A renderer takes a pino-like `logger`, which hears at debug, with component 'templates', templates.compiled (template, engine, part, durationMs) for each compile and templates.rendered (template, requested, locale, engine, durationMs, and cache: 'miss' when the render resolved or compiled a template itself, else 'hit') for each render; a record is built only when debug is written. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 9 ++++ src/templates/index.d.ts | 9 ++++ src/templates/renderer.js | 54 ++++++++++++++++--- tests/types/templates.test-d.ts | 1 + tests/unit/templates-renderer.test.js | 77 +++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 7 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index ce07d1f..4ddadb0 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -304,6 +304,15 @@ it to its end. Liquid's `renderLimit` (on by default) is checked as it renders; with the other engines, keep such templates out of the hands of those who could write one (see [who may write a template](#who-may-write-a-template)). +## Logging + +Given a pino-like `logger`, the renderer tells at `debug`, with +`component: 'templates'`, each compile (`templates.compiled`: `template`, +`engine`, `part`, `durationMs`) and each render (`templates.rendered`: +`template`, `requested`, `locale`, `engine`, `durationMs`, and `cache`, `miss` +when the render resolved or compiled a template itself, `hit` when it found +them kept). A record is built only when the logger writes `debug`. + ## Errors | Thrown | When | `retryable` | diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 06b7183..53e48f1 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -1,6 +1,7 @@ // Not ported: the types of ./templates, templates by name rendered by an // engine the application passes in. +import type { Logger } from '../../index.js'; import type { ReactEmailRenderOptions, RenderOutput, Renderer } from '../render/index.js'; /** The values a template renders. */ @@ -293,6 +294,14 @@ export interface TemplateRendererOptions { * stopped so: Liquid's `renderLimit` stops one. @default 30000 */ readonly renderTimeoutMilliseconds?: number; + /** + * A pino-like logger, which hears at `debug`, with `component: 'templates'`: + * `templates.compiled` (`template`, `engine`, `part`, `durationMs`) for each + * compile, and `templates.rendered` (`template`, `requested`, `locale`, + * `engine`, `cache`: `'hit'`, or `'miss'` when the render resolved or + * compiled a template itself, `durationMs`) for each render. None by default. + */ + readonly logger?: Logger; } /** A renderer of templates by name, a `Renderer` of `./render`. */ diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 0e678a6..7549683 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -10,6 +10,7 @@ const { decodeEntities } = require('../render/entities.js'); const { messageOf } = require('../internal/decorator.js'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { abortReason } = require('../internal/abort.js'); +const { createLog } = require('../internal/log.js'); const { timerMs } = require('../internal/sleep.js'); const { createCache, hasTimeout, memoize } = require('./cache.js'); const { @@ -178,6 +179,7 @@ const createRendererConfig = (options) => { ), maxOutputBytes: checkBound(options.maxOutputBytes, 'maxOutputBytes', DEFAULT_MAX_OUTPUT_BYTES), renderTimeoutMilliseconds: checkTimeout(options.renderTimeoutMilliseconds), + log: createLog(options.logger, { component: 'templates' }), }; }; @@ -260,6 +262,9 @@ const normalizeSubject = (subject) => { const isParts = (value) => typeof value === 'object' && value !== null; +// Milliseconds since `start`, to the microsecond. +const since = (start) => Math.round((performance.now() - start) * 1000) / 1000; + // A render, settled at its timeout or at the abort of its signal, whichever // comes first: what it waits for goes on, for the renders that share it, and // is not kept if it hangs. The timer keeps the process, as its caller waits. @@ -326,12 +331,29 @@ const createTemplateRenderer = (options = {}) => { // Without a cache, each render reads and compiles anew, includes too, // which its signal stops. const current = config.cache ? state : { ...state, files: makeIncludes(checked.signal) }; - const bounds = { includes: 0, maxIncludes: config.maxIncludes }; + // The bounds of the render, and what it made itself rather than found + // kept: a cache miss. + const bounds = { includes: 0, maxIncludes: config.maxIncludes, missed: false }; + const start = performance.now(); + let output; try { - return await inTime(config, checked, renderTemplate(current, checked, input, bounds)); + output = await inTime(config, checked, renderTemplate(current, checked, input, bounds)); } catch (error) { throw forRender(error, checked.template, checked.locale); } + config.log.write( + 'debug', + () => ({ + template: output.template, + requested: checked.template, + locale: checked.locale, + engine: bounds.engine, + cache: bounds.missed ? 'miss' : 'hit', + durationMs: since(start), + }), + 'templates.rendered', + ); + return output; }, clear() { // Engines key what they keep by the includes, so they make it anew too; @@ -401,11 +423,14 @@ const resolveTemplate = async (state, template, locale) => { throw new TemplateNotFoundError(template, tried, { phase: 'resolve' }); }; -const resolvedFor = (state, template, locale) => +const resolvedFor = (state, template, locale, bounds) => memoize( state.resolved, `${template}\u0000${locale ?? ''}`, - () => resolveTemplate(state, template, locale), + () => { + bounds.missed = true; + return resolveTemplate(state, template, locale); + }, state.config.renderTimeoutMilliseconds, ); @@ -431,6 +456,7 @@ const compileEntry = async (state, found, part, bounds) => { includes: state.files.includes, }; state.files.top(name); + const start = performance.now(); const render = await run('compile', name, engine, () => engine.compile(body, context), bounds); if (typeof render !== 'function') { throw new TemplateRenderError(name, 'compile', 'The engine gave no function.', { @@ -447,6 +473,11 @@ const compileEntry = async (state, found, part, bounds) => { () => engine.compile(fields.subject, { ...context, part: 'subject' }), bounds, ); + state.config.log.write( + 'debug', + () => ({ template: name, engine: engine.name, part, durationMs: since(start) }), + 'templates.compiled', + ); return { name, engine, render, subject, fields, context }; }; @@ -454,7 +485,10 @@ const compiledFor = (state, found, part, bounds) => memoize( state.compiled, `${found.engine.name}\u0000${part}\u0000${found.name}`, - () => compileEntry(state, found, part, bounds), + () => { + bounds.missed = true; + return compileEntry(state, found, part, bounds); + }, state.config.renderTimeoutMilliseconds, ); @@ -492,7 +526,7 @@ const applyLayout = async (state, entry, html, scope, locale, bounds) => { if (name === undefined) return { html, engine: entry.engine, context: entry.context }; let resolution; try { - resolution = await resolvedFor(state, name, locale); + resolution = await resolvedFor(state, name, locale, bounds); } catch (error) { if (!(error instanceof TemplateNotFoundError)) throw inPhase(error, 'layout'); throw new TemplateNotFoundError(name, error.tried, { @@ -527,7 +561,12 @@ const renderTemplate = async (state, checked, input, bounds) => { const { config } = state; let resolution; try { - resolution = await resolvedFor(state, checked.template, allowedLocale(config, checked.locale)); + resolution = await resolvedFor( + state, + checked.template, + allowedLocale(config, checked.locale), + bounds, + ); } catch (error) { throw inPhase(error, 'resolve'); } @@ -540,6 +579,7 @@ const renderTemplate = async (state, checked, input, bounds) => { resolution.html ? 'html' : 'text', bounds, ); + bounds.engine = main.engine.name; let html; let text; let subject; diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index e5ba948..12bb687 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -111,6 +111,7 @@ createTemplateRenderer({ load: (name, options) => (options?.signal?.aborted ? undefined : { name, source: '' }), }, renderTimeoutMilliseconds: 5000, + logger: { debug: (properties: object, message: string) => [properties, message] }, }); expectType(output); expectType(output.subject); diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index ab9ed0c..f1a1bb1 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -837,6 +837,83 @@ test('Templates: a render stops at the abort of its signal, with its reason', as ); }); +test('Templates: a logger hears each compile and each render at debug, a hit or a miss', async () => { + // Before: nothing told how long a compile or a render took, or whether the + // cache had it. + const records = []; + const logger = { + debug: (properties, message) => records.push({ message, ...properties }), + isLevelEnabled: (level) => level === 'debug', + }; + const { renderer } = rendererOf( + { 'a.tpl': '---\nsubject: S\n---\n

    {{n}}

    ', 'a.uk.tpl': '

    uk

    ' }, + { logger }, + ); + await renderer.render({ template: 'a', context: { n: 1 } }); + await renderer.render({ template: 'a', context: { n: 2 } }); + await renderer.render({ template: 'a', locale: 'uk' }); + const shape = (record) => { + assert.ok(record.durationMs >= 0, record.message); + return { ...record, durationMs: undefined }; + }; + assert.deepStrictEqual(records.map(shape), [ + { + message: 'templates.compiled', + component: 'templates', + template: 'a.tpl', + engine: 'fake', + part: 'html', + durationMs: undefined, + }, + { + message: 'templates.rendered', + component: 'templates', + template: 'a.tpl', + requested: 'a', + locale: undefined, + engine: 'fake', + cache: 'miss', + durationMs: undefined, + }, + { + message: 'templates.rendered', + component: 'templates', + template: 'a.tpl', + requested: 'a', + locale: undefined, + engine: 'fake', + cache: 'hit', + durationMs: undefined, + }, + { + message: 'templates.compiled', + component: 'templates', + template: 'a.uk.tpl', + engine: 'fake', + part: 'html', + durationMs: undefined, + }, + { + message: 'templates.rendered', + component: 'templates', + template: 'a.uk.tpl', + requested: 'a', + locale: 'uk', + engine: 'fake', + cache: 'miss', + durationMs: undefined, + }, + ]); + // Above debug, nothing is built. + const quiet = []; + const { renderer: info } = rendererOf( + { 'a.tpl': '

    ' }, + { logger: { debug: () => quiet.push(1), isLevelEnabled: () => false } }, + ); + await info.render({ template: 'a' }); + assert.strictEqual(quiet.length, 0); +}); + test('Templates: the renderer checks its options', () => { const engine = fakeEngine(); const loader = createMemoryLoader({}); From d260c243d36734db6d04e6b466c1675e74fcfc76 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:13:16 +0300 Subject: [PATCH 56/83] fix(templates): refuse a layout no template of the renderer would get A layout was skipped for a template that gave its parts at once, so a renderer of React Email components and a layout sent every message without it, and said nothing. An engine may say `parts: true`, as React Email's does, and a renderer given a layout refuses it with a TypeError; a template of another engine that gives its parts where a layout applies fails as a TemplateRenderError of phase `layout`. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 2 ++ src/templates/index.d.ts | 8 +++++++- src/templates/react-email.js | 2 ++ src/templates/renderer.js | 24 ++++++++++++++++++++++- tests/types/templates.test-d.ts | 1 + tests/unit/templates-engines.test.js | 2 ++ tests/unit/templates-renderer.test.js | 28 ++++++++++++++++++++++++++- 7 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 4ddadb0..7fe8803 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -118,6 +118,8 @@ pass unnoticed. `frontMatter: false` reads none. context and `body`, the HTML of the template, raw. A layout has no layout of its own. Engines that have layouts of their own (Pug's `extends`, Nunjucks's and Liquid's blocks, Handlebars partial blocks, Eta's `layout()`) keep them. +A React Email template gives its HTML whole, so a renderer with a `layout` +refuses the React Email engine, rather than send its messages without it. Every include, partial and layout an engine reads goes through the loader of the renderer, by one rule, the same in every engine: diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 53e48f1..1d95770 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -194,6 +194,11 @@ export interface TemplateEngine { finish?(html: string, context: TemplateCompileContext): string | Promise; /** The templates of an engine whose templates are code (React Email), when the renderer has no loader. */ readonly loader?: TemplateLoader; + /** + * Whether its templates give their parts at once, as React Email's do: a + * renderer with a `layout` refuses it, as no layout wraps them. + */ + readonly parts?: boolean; /** * Forgets what the engine keeps for these includes outside them, such as * EJS's entries in `ejs.cache`: called by the renderer's `clear()`. @@ -334,7 +339,8 @@ export interface TemplateRenderer< * template: 'reset-password', context: { url }, locale: 'uk', * }); * ``` - * @throws {TypeError} If an option is invalid. + * @throws {TypeError} If an option is invalid, or a layout is given with an + * engine that gives its parts at once. */ export declare function createTemplateRenderer< TInput extends TemplateInput = TemplateInput, diff --git a/src/templates/react-email.js b/src/templates/react-email.js index f8cfbc1..d0f2edb 100644 --- a/src/templates/react-email.js +++ b/src/templates/react-email.js @@ -42,6 +42,8 @@ const createReactEmailEngine = (options = {}) => { return { name: 'react-email', extensions: [], + // Its HTML, text and subject at once: a renderer with a layout refuses it. + parts: true, loader: createMemoryLoader(entries), compile(entry, context) { if (context.part !== 'html') { diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 7549683..7e46812 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -138,7 +138,19 @@ const createRendererConfig = (options) => { const partials = options.partials ?? []; if (!Array.isArray(partials)) throw new TypeError('partials must be an array of names.'); for (const directory of partials) checkTemplateName(directory, 'A directory of partials'); - if (options.layout !== undefined) checkTemplateName(options.layout, 'layout'); + if (options.layout !== undefined) { + checkTemplateName(options.layout, 'layout'); + // A layout wraps the HTML a template gives as text: one whose engine + // gives its parts at once would go without it. + for (const engine of engines) { + if (engine.parts === true) { + throw new TypeError( + `The ${engine.name} engine gives its parts at once, which no layout wraps: ` + + 'give it a renderer without one.', + ); + } + } + } const globals = options.globals; if (globals !== undefined && typeof globals !== 'function' && typeof globals !== 'object') { throw new TypeError('globals must be an object or a function.'); @@ -590,6 +602,16 @@ const renderTemplate = async (state, checked, input, bounds) => { if (resolution.html === undefined) text = decodeEntities(output); else html = output; } else if (isParts(output)) { + if (layoutNameOf(config, main) !== undefined) { + throw new TemplateRenderError( + main.name, + 'layout', + 'It gave its parts, which no layout wraps.', + { + engine: main.engine.name, + }, + ); + } ({ html, text, subject } = output); } else { throw new TemplateRenderError(main.name, 'render', 'The template gave no string.', { diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index 12bb687..e673dc5 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -125,6 +125,7 @@ createMessage({ content: fields.content, }); createTemplateRenderer({ engines: [hbs, react], loader: createMemoryLoader({ welcome: '

    ' }) }); +expectType(react.parts); expectError(templates.render({ context: {} })); // Typed templates: each name with its own context. diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 6c263eb..2951a90 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -1008,6 +1008,8 @@ test('Templates: React Email components are templates, each with its subject', a }, }); const renderer = createTemplateRenderer({ engine }); + assert.strictEqual(engine.parts, true); + assert.throws(() => createTemplateRenderer({ engine, layout: 'main' }), /no layout wraps/); const en = await renderer.render({ template: 'welcome', context: { name: 'Ada' } }); assert.match(en.html, /

    Hi Ada<\/p>/); assert.strictEqual(en.text, 'Hi Ada'); diff --git a/tests/unit/templates-renderer.test.js b/tests/unit/templates-renderer.test.js index f1a1bb1..addcc58 100644 --- a/tests/unit/templates-renderer.test.js +++ b/tests/unit/templates-renderer.test.js @@ -625,7 +625,6 @@ test('Templates: an engine that renders its parts gives HTML, text and subject a const renderer = createTemplateRenderer({ engine: parts, loader: createMemoryLoader({ welcome: 'Subject here' }), - layout: 'ignored', }); assert.deepStrictEqual(await renderer.render({ template: 'welcome', context: { n: 1 } }), { template: 'welcome', @@ -675,6 +674,33 @@ test('Templates: an engine that renders its parts gives HTML, text and subject a ); }); +test('Templates: no layout goes silently unapplied to an engine that gives its parts', async () => { + // Before: a layout was skipped for a template that gave its parts, React + // Email's, and the message went without it. + const parts = { + name: 'parts', + extensions: [], + compile: () => () => ({ html: '' }), + }; + assert.throws( + () => + createTemplateRenderer({ + engine: { ...parts, parts: true }, + loader: createMemoryLoader({}), + layout: 'main', + }), + /parts engine gives its parts at once, which no layout wraps/, + ); + const renderer = createTemplateRenderer({ + engine: parts, + loader: createMemoryLoader({ a: '', main: '' }), + layout: 'main', + }); + const error = await renderer.render({ template: 'a' }).catch((e) => e); + assert.ok(error instanceof TemplateRenderError); + assert.deepStrictEqual([error.phase, error.engine], ['layout', 'parts']); +}); + test('Templates: engines are chosen by extension, the longest first', async () => { const hbs = fakeEngine({ name: 'hbs', extensions: ['.hbs'] }); const mjml = fakeEngine({ name: 'mjml', extensions: ['.mjml.hbs'] }); From 168f4d62c4edf7a80e2cf2ca161b5690a5b0c542 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:13:18 +0300 Subject: [PATCH 57/83] fix(templates): type the loaders and the throws as they are The types gave the file and memory loaders an optional loadSync, which both always have, and named a TypeError for a locale that is no BCP 47 tag, where Intl throws a RangeError; and `autoText: false` of renderFields, which a template renderer never sees, was not said to leave its text. createFileLoader and createMemoryLoader give a SyncTemplateLoader, render and createTemplateRenderer name the RangeError, and the guide and autoText say to give it to the renderer. Co-Authored-By: Claude Opus 5.5 --- docs/messages/templates.md | 4 +++- src/templates/index.d.ts | 17 ++++++++++++++--- tests/types/templates.test-d.ts | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/messages/templates.md b/docs/messages/templates.md index 7fe8803..9e597f8 100644 --- a/docs/messages/templates.md +++ b/docs/messages/templates.md @@ -63,7 +63,9 @@ subject: Reset your password, <%= name %> `render()` gives `{ html, text, subject, template, locale }`: the text is derived from the HTML unless a text template is there, and `template` names the -file the message came from. +file the message came from. The renderer derives the text itself, so +`autoText: false` of `renderFields` leaves it; give `autoText: false` to the +renderer instead. ## Names, variants and locales diff --git a/src/templates/index.d.ts b/src/templates/index.d.ts index 1d95770..56b5681 100644 --- a/src/templates/index.d.ts +++ b/src/templates/index.d.ts @@ -83,6 +83,11 @@ export interface TemplateLoader { loadSync?(name: string): TemplateSource | undefined; } +/** A loader that reads synchronously too, as the file and memory loaders do. */ +export interface SyncTemplateLoader extends TemplateLoader { + loadSync(name: string): TemplateSource | undefined; +} + /** The options of {@link createFileLoader}. */ export interface FileLoaderOptions { /** The directory of the templates. Nothing outside it is read, through `..` or a symbolic link. */ @@ -98,7 +103,7 @@ export interface FileLoaderOptions { * * @throws {TypeError} If an option is invalid. */ -export declare function createFileLoader(options: FileLoaderOptions): TemplateLoader; +export declare function createFileLoader(options: FileLoaderOptions): SyncTemplateLoader; /** * Makes a loader of templates held in memory, by name. @@ -107,7 +112,7 @@ export declare function createFileLoader(options: FileLoaderOptions): TemplateLo */ export declare function createMemoryLoader( templates: Readonly>, -): TemplateLoader; +): SyncTemplateLoader; /** What an engine compiles: the HTML of a message, its text, or its subject. */ export type TemplatePart = 'html' | 'text' | 'subject'; @@ -273,7 +278,11 @@ export interface TemplateRendererOptions { readonly frontMatter?: boolean; /** Inlines the CSS of the HTML, after MJML and before the text. None by default. */ readonly inlineCss?: CssInliner; - /** The text from the HTML with htmlToText when there is no text template. @default true */ + /** + * The text from the HTML with htmlToText when there is no text template. + * The renderer derives it itself, so `autoText: false` of `renderFields` + * or `renderContent` leaves it: give it to the renderer. @default true + */ readonly autoText?: boolean; /** false reads and compiles every template on each render, for development. @default true */ readonly cache?: boolean | TemplateCacheOptions; @@ -318,6 +327,7 @@ export interface TemplateRenderer< * @throws {TemplateRenderError} If it does not render. * @throws {TemplateLoadError} If the loader failed to read one. * @throws {TypeError} If the input is invalid. + * @throws {RangeError} If the locale is no BCP 47 tag. */ render(input: TInput): Promise; /** Forgets compiled templates and includes; the next render reads them again. */ @@ -341,6 +351,7 @@ export interface TemplateRenderer< * ``` * @throws {TypeError} If an option is invalid, or a layout is given with an * engine that gives its parts at once. + * @throws {RangeError} If `defaultLocale` or one of `locales` is no BCP 47 tag. */ export declare function createTemplateRenderer< TInput extends TemplateInput = TemplateInput, diff --git a/tests/types/templates.test-d.ts b/tests/types/templates.test-d.ts index e673dc5..ebb5fd8 100644 --- a/tests/types/templates.test-d.ts +++ b/tests/types/templates.test-d.ts @@ -125,6 +125,7 @@ createMessage({ content: fields.content, }); createTemplateRenderer({ engines: [hbs, react], loader: createMemoryLoader({ welcome: '

    ' }) }); +expectType(createFileLoader({ dir: 'emails' }).loadSync('welcome.hbs')?.source); expectType(react.parts); expectError(templates.render({ context: {} })); From 1aa201c81082d0d8aa62365be9556dc58ef81370 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:14:43 +0300 Subject: [PATCH 58/83] refactor(templates): look up includes in one pass, as the loop rules ask A lookup built every name an include may be, in two arrays, before it tried the first, and its names were split into arrays to be joined; the renderer made array literals to iterate on every render, and tested membership of the locales tried and of the empty extension in arrays. The names are generated as they are tried, the first found ending the lookup and the list made again only for the error of none; a name is read in one pass of indexOf; the parts and options iterated are module constants, and the locales tried are deduplicated by a Set. Co-Authored-By: Claude Opus 5.5 --- src/templates/includes.js | 107 ++++++++++++++++++++++---------------- src/templates/renderer.js | 40 ++++++++------ 2 files changed, 86 insertions(+), 61 deletions(-) diff --git a/src/templates/includes.js b/src/templates/includes.js index e67f35f..2307e90 100644 --- a/src/templates/includes.js +++ b/src/templates/includes.js @@ -7,20 +7,24 @@ const { checkTemplateName, isTemplateName, notTemplateName } = require('./loader const { createCache, memoize } = require('./cache.js'); const { hasExtension } = require('./environment.js'); -// The segments of a name, `.` dropped and `..` taking the one before; undefined -// when `..` leads out of the root. -const normalize = (segments) => { - const kept = []; - for (const segment of segments) { - if (segment === '' || segment === '.') continue; +// The name a path stands for, its segments read in one pass, `.` dropped and +// `..` taking the one before; undefined when `..` leads out of the root. +const normalize = (written) => { + let name = ''; + let start = 0; + for (;;) { + const slash = written.indexOf('/', start); + const segment = written.slice(start, slash === -1 ? written.length : slash); if (segment === '..') { - if (kept.length === 0) return undefined; - kept.pop(); - } else { - kept.push(segment); + if (name === '') return undefined; + const last = name.lastIndexOf('/'); + name = last === -1 ? '' : name.slice(0, last); + } else if (segment !== '' && segment !== '.') { + name = name === '' ? segment : `${name}/${segment}`; } + if (slash === -1) break; + start = slash + 1; } - const name = kept.join('/'); return isTemplateName(name) ? name : undefined; }; @@ -29,6 +33,8 @@ const directoryOf = (name) => { return slash === -1 ? '' : name.slice(0, slash); }; +const isRelative = (include) => include.startsWith('./') || include.startsWith('../'); + /** * The name an include stands for in a template: `./x` and `../x` against the * directory of `from`, `/x` from the root, any other as it is. @@ -40,39 +46,48 @@ const directoryOf = (name) => { */ const joinName = (from, include) => { if (typeof include !== 'string' || include === '') return undefined; - if (include[0] === '/') return normalize(include.split('/')); - const relative = include.startsWith('./') || include.startsWith('../'); - if (!relative || from === undefined) return normalize(include.split('/')); - return normalize(`${directoryOf(from)}/${include}`.split('/')); + if (include[0] === '/' || !isRelative(include) || from === undefined) return normalize(include); + return normalize(`${directoryOf(from)}/${include}`); }; -// The names an include may be: as joined, then under each partials directory -// for a bare name; each with an extension of the engine unless it has one, -// then as it is. -const candidatesOf = (include, from, extensions, partials) => { - const bases = []; - const joined = joinName(from, include); - if (joined !== undefined) bases.push(joined); - const bare = include[0] !== '/' && !include.startsWith('./') && !include.startsWith('../'); - if (bare) { - for (const directory of partials) { - const name = joinName(undefined, `${directory}/${include}`); - if (name !== undefined) bases.push(name); - } +// The names a base may be: itself when it has an extension of the engine, +// else with each of them, then as it is. +const namesOf = function* (base, extensions) { + if (hasExtension(base, extensions)) { + yield base; + return; } - const names = []; - for (const base of bases) { - if (hasExtension(base, extensions)) { - names.push(base); - continue; - } - for (const extension of extensions) names.push(base + extension); - if (!extensions.includes('')) names.push(base); + let empty = false; + for (const extension of extensions) { + if (extension === '') empty = true; + yield base + extension; + } + if (!empty) yield base; +}; + +// The names an include may be, in the order they are tried: as joined, then +// under each directory of partials for a bare name, as they are tried, so +// that the first found ends the lookup. +const candidatesOf = function* (include, from, extensions, partials) { + const joined = joinName(from, include); + if (joined !== undefined) yield* namesOf(joined, extensions); + if (include[0] === '/' || isRelative(include)) return; + for (const directory of partials) { + const name = joinName(undefined, `${directory}/${include}`); + if (name !== undefined) yield* namesOf(name, extensions); } - return names; +}; + +// The names tried, for the error of an include that is not found. +const triedOf = (include, from, extensions, partials) => { + const tried = []; + for (const name of candidatesOf(include, from, extensions, partials)) tried.push(name); + return tried; }; const READ = 'The name of a template read'; +// A lookup of a name as it is. +const NO_EXTENSIONS = Object.freeze(['']); // What a loader threw, as an error that names the template: one of // ./templates as it is, and any other, such as EMFILE or a database timeout, @@ -213,29 +228,31 @@ const createIncludes = (loader, options) => { join: joinName, read, readSync, - async lookup(include, from, extensions = ['']) { + async lookup(include, from, extensions = NO_EXTENSIONS) { spend(); const depth = depths.of(include, from); - const names = candidatesOf(include, from, extensions, partials); - for (const name of names) { + for (const name of candidatesOf(include, from, extensions, partials)) { const source = await read(name); if (source === undefined) continue; depths.reached(source.name, depth); return source; } - throw new TemplateNotFoundError(include, names, { from }); + throw new TemplateNotFoundError(include, triedOf(include, from, extensions, partials), { + from, + }); }, - lookupSync(include, from, extensions = ['']) { + lookupSync(include, from, extensions = NO_EXTENSIONS) { spend(); const depth = depths.of(include, from); - const names = candidatesOf(include, from, extensions, partials); - for (const name of names) { + for (const name of candidatesOf(include, from, extensions, partials)) { const source = readSync(name); if (source === undefined) continue; depths.reached(source.name, depth); return source; } - throw new TemplateNotFoundError(include, names, { from }); + throw new TemplateNotFoundError(include, triedOf(include, from, extensions, partials), { + from, + }); }, }); return { includes, top: depths.top }; diff --git a/src/templates/renderer.js b/src/templates/renderer.js index 7e46812..8351e7f 100644 --- a/src/templates/renderer.js +++ b/src/templates/renderer.js @@ -34,6 +34,11 @@ const DEFAULT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024; // compile, is made anew by the next render rather than waited for. const DEFAULT_RENDER_TIMEOUT_MS = 30_000; const NO_LAYOUT = new Set(['none', 'false']); +const BOOLEANS = Object.freeze(['frontMatter', 'autoText']); +// The parts a template is found by, its HTML, or its text alone; and those a +// template gives. +const MAIN_PARTS = Object.freeze(['html', 'text']); +const PARTS = Object.freeze(['html', 'text', 'subject']); const defaultVariantName = (template, variant) => { const locale = variant.locale === undefined ? '' : `.${variant.locale}`; @@ -155,7 +160,7 @@ const createRendererConfig = (options) => { if (globals !== undefined && typeof globals !== 'function' && typeof globals !== 'object') { throw new TypeError('globals must be an object or a function.'); } - for (const name of ['frontMatter', 'autoText']) { + for (const name of BOOLEANS) { if (options[name] !== undefined && typeof options[name] !== 'boolean') { throw new TypeError(`${name} must be a boolean.`); } @@ -196,21 +201,24 @@ const createRendererConfig = (options) => { }; // The locales a template is looked up in: the one asked for, its subtags -// dropped from the end, then the default one so, then none. +// dropped from the end, then the default one so, each once, then none. +const addLocale = (chain, seen, tag) => { + let current = tag; + while (current !== undefined) { + if (!seen.has(current)) { + seen.add(current); + chain.push(current); + } + const dash = current.lastIndexOf('-'); + current = dash === -1 ? undefined : current.slice(0, dash); + } +}; + const localeChain = (locale, defaultLocale) => { const chain = []; - const add = (tag) => { - if (tag === undefined) return; - let current = tag; - for (;;) { - if (!chain.includes(current)) chain.push(current); - const dash = current.lastIndexOf('-'); - if (dash === -1) break; - current = current.slice(0, dash); - } - }; - add(locale); - add(defaultLocale); + const seen = new Set(); + addLocale(chain, seen, locale); + addLocale(chain, seen, defaultLocale); chain.push(undefined); return chain; }; @@ -419,7 +427,7 @@ const resolveTemplate = async (state, template, locale) => { const { base, choices } = choicesOf(state.config, template); const chain = localeChain(locale, state.config.defaultLocale); const tried = []; - for (const main of ['html', 'text']) { + for (const main of MAIN_PARTS) { for (const level of chain) { const found = await findPart(state, base, level, main, choices, tried); if (found === undefined) continue; @@ -648,7 +656,7 @@ const renderTemplate = async (state, checked, input, bounds) => { const outputOf = (config, main, resolution, parts) => { const { html } = parts; - for (const key of ['html', 'text', 'subject']) { + for (const key of PARTS) { if (parts[key] === undefined) continue; if (typeof parts[key] !== 'string') { throw new TemplateRenderError( From 9ca64ccc5ae17f948e9b531aa8b49184211197e6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:15:23 +0300 Subject: [PATCH 59/83] feat(otel): spans and metrics of the worker of a queue ./otel instrumented a transport alone: the worker of a queue had logs and an audit but no span and no metric, so the backlog, the attempts and the failures of the queue could not be graphed or alerted on, nor a send traced back to the request that enqueued it. instrumentHandler(handler, { tracerProvider, meterProvider, withSpan }) wraps any queue handler: a span mailer.queue.process per run, linked to the producer's trace, and the metrics mailer.queue.jobs (by outcome and code), mailer.queue.lag, mailer.queue.job.duration and mailer.queue.redeliveries. The mail queue's traceContext gives each job the producer's traceparent as its trace, outside its hash and signature; withSpan lets the application make the spans of an instrumented transport children of the job's, with no @opentelemetry/api required here. The preparation of a job moves out of createMailQueue. ./otel goes from 5.4 to 6.1 KB min+gzip: its budget goes from 5.5 to 6.5 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + docs/guide/queue.md | 5 + docs/transports/otel.md | 40 +++++++ scripts/size-budgets.json | 2 +- src/otel/handler.js | 144 ++++++++++++++++++++++++ src/otel/index.d.ts | 52 ++++++++- src/otel/index.js | 2 + src/queue/index.d.ts | 6 + src/queue/mail-queue.js | 34 ++++-- tests/types/otel.test-d.ts | 13 +++ tests/unit/otel-handler.test.js | 192 ++++++++++++++++++++++++++++++++ 11 files changed, 482 insertions(+), 13 deletions(-) create mode 100644 src/otel/handler.js create mode 100644 tests/unit/otel-handler.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a8455d3..0a42cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,11 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `delivered`, `bounced`, `complained`, `failed`, `delivery_unknown`) and their sinks: `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks`, `stateOf`, and `auditEventOfDeliveryEvent` for the events of `./webhooks`. +- **`./otel`**: `instrumentHandler(handler, { tracerProvider, meterProvider, withSpan })`, a span + `mailer.queue.process` per run of a job, linked to the producer's trace (the `traceparent` the + mail queue's `traceContext` gives each job as `trace`), and the metrics `mailer.queue.jobs` (by + outcome and code), `mailer.queue.lag`, `mailer.queue.job.duration` and + `mailer.queue.redeliveries`. - **`./render`**: `RenderOutput.subject` and `renderFields(renderer, input)`, which gives `{ subject, content }`. - **The root**: the contracts of the reliability layer as types (`QueueDriver`, `QueueSource`, diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 2bd072f..6dd683a 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -245,6 +245,11 @@ enqueued, rather than a job enqueued and reported as failed. A driver that writes no transaction (BullMQ, the memory queue) refuses `{ transaction }` with a `TypeError` rather than ignore it; the outbox is `transactional`. +Spans and metrics of the worker (`mailer.queue.jobs`, `mailer.queue.lag`, +`mailer.queue.job.duration`, `mailer.queue.redeliveries`) come from +[`instrumentHandler`](/transports/otel#the-worker-of-a-queue) of `./otel`, +linked to the producer's trace that `traceContext` gives each job. + `logger`, pino-like, hears what the queue does, never an address: the enqueues and the starts at `debug`, the sends at `info`, the retries, redeliveries and releases at `warn`, the failures at `error` with their code, diff --git a/docs/transports/otel.md b/docs/transports/otel.md index b403d0d..26fe1d8 100644 --- a/docs/transports/otel.md +++ b/docs/transports/otel.md @@ -316,6 +316,46 @@ showing its relationship to the broader user registration flow. This makes it easy to understand how email delivery affects overall request performance and to identify bottlenecks in your system. +## The worker of a queue + +`instrumentHandler` wraps the handler of a [queue](/guide/queue)'s worker: a +span `mailer.queue.process` per run of a job, linked to the trace of the +request that enqueued it, and the metrics of the queue. + +```js +const { context, metrics, propagation, trace } = require('@opentelemetry/api'); +const { createMailJobHandler, createMailQueue } = require('@alexify/mailer/queue'); +const { instrumentHandler } = require('@alexify/mailer/otel'); + +const queue = createMailQueue({ + driver, + // Each job carries the producer's traceparent as its `trace`. + traceContext: () => { + const carrier = {}; + propagation.inject(context.active(), carrier); + return carrier.traceparent; + }, +}); + +const handler = instrumentHandler(createMailJobHandler({ transport, renderer }), { + tracerProvider: trace.getTracerProvider(), + meterProvider: metrics.getMeterProvider(), + // The spans of an instrumented transport become children of the job's. + withSpan: (span, run) => context.with(trace.setSpan(context.active(), span), run), +}); +const consumer = driver.consume(handler, { concurrency: 4, audit }); +``` + +| Metric | Of | +| --- | --- | +| `mailer.queue.jobs` | runs, by `outcome` (`ack`, `retry`, `fail`, `release`, `thrown`) and `code` | +| `mailer.queue.lag` | ms from the enqueue of a job to the start of a run: the backlog, to alert on | +| `mailer.queue.job.duration` | ms of a run, by `outcome` and `code` | +| `mailer.queue.redeliveries` | runs again of jobs whose run before did not settle | + +The `trace` of a job is outside its hash and its signature, and stays in clear +in a sealed job. + ## Metrics and monitoring The OpenTelemetry transport automatically collects comprehensive metrics diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index c09f50b..c527b97 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -29,7 +29,7 @@ "./pool": 3.5, "./retry": 3.5, "./mock": 2, - "./otel": 5.5, + "./otel": 6.5, "./logger": 3, "./breaker": 3.5, "./ratelimit": 4, diff --git a/src/otel/handler.js b/src/otel/handler.js new file mode 100644 index 0000000..73db166 --- /dev/null +++ b/src/otel/handler.js @@ -0,0 +1,144 @@ +// Not ported: OpenTelemetry for the worker of a queue, as a handler around +// any handler: a span per run of a job, linked to the trace of the producer +// that enqueued it, and the metrics of the queue (runs by outcome, the time a +// job waited, how long a run took, the runs again of jobs whose worker died). +// The providers are passed in, and @opentelemetry/api is never required. + +const metadata = require('../version.js'); +const { checkOptionalFunction } = require('../internal/options.js'); + +// SpanKind.CONSUMER and the SpanStatusCode values of the OpenTelemetry API. +const CONSUMER = 4; +const STATUS_OK = 1; +const STATUS_ERROR = 2; +// A W3C traceparent: version, trace id, parent id and flags. +const TRACEPARENT = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/; + +/** + * The span context of a W3C traceparent, as a link takes it, or nothing. + * + * @param traceparent The text, from the job's `trace`. + * @returns `{ traceId, spanId, traceFlags, isRemote }`, or `undefined`. + * @internal + */ +const spanContextOf = (traceparent) => { + if (typeof traceparent !== 'string') return undefined; + const parts = TRACEPARENT.exec(traceparent); + if (parts === null || parts[1] === 'ff') return undefined; + if (/^0+$/.test(parts[2]) || /^0+$/.test(parts[3])) return undefined; + return { + traceId: parts[2], + spanId: parts[3], + traceFlags: Number.parseInt(parts[4], 16), + isRemote: true, + }; +}; + +const checkOptions = (handler, options) => { + if (typeof handler !== 'function') throw new TypeError('instrumentHandler needs a handler.'); + const tracing = options.tracing ?? true; + const metrics = options.metrics ?? true; + if (tracing && typeof options.tracerProvider?.getTracer !== 'function') { + throw new TypeError('instrumentHandler needs a tracerProvider while tracing is on.'); + } + if (metrics && typeof options.meterProvider?.getMeter !== 'function') { + throw new TypeError('instrumentHandler needs a meterProvider while metrics are on.'); + } + if (options.withSpan !== undefined && typeof options.withSpan !== 'function') { + throw new TypeError('withSpan must be a function.'); + } + return { tracing, metrics }; +}; + +const instrumentsOf = (meterProvider) => { + const meter = meterProvider.getMeter(metadata.name, metadata.version); + return { + jobs: meter.createCounter('mailer.queue.jobs', { + description: 'The runs of mail jobs, by outcome and code.', + }), + lag: meter.createHistogram('mailer.queue.lag', { + unit: 'ms', + description: 'How long a job waited from its enqueue to the start of a run.', + }), + duration: meter.createHistogram('mailer.queue.job.duration', { + unit: 'ms', + description: 'How long a run of a mail job took.', + }), + redeliveries: meter.createCounter('mailer.queue.redeliveries', { + description: 'The runs again of jobs whose run before did not settle.', + }), + }; +}; + +/** + * Wraps a queue handler, such as `createMailJobHandler()`, with OpenTelemetry: + * a span `mailer.queue.process` per run, linked to the producer's trace (the + * `traceparent` the mail queue's `traceContext` wrote as the job's `trace`), + * and the metrics `mailer.queue.jobs` (by `outcome` and `code`), + * `mailer.queue.lag`, `mailer.queue.job.duration` and + * `mailer.queue.redeliveries`. + * + * @param handler The handler. + * @param options `tracerProvider`, `meterProvider`; `tracing` and `metrics` + * (true) to turn either off; `withSpan(span, run)`, which runs the handler + * with the span active, so that the spans of the transport are its children + * (`(span, run) => context.with(trace.setSpan(context.active(), span), run)`); + * `now`. + * @returns The handler, instrumented. + * @throws {TypeError} If an option is invalid. + */ +const instrumentHandler = (handler, options = {}) => { + const { tracing, metrics } = checkOptions(handler, options); + const tracer = tracing + ? options.tracerProvider.getTracer(metadata.name, metadata.version) + : undefined; + const instruments = metrics ? instrumentsOf(options.meterProvider) : undefined; + const now = checkOptionalFunction(options.now, 'now') ?? Date.now; + return async (job, signal) => { + const started = now(); + const link = spanContextOf(job?.data?.trace); + const span = tracer?.startSpan('mailer.queue.process', { + kind: CONSUMER, + attributes: { + 'messaging.system': 'mailer', + 'messaging.message.id': job.id, + 'mailer.job.attempt': job.attempt, + 'mailer.job.redelivered': job.redelivered === true, + }, + links: link === undefined ? [] : [{ context: link }], + }); + if (job.redelivered) instruments?.redeliveries.add(1); + if (typeof job.enqueuedAt === 'number') instruments?.lag.record(started - job.enqueuedAt); + let outcome; + try { + outcome = + span !== undefined && options.withSpan !== undefined + ? await options.withSpan(span, () => handler(job, signal)) + : await handler(job, signal); + } catch (error) { + instruments?.jobs.add(1, { outcome: 'thrown' }); + span?.setStatus({ code: STATUS_ERROR, message: 'The handler threw.' }); + span?.end(); + throw error; + } + const attributes = { outcome: outcome?.type ?? 'none' }; + if (typeof outcome?.code === 'string') attributes.code = outcome.code; + instruments?.jobs.add(1, attributes); + instruments?.duration.record(now() - started, attributes); + if (span !== undefined) { + span.setAttributes({ + 'mailer.job.outcome': attributes.outcome, + 'mailer.job.code': attributes.code ?? '', + }); + span.setStatus( + outcome?.type === 'fail' + ? { code: STATUS_ERROR, message: attributes.code } + : { code: STATUS_OK }, + ); + span.end(); + } + return outcome; + }; +}; + +module.exports = { instrumentHandler, spanContextOf }; diff --git a/src/otel/index.d.ts b/src/otel/index.d.ts index 168d0dc..c5fe7be 100644 --- a/src/otel/index.d.ts +++ b/src/otel/index.d.ts @@ -5,6 +5,7 @@ import type { Message, + QueueHandler, RawMessage, RawTransport, Receipt, @@ -37,9 +38,24 @@ export interface Span { end(): void; } +/** The context of a span another span links to: a remote one, from a `traceparent`. */ +export interface SpanContextLike { + traceId: string; + spanId: string; + traceFlags: number; + isRemote?: boolean; +} + /** A tracer of the OpenTelemetry API. */ export interface Tracer { - startSpan(name: string, options?: { kind?: number; attributes?: Attributes }): Span; + startSpan( + name: string, + options?: { + kind?: number; + attributes?: Attributes; + links?: { context: SpanContextLike; attributes?: Attributes }[]; + }, + ): Span; } /** A tracer provider of the OpenTelemetry API, such as `trace.getTracerProvider()`. */ @@ -530,3 +546,37 @@ export declare function createErrorClassifier(options?: { readonly patterns?: Record; readonly fallback?: string; }): ErrorClassifier; + +/** The options of {@link instrumentHandler}. */ +export interface InstrumentHandlerOptions { + /** Required while `tracing` is on. */ + readonly tracerProvider?: TracerProvider; + /** Required while `metrics` are on. */ + readonly meterProvider?: MeterProvider; + /** @default true */ + readonly tracing?: boolean; + /** @default true */ + readonly metrics?: boolean; + /** + * Runs the handler with the span active, so that the spans of the + * transport are its children: + * `(span, run) => context.with(trace.setSpan(context.active(), span), run)`. + */ + readonly withSpan?: (span: Span, run: () => Promise) => Promise; + readonly now?: () => number; +} + +/** + * Wraps a queue handler, such as `createMailJobHandler()` of `./queue`, with + * OpenTelemetry: a span `mailer.queue.process` per run, linked to the trace + * of the producer (the job's `trace`, from `traceContext` of the mail queue), + * and the metrics `mailer.queue.jobs` (by `outcome` and `code`), + * `mailer.queue.lag` (ms from the enqueue to the run), `mailer.queue.job.duration` + * and `mailer.queue.redeliveries`. + * + * @throws {TypeError} If a provider is missing or an option is invalid. + */ +export declare function instrumentHandler( + handler: QueueHandler, + options?: InstrumentHandlerOptions, +): QueueHandler; diff --git a/src/otel/index.js b/src/otel/index.js index 0708a5c..8ec0776 100644 --- a/src/otel/index.js +++ b/src/otel/index.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { OpenTelemetryTransport } = require('./opentelemetry-transport.js'); +const { instrumentHandler } = require('./handler.js'); const { defaultErrorClassifier } = require('./config.js'); /** @@ -234,6 +235,7 @@ const createErrorClassifier = (options = {}) => { }; module.exports = { + instrumentHandler, OpenTelemetryTransport, defaultErrorClassifier, createOpenTelemetryTransport, diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index b16ae10..5de8634 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -382,6 +382,12 @@ export interface MailQueueOptions extends Omit { readonly auditStrict?: boolean; /** Seals the fields of each job in the store; its workers open them. */ readonly codec?: MailJobCodec; + /** + * The W3C `traceparent` of the producer, which each job carries as `trace` + * for `instrumentHandler` of `./otel` to link its span to: + * `() => { const carrier = {}; propagation.inject(context.active(), carrier); return carrier.traceparent; }`. + */ + readonly traceContext?: () => string | undefined; readonly logger?: Logger; readonly now?: () => number; } diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 8352a15..5b9e149 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -4,6 +4,7 @@ const { warnThrown } = require('../internal/callback.js'); const { createLog } = require('../internal/log.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { toUuid } = require('../internal/uuid.js'); const { checkCodec, sealMailJob } = require('./envelope.js'); const { MailQueueError } = require('./errors.js'); @@ -11,6 +12,9 @@ const { createMailJobHandler } = require('./handler.js'); const { checkIdSecret, prepareMailJob } = require('./job.js'); const { keyringOf } = require('./signing.js'); +// A W3C traceparent, as `traceContext` gives it. +const TRACEPARENT = /^[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/; + const checkDriver = (driver) => { if (typeof driver?.enqueue !== 'function' || typeof driver?.consume !== 'function') { throw new TypeError('A mail queue needs a driver, with enqueue() and consume().'); @@ -134,6 +138,7 @@ const createQueueConfig = (options) => { driver, audit, auditStrict, + traceContext: checkOptionalFunction(options.traceContext, 'traceContext'), codec: checkCodec(options.codec), now, jobOptions, @@ -141,6 +146,19 @@ const createQueueConfig = (options) => { }; }; +// The job of a message: its trace, and its data sealed when the queue has a +// codec; the driver stores the sealed data, the audit reads the clear. +const prepareJob = async (queue, input) => { + const prepared = prepareMailJob(input, queue.jobOptions); + // The producer's trace, which the span of the worker links to; outside the + // hash and the signature, as it differs for the same message. + const trace = queue.traceContext?.(); + if (typeof trace === 'string' && TRACEPARENT.test(trace)) prepared.data.trace = trace; + if (queue.codec === undefined) return prepared; + const data = await sealMailJob(prepared.data, queue.codec); + return { data: prepared.data, request: { ...prepared.request, data } }; +}; + /** * Makes a mail queue over a driver. * @@ -149,27 +167,21 @@ const createQueueConfig = (options) => { * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job * and which its workers verify; `codec`, which seals the fields of each job * in its store and opens them in its workers; `idSecret`, under which the id - * of a job is derived from its key; `logger`, `generateId`, `now`. + * of a job is derived from its key; `traceContext`, which gives the W3C + * traceparent of the producer each job carries; `logger`, `generateId`, + * `now`. * @returns `{ driver, enqueue, enqueueMany, createWorker }`. * @throws {TypeError} If an option is invalid. */ const createMailQueue = (options = {}) => { const queue = createQueueConfig(options); const { driver, codec } = queue; - // The job, its data sealed when the queue has a codec: the driver stores - // the sealed data, the audit reads the clear. - const prepare = async (input) => { - const prepared = prepareMailJob(input, queue.jobOptions); - if (codec === undefined) return prepared; - const data = await sealMailJob(prepared.data, codec); - return { data: prepared.data, request: { ...prepared.request, data } }; - }; return { driver, async enqueue(input, enqueueOptions = {}) { enqueueOptions.signal?.throwIfAborted(); const transaction = transactionOf(driver, enqueueOptions); - const prepared = await prepare(input); + const prepared = await prepareJob(queue, input); if (transaction === undefined) await appendQueued(queue, [prepared], undefined); const answer = answerOf( queue, @@ -186,7 +198,7 @@ const createMailQueue = (options = {}) => { enqueueOptions.signal?.throwIfAborted(); const transaction = transactionOf(driver, enqueueOptions); const prepared = []; - for (const input of inputs) prepared.push(await prepare(input)); + for (const input of inputs) prepared.push(await prepareJob(queue, input)); if (transaction === undefined) await appendQueued(queue, prepared, undefined); let results; if (typeof driver.enqueueMany === 'function') { diff --git a/tests/types/otel.test-d.ts b/tests/types/otel.test-d.ts index 79b861f..7034de6 100644 --- a/tests/types/otel.test-d.ts +++ b/tests/types/otel.test-d.ts @@ -8,6 +8,7 @@ import { createErrorClassifier, createOpenTelemetryTransport, defaultErrorClassifier, + instrumentHandler, type MeterProvider, type OpenTelemetryConfig, type TracerProvider, @@ -43,3 +44,15 @@ expectError(new OpenTelemetryTransport(new MockTransport(), { tracerProvider: 'g expectError(new OpenTelemetryTransport(new MockTransport(), { metrics: { samplingRate: 0.5 } })); // Exporters and resource attributes belong to the SDK of the application. expectError(createOpenTelemetryTransport(new MockTransport(), { serviceName: 'billing' })); + +// The worker of a queue, instrumented. +declare const queueHandler: import('../../index.js').QueueHandler; +expectType( + instrumentHandler(queueHandler, { tracing: false, metrics: false }), +); +expectError(instrumentHandler(queueHandler, { withSpan: 1 })); +// The providers of @opentelemetry/api fit. +instrumentHandler(queueHandler, { + tracerProvider: trace.getTracerProvider(), + meterProvider: metrics.getMeterProvider(), +}); diff --git a/tests/unit/otel-handler.test.js b/tests/unit/otel-handler.test.js new file mode 100644 index 0000000..aa6387d --- /dev/null +++ b/tests/unit/otel-handler.test.js @@ -0,0 +1,192 @@ +// Not ported: tests of instrumentHandler of ./otel, the span and metrics of +// each run of a job, and of the trace context a mail queue gives its jobs. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { instrumentHandler } = require('../../src/otel/index.js'); +const { spanContextOf } = require('../../src/otel/handler.js'); +const { createMailQueue, createMemoryQueue, parseMailJob } = require('../../src/queue/index.js'); +const { MockMeterProvider } = require('../helpers/otel/mock-otel-providers.js'); + +const TRACEPARENT = '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01'; + +// A tracer that keeps the options of each span it starts. +const recordingTracer = () => { + const spans = []; + return { + spans, + getTracer: () => ({ + startSpan(name, options) { + const span = { + name, + options, + attributes: {}, + ended: false, + setAttributes(attributes) { + Object.assign(span.attributes, attributes); + }, + addEvent() {}, + setStatus(status) { + span.status = status; + }, + end() { + span.ended = true; + }, + }; + spans.push(span); + return span; + }, + }), + }; +}; + +const recordsOf = (meterProvider, name) => { + for (const record of meterProvider.getAllRecords()) { + if (record.name === name) return record.records; + } + return []; +}; + +const jobOf = (extra = {}) => ({ + id: 'job-1', + name: 'mail', + data: { v: 1, trace: TRACEPARENT }, + attempt: 1, + maxAttempts: 5, + enqueuedAt: 1000, + redelivered: false, + ...extra, +}); + +test('OTel handler: a run is a span linked to its producer, and counted by outcome', async () => { + const tracerProvider = recordingTracer(); + const meterProvider = new MockMeterProvider(); + let clock = 5000; + const handler = instrumentHandler( + async () => { + clock += 250; + return { type: 'ack', result: { provider: 'p' } }; + }, + { tracerProvider, meterProvider, now: () => clock }, + ); + const outcome = await handler(jobOf(), new AbortController().signal); + assert.deepStrictEqual(outcome, { type: 'ack', result: { provider: 'p' } }); + const [span] = tracerProvider.spans; + assert.strictEqual(span.name, 'mailer.queue.process'); + assert.strictEqual(span.options.kind, 4); + assert.deepStrictEqual(span.options.links, [ + { + context: { + traceId: '0af7651916cd43dd8448eb211c80319c', + spanId: 'b7ad6b7169203331', + traceFlags: 1, + isRemote: true, + }, + }, + ]); + assert.strictEqual(span.attributes['mailer.job.outcome'], 'ack'); + assert.deepStrictEqual(span.status, { code: 1 }); + assert.ok(span.ended); + assert.deepStrictEqual(recordsOf(meterProvider, 'mailer.queue.jobs'), [ + { value: 1, attributes: { outcome: 'ack' } }, + ]); + assert.deepStrictEqual(recordsOf(meterProvider, 'mailer.queue.lag'), [ + { value: 4000, attributes: undefined }, + ]); + assert.deepStrictEqual(recordsOf(meterProvider, 'mailer.queue.job.duration'), [ + { value: 250, attributes: { outcome: 'ack' } }, + ]); +}); + +test('OTel handler: a failure, a run again and a throw are told', async () => { + const tracerProvider = recordingTracer(); + const meterProvider = new MockMeterProvider(); + const failing = instrumentHandler( + async () => ({ type: 'fail', code: 'x.rejected', reason: 'no' }), + { tracerProvider, meterProvider }, + ); + await failing(jobOf({ redelivered: true, data: { v: 1 } }), new AbortController().signal); + const [span] = tracerProvider.spans; + assert.deepStrictEqual(span.status, { code: 2, message: 'x.rejected' }); + assert.deepStrictEqual(span.options.links, []); + assert.strictEqual(recordsOf(meterProvider, 'mailer.queue.redeliveries').length, 1); + assert.deepStrictEqual(recordsOf(meterProvider, 'mailer.queue.jobs')[0].attributes, { + outcome: 'fail', + code: 'x.rejected', + }); + const throwing = instrumentHandler( + async () => { + throw new Error('bug'); + }, + { tracerProvider, meterProvider }, + ); + await assert.rejects(throwing(jobOf(), new AbortController().signal), /bug/); + assert.strictEqual(tracerProvider.spans[1].status.code, 2); + assert.ok(tracerProvider.spans[1].ended); +}); + +test('OTel handler: withSpan runs the handler with its span; each side can be off', async () => { + const tracerProvider = recordingTracer(); + const seen = []; + const handler = instrumentHandler(async () => ({ type: 'ack' }), { + tracerProvider, + metrics: false, + withSpan: (span, run) => { + seen.push(span.name); + return run(); + }, + }); + await handler(jobOf(), new AbortController().signal); + assert.deepStrictEqual(seen, ['mailer.queue.process']); + const bare = instrumentHandler(async () => ({ type: 'release' }), { + tracing: false, + metrics: false, + }); + assert.deepStrictEqual(await bare(jobOf(), new AbortController().signal), { type: 'release' }); + assert.throws(() => instrumentHandler(), /needs a handler/); + assert.throws(() => instrumentHandler(async () => {}), /tracerProvider/); + assert.throws(() => instrumentHandler(async () => {}, { tracing: false }), /meterProvider/); + assert.throws( + () => instrumentHandler(async () => {}, { tracing: false, metrics: false, withSpan: 1 }), + /withSpan/, + ); +}); + +test('OTel handler: only a valid traceparent is a link', () => { + assert.strictEqual(spanContextOf(undefined), undefined); + assert.strictEqual(spanContextOf('nope'), undefined); + assert.strictEqual(spanContextOf(TRACEPARENT.replace(/^00/, 'ff')), undefined); + assert.strictEqual(spanContextOf(`00-${'0'.repeat(32)}-b7ad6b7169203331-01`), undefined); + assert.strictEqual( + spanContextOf(`00-0af7651916cd43dd8448eb211c80319c-${'0'.repeat(16)}-01`), + undefined, + ); + assert.strictEqual(spanContextOf(TRACEPARENT).traceFlags, 1); +}); + +test('OTel handler: a mail queue gives each job the producer’s trace, outside its hash and signature', async () => { + let current = TRACEPARENT; + const key = { id: 'k', secret: 'k'.repeat(32) }; + const queue = createMailQueue({ + driver: createMemoryQueue(), + signingKeys: [key], + traceContext: () => current, + }); + const input = { + kind: 'content', + subject: 's', + content: { text: 't' }, + from: 'app@example.com', + to: 'ada@example.com', + }; + await queue.enqueue({ ...input, idempotencyKey: 'a' }); + current = 'not a traceparent'; + await queue.enqueue({ ...input, idempotencyKey: 'b' }); + const [traced, untraced] = queue.driver.list(); + assert.strictEqual(traced.data.trace, TRACEPARENT); + assert.strictEqual(untraced.data.trace, undefined); + assert.strictEqual(parseMailJob(traced.data, { signingKeys: [key] }).trace, TRACEPARENT); + assert.throws( + () => createMailQueue({ driver: createMemoryQueue(), traceContext: 1 }), + /traceContext/, + ); +}); From 37c9762520306fe0bb3e321b256acac740b44a88 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:16:32 +0300 Subject: [PATCH 60/83] test(templates): cover an include gone between its lookup and its read Liquid, Eta and Pug look an include up, then read it, and without a cache each read asks the loader: an include deleted in between is a missing template, which no test reached. Eta's cacher keeps define and get alone, the two Eta calls, as the engine registers no template of its own. Co-Authored-By: Claude Opus 5.5 --- src/templates/cache.js | 3 --- src/templates/eta.js | 8 ++------ tests/unit/templates-engines.test.js | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/templates/cache.js b/src/templates/cache.js index 9c91a73..2f97ddb 100644 --- a/src/templates/cache.js +++ b/src/templates/cache.js @@ -32,9 +32,6 @@ const createCache = (maxEntries, onDrop) => { onDrop?.(oldest); } }, - delete(key) { - entries.delete(key); - }, // Forgets an entry, unless another value took its place meanwhile. forget(key, value) { if (entries.get(key) === value) entries.delete(key); diff --git a/src/templates/eta.js b/src/templates/eta.js index 9a4329e..866b9aa 100644 --- a/src/templates/eta.js +++ b/src/templates/eta.js @@ -20,17 +20,13 @@ const UNESCAPED = new Map([ ]); // A Cacher of Eta, which keeps every template it includes, as it renders, -// bounded as the caches of the renderer are. +// bounded as the caches of the renderer are: Eta defines and gets alone, as +// the engine registers no template of its own. const cacherOf = (maxEntries) => { const cache = createCache(maxEntries); return { define: (key, value) => cache.set(key, value), get: (key) => cache.get(key), - remove: (key) => cache.delete(key), - reset: () => cache.clear(), - load(record) { - for (const key of Object.keys(record)) cache.set(key, record[key]); - }, }; }; diff --git a/tests/unit/templates-engines.test.js b/tests/unit/templates-engines.test.js index 2951a90..0b05625 100644 --- a/tests/unit/templates-engines.test.js +++ b/tests/unit/templates-engines.test.js @@ -983,6 +983,33 @@ test('Templates: a loader that fails under an engine fails as a load, naming the } }); +test('Templates: an include that is gone between its lookup and its read is missing', async () => { + // Without a cache, each read asks the loader: the include is there when + // the engine looks it up, and gone when it reads it. + const cases = [ + [createLiquidEngine({ Liquid }), 'a.liquid', '{% render "b" %}', 'b.liquid'], + [createEtaEngine({ Eta }), 'a.eta', '<%~ include("b") %>', 'b.eta'], + [createPugEngine({ pug }), 'a.pug', 'include b', 'b.pug'], + ]; + for (const [engine, name, source, include] of cases) { + let reads = 0; + const read = (asked) => { + if (asked === name) return { name, source }; + if (asked !== include) return undefined; + reads++; + return reads === 1 ? { name: include, source: 'b' } : undefined; + }; + const renderer = createTemplateRenderer({ + engine, + loader: { load: read, loadSync: read }, + cache: false, + }); + const error = await failureOf(renderer.render({ template: 'a' })); + assert.ok(error instanceof TemplateNotFoundError, engine.name); + assert.deepStrictEqual([error.template, error.tried], [include, [include]], engine.name); + } +}); + test('Templates: an engine that reads synchronously needs a loader with loadSync', async () => { const loader = { load: (name) => (name === 'a.ejs' ? { name, source: '<%- include("b") %>' } : undefined), From c670544a70e45ed8c640abe86beb57158356777d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:19:59 +0300 Subject: [PATCH 61/83] chore: run the suites of the real stores as one step that cannot be skipped pnpm test:mongodb and pnpm test:bullmq skip their tests without a store, so a release could pass them unrun, and no CI runs them. pnpm test:stores runs both, serial, with a timeout per test, and fails when MONGODB_URL or BULLMQ_REDIS_URL is not given. It is a step of the release, by hand. Co-Authored-By: Claude Opus 5.5 --- package.json | 1 + scripts/test-stores.js | 39 ++++++++++++++++++++++++++++++++++++++ tests/unit/scripts.test.js | 18 ++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 scripts/test-stores.js diff --git a/package.json b/package.json index 199fe48..d4b2102 100644 --- a/package.json +++ b/package.json @@ -210,6 +210,7 @@ "test:e2e": "node --test --test-concurrency=1 tests/e2e/*.test.js", "test:bullmq": "node --test --test-concurrency=1 tests/bullmq/*.test.js", "test:mongodb": "node --test --test-concurrency=1 tests/mongodb/*.test.js", + "test:stores": "node scripts/test-stores.js", "test:coverage": "c8 --all --include 'src/**' --check-coverage --lines 98 --statements 98 --branches 90 --functions 100 --reporter text --reporter lcov node --test tests/unit/*.test.js", "test:types": "tsd", "check:dts": "node scripts/check-dts.js", diff --git a/scripts/test-stores.js b/scripts/test-stores.js new file mode 100644 index 0000000..dcc5f47 --- /dev/null +++ b/scripts/test-stores.js @@ -0,0 +1,39 @@ +// Runs the suites of the real stores, tests/mongodb and tests/bullmq, as a +// step of a release: unlike `pnpm test:mongodb` and `pnpm test:bullmq`, which +// skip their tests without a store, it fails when either is not given, so a +// release cannot pass them unrun. +// +// docker compose -f tests/mongodb/docker-compose.yml up -d --wait +// docker compose -f tests/bullmq/docker-compose.yml up -d --wait +// MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0' \ +// BULLMQ_REDIS_URL=redis://localhost:6390 pnpm test:stores + +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); + +const ROOT = path.join(__dirname, '..'); +const SUITES = [ + { variable: 'MONGODB_URL', directory: 'tests/mongodb' }, + { variable: 'BULLMQ_REDIS_URL', directory: 'tests/bullmq' }, +]; + +const missing = []; +for (const suite of SUITES) { + if ((process.env[suite.variable] ?? '') === '') missing.push(suite.variable); +} +if (missing.length > 0) { + console.error( + `test:stores needs ${missing.join(' and ')}: start the stores of tests/mongodb and ` + + 'tests/bullmq (docker compose), and give their URLs.', + ); + process.exit(1); +} + +for (const suite of SUITES) { + const result = spawnSync( + process.execPath, + ['--test', '--test-concurrency=1', '--test-timeout=120000', `${suite.directory}/*.test.js`], + { cwd: ROOT, stdio: 'inherit' }, + ); + if (result.status !== 0) process.exit(result.status ?? 1); +} diff --git a/tests/unit/scripts.test.js b/tests/unit/scripts.test.js index 01c1a90..6e6dfb1 100644 --- a/tests/unit/scripts.test.js +++ b/tests/unit/scripts.test.js @@ -163,3 +163,21 @@ test('Scripts: check-dts compiles the types of every entry of the exports map', ); assert.strictEqual(typesOf(pkg.exports)[0], './index.d.ts'); }); + +test('Scripts: test:stores fails rather than skip when a store is not given', () => { + const env = { ...process.env, MONGODB_URL: '', BULLMQ_REDIS_URL: '' }; + const result = spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-stores.js')], { + cwd: ROOT, + encoding: 'utf8', + env, + }); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /needs MONGODB_URL and BULLMQ_REDIS_URL/); + const half = spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-stores.js')], { + cwd: ROOT, + encoding: 'utf8', + env: { ...env, MONGODB_URL: 'mongodb://localhost:1' }, + }); + assert.strictEqual(half.status, 1); + assert.match(half.stderr, /needs BULLMQ_REDIS_URL:/); +}); From afa9cae4c34479b71bb9d1900f0886784eb4257c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:19:59 +0300 Subject: [PATCH 62/83] chore(bench): measure the throughput of the outbox bench/outbox.js runs the outbox on a replica set (MONGODB_URL), a handler that acknowledges at once, for 1 and 4 workers of 1 and 10 lanes, with and without a change stream, and counts the round trips of each job through a proxy of the collection. The guide of the outbox gives its numbers, against those of the code before the review (4 round trips a job, 837 jobs a second for 1 x 10, now 2 and 2308), and when to take BullMQ instead. Co-Authored-By: Claude Opus 5.5 --- bench/outbox.js | 118 +++++++++++++++++++++++++++++++++++++++++++ docs/guide/outbox.md | 23 +++++++++ 2 files changed, 141 insertions(+) create mode 100644 bench/outbox.js diff --git a/bench/outbox.js b/bench/outbox.js new file mode 100644 index 0000000..2f290e7 --- /dev/null +++ b/bench/outbox.js @@ -0,0 +1,118 @@ +/** + * The throughput of the MongoDB outbox: how many jobs a second its workers + * take and settle, and how many round trips each job costs, for 1 and 4 + * workers of 1 and 10 lanes, with and without a change stream. The handler + * acknowledges at once, so what is measured is the queue alone, not a send. + * + * Run (manual only, outside the gates): + * docker compose -f tests/mongodb/docker-compose.yml up -d --wait + * MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0' node bench/outbox.js [jobs] + * + * Each worker is an outbox of its own over the collection, as a process of + * its own would be, its caches and its change stream apart from the others. + */ + +const { randomUUID } = require('node:crypto'); +const { MongoClient } = require('mongodb'); +const { createMongoOutbox } = require('../src/mongodb/index.js'); + +const URL = process.env.MONGODB_URL; +const JOBS = Number(process.argv[2] ?? 2000); + +// A collection that counts the round trips made through it. +const counting = (collection) => { + const counts = { total: 0 }; + return { + counts, + collection: new Proxy(collection, { + get(target, property) { + const value = target[property]; + if (typeof value !== 'function') return value; + return (...args) => { + counts.total += 1; + counts[property] = (counts[property] ?? 0) + 1; + return value.apply(target, args); + }; + }, + }), + }; +}; + +const requestOf = (index) => ({ + id: randomUUID(), + name: 'mail', + data: { v: 1, n: index }, + hash: `h${index}`, + priority: index % 10 === 0 ? 'high' : 'normal', + maxAttempts: 5, + enqueuedAt: Date.now(), + unique: true, +}); + +const run = async (db, { workers, lanes, changeStream }) => { + const raw = db.collection(`outbox_${randomUUID().replaceAll('-', '')}`); + const { collection, counts } = counting(raw); + const setup = createMongoOutbox({ collection: raw }); + await setup.ensureIndexes(); + for (let index = 0; index < JOBS; index++) await setup.source.enqueue(requestOf(index)); + let done = 0; + let finished; + const all = new Promise((resolve) => { + finished = resolve; + }); + const handler = async () => { + done += 1; + if (done === JOBS) finished(); + return { type: 'ack' }; + }; + counts.total = 0; + const started = performance.now(); + const consumers = []; + for (let worker = 0; worker < workers; worker++) { + const outbox = createMongoOutbox({ collection, changeStream, pollIntervalMilliseconds: 50 }); + consumers.push(outbox.consume(handler, { concurrency: lanes })); + } + await all; + const seconds = (performance.now() - started) / 1000; + for (const consumer of consumers) await consumer.close({ timeoutMilliseconds: 1000 }); + await raw.drop(); + return { jobsPerSecond: Math.round(JOBS / seconds), roundTrips: counts.total / JOBS }; +}; + +const main = async () => { + if (URL === undefined || URL === '') { + console.error('bench/outbox.js needs MONGODB_URL: a replica set, as for pnpm test:mongodb.'); + process.exit(1); + } + const client = new MongoClient(URL); + await client.connect(); + const db = client.db(`mailer_bench_${randomUUID().replaceAll('-', '')}`); + const rows = []; + try { + for (const changeStream of [false, true]) { + for (const workers of [1, 4]) { + for (const lanes of [1, 10]) { + const result = await run(db, { workers, lanes, changeStream }); + rows.push({ workers, lanes, changeStream, ...result }); + } + } + } + } finally { + await db.dropDatabase(); + await client.close(); + } + console.log(`${JOBS} jobs, acknowledged at once\n`); + console.log('| workers | lanes | change stream | jobs/s | round trips a job |'); + console.log('| --- | --- | --- | --- | --- |'); + for (const row of rows) { + console.log( + `| ${row.workers} | ${row.lanes} | ${row.changeStream ? 'yes' : 'no'} | ` + + `${row.jobsPerSecond} | ${row.roundTrips.toFixed(1)} |`, + ); + } +}; + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/docs/guide/outbox.md b/docs/guide/outbox.md index bac6ccc..239eb1b 100644 --- a/docs/guide/outbox.md +++ b/docs/guide/outbox.md @@ -93,6 +93,29 @@ only the fallback; the outbox's own leases and settlings wake none. A stream that fails is opened again a second later, then up to every 30 seconds (`queue.change_stream_failed`). +## Throughput + +`bench/outbox.js` measures the queue alone (a handler that acknowledges at +once) on a replica set of one, MongoDB 7 in Docker on a laptop: numbers to +compare on one machine, not a promise. + +| Workers × lanes | Jobs a second (change stream) | Round trips a job | +| --- | --- | --- | +| 1 × 1 | 354 (261) | 2.1 | +| 1 × 10 | 2308 (2214) | 2.0 | +| 4 × 1 | 1081 (1084) | 2.1 | +| 4 × 10 | 4723 (4707) | 2.1 | + +A job costs its lease and its settling; before 1.2.0 was released, the same +bench gave 4 round trips a job and 837 jobs a second for 1 × 10. A send through +a provider takes tens to hundreds of milliseconds, so the outbox is rarely what +limits a worker: past a few thousand jobs a second, or for delays and rate +limits at that scale, take [BullMQ](/transports/bullmq) on Valkey. + +```bash +MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0' node bench/outbox.js 2000 +``` + ## Indexes and retention `ensureIndexes()` makes: From a2474f39eff50a0af8ec93694e8f923dd0e39362 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:20:00 +0300 Subject: [PATCH 63/83] docs: keep CLAUDE.md and the release skill true to 1.2 CLAUDE.md still said the outbox was to implement the idempotency store, that scheduling through the queue was to come and nothing of the audit was done, and named no page of 1.2; the release skill asked for test:e2e and test:mailpit but not the stores, though CLAUDE.md said it did. Both now say what 1.2 holds, and the release process adds the gates no CI runs (test:stores, Bun and Deno where installed, the bench of the outbox), the merge before the tag on main, and the manual publish. A commit per subpath and per fix is a rule of the conventions. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/npm-publish/SKILL.md | 16 +++++++++ CLAUDE.md | 52 ++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.claude/skills/npm-publish/SKILL.md b/.claude/skills/npm-publish/SKILL.md index 2b09346..bb5aca0 100644 --- a/.claude/skills/npm-publish/SKILL.md +++ b/.claude/skills/npm-publish/SKILL.md @@ -193,6 +193,19 @@ pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run tes Must exit 0. If it fails, fix and re-run. +Then run the stores, which no CI runs and a release may not skip: + +```bash +docker compose -f tests/mongodb/docker-compose.yml up -d --wait +docker compose -f tests/bullmq/docker-compose.yml up -d --wait +MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0' BULLMQ_REDIS_URL=redis://localhost:6390 pnpm test:stores +``` + +`test:stores` fails without either variable, rather than skipping. When the outbox changed, run +its bench (`MONGODB_URL=… node bench/outbox.js`) and compare with the numbers in +`docs/guide/outbox.md`. Run `pnpm test:bun` and `pnpm test:deno` where Bun and Deno are +installed. + Then ask the user whether to run the opt-in suites against the real services, which only they can: `pnpm test:e2e` with the sandbox credentials of the providers (each suite is skipped until its variables are set; `tests/e2e/providers.e2e.test.js` names those of the providers of 1.1), @@ -200,6 +213,9 @@ and `pnpm test:mailpit` against the local Mailpit. A release that changed a prov its error handling should have run that provider's suite. Report which suites ran and which were skipped. +The release is published by hand (`pnpm run release`), after the merge into `main` and the tag +on the commit of `main`; no workflow publishes. + ## Step 8: Summary & Next Steps Present a summary to the user: diff --git a/CLAUDE.md b/CLAUDE.md index 4efa1ce..415d725 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,11 +19,14 @@ Nodemailer, emailjs or Upyo, made of two parts: `docs/comparison.json` has: a persistent queue and outbox, an append-only audit log, one model of delivery events, suppression, idempotency and failover between providers. 1.1 has the model of delivery events with the verified webhooks of fifteen providers (`./webhooks`) and the - idempotency decorator whose store interface the outbox is to implement (`./idempotency`). + idempotency decorator (`./idempotency`), whose store the outbox of 1.2 implements. 1.2 adds the queue (`./queue`, a driver-neutral contract, its worker and a memory queue; `./bullmq`, the BullMQ driver; `./mongodb`, the transactional outbox with its audit sink and - idempotency store), templates by name over nine engines passed in (`./templates`) and the - audit of the states of a message (`./audit`). + idempotency store), templates by name over nine engines passed in (`./templates`), the + audit of the states of a message (`./audit`) and the spans and metrics of the worker + (`instrumentHandler` of `./otel`). Jobs are signed (HMAC) and may be sealed (a codec), their + message dropped once done with; sources hold one contract (leases, redelivery, fencing, a + limit of runs that end unsettled), with tenants in turn and a pause per tenant. It has zero runtime dependencies and runs on Node.js ≥ 22, and on Bun and Deno through their Node compatibility (see "Platform"; no browser entry). @@ -52,6 +55,7 @@ pnpm test:mailpit # opt-in: tests/mailpit against a local Mailp pnpm test:e2e # opt-in: tests/e2e against real providers, from env variables pnpm test:bullmq # opt-in: tests/bullmq on BullMQ and Valkey (compose file there) pnpm test:mongodb # opt-in: tests/mongodb on a MongoDB replica set (compose file there) +pnpm test:stores # both above, failing (not skipping) without MONGODB_URL and BULLMQ_REDIS_URL node --test tests/unit/mailer.test.js # a single file node --test --test-name-pattern="Bundle" tests/unit/*.test.js # filter by name pnpm run test:coverage # c8 over src/, gated at 98 lines/statements, 90 branches, 100 functions @@ -913,12 +917,14 @@ model of states and events. `tenantId` is in the context from the start. audit sink and idempotency store, and the memory queue.) - **The audit log** of every state, with a keyed HMAC plus an encrypted block for the recipient (crypto-shredding), so the right to erasure does not fight an append-only log. 3–6 records per - message; optional hash chain for tamper evidence. + message; optional hash chain for tamper evidence. (Partly done in 1.2: `./audit` and the + MongoDB sink, events with no recipient; the crypto-shredding and the hash chain are next.) - **Idempotency:** a deterministic key (tenant + purpose + recipient + window) deduplicated in the outbox. At least once with deduplication, never a promise of exactly once. A provider without an idempotency key (SES) leads to `delivery_unknown` and reconciliation from events, not to a blind - retry. `./idempotency` (1.1) reserves keys in a store of `reserve`, `complete` and `release`, - which the outbox is to implement. + retry. `./idempotency` (1.1) reserves keys in a store of `reserve`, `complete` and `release`; + `createMongoIdempotencyStore` (1.2) is that store in MongoDB, and `redelivered: + 'delivery_unknown'` of the worker stops a blind retry where no key is taken. - **Errors classified:** permanent (5xx, rejected address) are not retried; temporary ones are, with jitter and `Retry-After`, under an attempt limit. - **The correctness suite:** golden `.eml` files, DKIM checked by an independent verifier in CI, @@ -939,8 +945,9 @@ store of buckets that processes share). **Phase 3, multi-tenant (P1–P2).** Provider credentials per tenant (bring your own SES), domains and DKIM per tenant, quotas; List-Unsubscribe one-click (RFC 8058, done in 1.1) and the split -between transactional and bulk mail; scheduled sending as a queue delay (1.1 schedules through the -providers that can); a `Renderer` interface with an optional React Email adapter (no template +between transactional and bulk mail (tenants in turn and a pause per tenant in 1.2); scheduled +sending as a queue delay (done in 1.2, `scheduledAt`; 1.1 schedules through the providers that +can); a `Renderer` interface with an optional React Email adapter (no template engine of its own; done in 1.1, `./render`); more providers (done in 1.1). **Decisions:** @@ -1045,6 +1052,8 @@ separate entry with no `node:*`, and a platform test would enforce it as in prot - Built-ins with the `node:` prefix; relative requires always end in `.js`. - Self-descriptive code; comments explain *why*, not what. - Conventional Commits (`feat(scope):`, `fix:`, `docs:`, `chore:`, `ci:`, `test:`, `refactor!:`). + A commit per subpath and per fix: a feature that spans subpaths lands as a commit for each, so + that a review reads one at a time (the 1.2 commit of four subpaths, 88 files, was not). ## Skills @@ -1098,8 +1107,10 @@ change. **This file wins** if they ever disagree. non-retryable `auth` failure) and `tests/bullmq/` (`pnpm test:bullmq`, serial; `docker compose -f tests/bullmq/docker-compose.yml up -d` and `BULLMQ_REDIS_URL=redis://localhost:6390`: duplicates, retries after `Retry-After`, priorities, - delays, a release on close and a Worker without the strategy, on BullMQ itself). The release - skill asks for them before a release. + delays, a release on close and a Worker without the strategy, on BullMQ itself). + `pnpm test:stores` runs `tests/mongodb` and `tests/bullmq` and fails without their variables, + rather than skipping: it is a step of every release (see "Release process"), as GitHub CI does + not run the stores. The release skill asks for all of them before a release. - **The template engines, BullMQ, ioredis and mongodb are devDependencies** for the tests of `./templates`, `./bullmq` and `./mongodb` (the real engines, a fake BullMQ in `tests/helpers/bullmq/`, a collection in memory in `tests/helpers/mongodb/`, and tsd against @@ -1107,7 +1118,9 @@ change. **This file wins** if they ever disagree. serial; `docker compose -f tests/mongodb/docker-compose.yml up -d --wait` and `MONGODB_URL='mongodb://localhost:27027/?replicaSet=rs0'`) runs the outbox, the audit sink and the store on a replica set: transactions committed and aborted, workers of several processes, - leases, concurrent reserves of a key and the change stream. + leases, concurrent reserves of a key, the change stream, the plan of the next-due query + (`explain()`), and the contract of a source (`tests/helpers/queue/source-contract.js`), which + `pnpm test` also runs on the memory queue and a fake collection, whole and under fairness. - **Platform-dependent tests:** the mock SMTP server takes a client that has gone as no failure: a reset (`ECONNRESET`; macOS and Windows send an RST where Linux sends a FIN) and a reply written after the client closed (`EPIPE` on Linux, `ECONNABORTED` on Windows), which the 221 to @@ -1155,19 +1168,26 @@ published to npm, and lint/format do not cover it. examples in CommonJS on `@alexify/mailer/`. `/transports/logger` is new (pino-like logger) and `/transports/otel` passes the providers in. The pages of 1.1 are new: `/messages/render`, `/guide/webhooks`, a page per provider and decorator, and Bun and Deno in - getting started. One sidebar (`guide` in `config.mts`) covers the guide, the messages, the - transports and the decorators. + getting started. Those of 1.2 are `/guide/queue`, `/guide/outbox`, `/guide/audit`, + `/messages/templates` and `/transports/bullmq` (a "Reliability" group). One sidebar (`guide` + in `config.mts`) covers the guide, the messages, the transports and the decorators. - When behavior changes, update README and the matching docs page together. Keep example outputs in the docs true to the code. ## Release process -Manual, as in the sibling repositories. The `npm-publish` skill walks through it step by step: +Manual, as in the sibling repositories: no workflow publishes, and GitHub CI is not changed for +it. The `npm-publish` skill walks through it step by step: 1. Bump `version` in `package.json` and in `src/version.js` (`tests/unit/version.test.js` fails while they differ). 2. Turn `## [Unreleased]` in CHANGELOG.md into `## [X.Y.Z] - YYYY-MM-DD` and add its link reference. -3. Commit, then tag `vX.Y.Z`. -4. Run `pnpm run release` (`pnpm publish`, gated by `prepublishOnly`, which includes the size +3. Run the gates no CI runs: `pnpm test:stores` on the compose files of `tests/mongodb` and + `tests/bullmq`; `pnpm test:bun` and `pnpm test:deno` where they are installed; the bench of the + outbox (`MONGODB_URL=… node bench/outbox.js`) when the outbox changed; and, as before, + `pnpm test:mailpit` and the `pnpm test:e2e` suites of the providers that changed. +4. Commit, merge into `main` (a squash merge, as #2 and #3), then tag `vX.Y.Z` on the commit of + `main`, never on the feature branch. +5. Run `pnpm run release` (`pnpm publish`, gated by `prepublishOnly`, which includes the size gate and the tarball check; `publishConfig.access` is public). From d06b68ba7ccccd2220536d5fbb7ee08a5fe315e2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:20:32 +0300 Subject: [PATCH 64/83] test(templates): skip the tests that replace node:fs where it cannot be replaced The tests of the file loader that replace functions of node:fs, the first unit tests to, would fail rather than test anything on a runtime whose node:fs is not writable: they tell whether each function was replaced, and skip, saying so, where it was not. Co-Authored-By: Claude Opus 5.5 --- tests/unit/templates-loader.test.js | 87 ++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index 796be2f..e8f33e3 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -14,6 +14,21 @@ const { } = require('../../src/templates/index.js'); const ejs = require('ejs'); +// Replaces functions of node:fs for a test, and tells whether it could: a +// runtime whose node:fs is not writable leaves the test to the others. +const replace = (t, pairs) => { + for (const [object, name, implementation] of pairs) { + const before = object[name]; + try { + t.mock.method(object, name, implementation); + } catch { + return false; + } + if (object[name] === before) return false; + } + return true; +}; + const makeTree = (t) => { const base = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-templates-')); t.after(() => fs.rmSync(base, { recursive: true, force: true })); @@ -70,15 +85,25 @@ test('Templates: the file loader reads no more than maxBytes of a file that grew // after, so a file that grew in between, or was replaced, was read whole. const { root } = makeTree(t); const small = { isFile: () => true, size: 10 }; - t.mock.method(fsp, 'stat', async () => small); - t.mock.method(fs, 'statSync', () => small); const open = fsp.open; - t.mock.method(fsp, 'open', async (...args) => { - const handle = await open(...args); - handle.stat = async () => small; - return handle; - }); - t.mock.method(fs, 'fstatSync', () => small); + const replaced = replace(t, [ + [fsp, 'stat', async () => small], + [fs, 'statSync', () => small], + [ + fsp, + 'open', + async (...args) => { + const handle = await open(...args); + handle.stat = async () => small; + return handle; + }, + ], + [fs, 'fstatSync', () => small], + ]); + if (!replaced) { + t.skip('node:fs cannot be replaced on this runtime'); + return; + } const loader = createFileLoader({ dir: root, maxBytes: 1024 }); await assert.rejects(loader.load('big.ejs'), /more than 1024 bytes/); assert.throws(() => loader.loadSync('big.ejs'), /more than 1024 bytes/); @@ -98,11 +123,21 @@ test('Templates: the file loader stops at the abort of its signal', async (t) => // Aborted after it found the file, before it opens it. const controller = new AbortController(); const realpath = fsp.realpath; - t.mock.method(fsp, 'realpath', async (...args) => { - const real = await realpath(...args); - if (args[0].endsWith('welcome.ejs')) controller.abort(reason); - return real; - }); + const replaced = replace(t, [ + [ + fsp, + 'realpath', + async (...args) => { + const real = await realpath(...args); + if (args[0].endsWith('welcome.ejs')) controller.abort(reason); + return real; + }, + ], + ]); + if (!replaced) { + t.skip('node:fs cannot be replaced on this runtime'); + return; + } await assert.rejects( loader.load('welcome.ejs', { signal: controller.signal }), (e) => e === reason, @@ -116,12 +151,26 @@ test('Templates: the file loader takes a directory it cannot open for none', asy const { root } = makeTree(t); const denied = (code) => Object.assign(new Error(code), { code }); let code = 'EISDIR'; - t.mock.method(fsp, 'open', async () => { - throw denied(code); - }); - t.mock.method(fs, 'openSync', () => { - throw denied(code); - }); + const replaced = replace(t, [ + [ + fsp, + 'open', + async () => { + throw denied(code); + }, + ], + [ + fs, + 'openSync', + () => { + throw denied(code); + }, + ], + ]); + if (!replaced) { + t.skip('node:fs cannot be replaced on this runtime'); + return; + } const loader = createFileLoader({ dir: root }); assert.strictEqual(await loader.load('folder.ejs'), undefined); assert.strictEqual(loader.loadSync('folder.ejs'), undefined); From 288177651337e2a290f706b73d781d78118e47b4 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:21:20 +0300 Subject: [PATCH 65/83] docs: one table of the codes of the queue, kept in step with the code The codes of the queue, its drivers and its worker (the failure of a job, the messages of its log records, its process warnings) were spread over the code, and a dashboard or an alert had no list to go by. The queue guide has the table, and tests/unit/codes.test.js fails on a code of the code it does not list, or one it lists that the code no longer has. Co-Authored-By: Claude Opus 5.5 --- docs/guide/queue.md | 65 ++++++++++++++++++++++++++++++++++++++++ tests/unit/codes.test.js | 47 +++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tests/unit/codes.test.js diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 6dd683a..202de83 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -286,3 +286,68 @@ MongoDB outbox: the order of priorities, leases, redelivery, fencing by the token, releases that spend no attempt, and the pause. The repository's `tests/helpers/queue/source-contract.js` is that contract as tests; copy it, and run it on your source (`createMemoryQueue().source` is one to compare). + +## Codes + +The codes the queue, its drivers and its worker give: in the `failure` of a job +and the `error` of its audit event, as the message of a log record, or as the +code of a process warning. `tests/unit/codes.test.js` keeps this table and the +code in step. + + + +| Code | Where | What | +| --- | --- | --- | +| `queue.invalid_job` | failure | the data of a job is not that of a mail job | +| `queue.unsupported_version` | failure | written by a later version: given back, then failed after a week | +| `queue.hash_mismatch` | failure | the job is not what was enqueued | +| `queue.signature_invalid` | failure | not signed by a key of `signingKeys` | +| `queue.codec_missing` | failure | sealed, and the worker has no codec: given back, then failed after a week | +| `queue.codec_failed` | failure | the codec could not open it | +| `queue.key_reused` | thrown | a key enqueued before for another message | +| `queue.refused` | failure | `checkJob` threw | +| `queue.expired` | failure | past `expiresAt`, or its retry would be | +| `queue.attempts_exhausted` | failure | a retryable failure on the last attempt | +| `queue.redelivery_exhausted` | failure | its runs ended unsettled more than `maxRedeliveries` times | +| `queue.redelivered` | failure | a run again after a run that may have sent it (`redelivered: 'delivery_unknown'`) | +| `queue.attachment_unresolved` | failure | an attachment by reference, and no `resolveAttachment` | +| `queue.attachment_failed` | failure | `resolveAttachment` threw or gave no content | +| `queue.invalid_message` | failure | the job does not make a message | +| `queue.invalid_receipt` | failure | the transport gave no receipt | +| `queue.aborted` | failure | the worker stopped while the message was sent | +| `delivery_unknown` | failure | the send may have gone (a timeout after the request, a key in flight) | +| `queue.thrown` | failure | a bug of the transport or of the worker | +| `queue.handler_threw` | failure | the handler of a driver threw | +| `queue.invalid_outcome` | failure | the handler gave no outcome | +| `queue.lost` | failure | BullMQ gave the job up itself (stalled too often) | +| `queue.released` | failure | a release without a token, a retry on BullMQ | +| `render.no_renderer` | failure | a template job, and the worker has no renderer | +| `render.no_subject` | failure | neither the template nor the job gives a subject | +| `render.failed` | failure | the template failed to render or was not found | +| `audit.failed` | failure | a strict sink failed before the send: retried | +| `queue.enqueued` | log | a job enqueued | +| `queue.duplicate` | log | a job enqueued again under its key | +| `queue.job.started` | log | a run starts | +| `queue.job.completed` | log | a run accepted | +| `queue.job.retrying` | log | a run to be retried | +| `queue.job.released` | log | a run given back, its attempt not spent | +| `queue.job.failed` | log | a run failed for good | +| `queue.job.redelivered` | log | a run again of an attempt whose run died | +| `queue.render.failed` | log | the text of a failed render, at `debug` | +| `queue.worker.error` | log | what escaped the steps of the worker | +| `queue.audit.failed` | log | the audit sink failed | +| `queue.lease.extend_failed` | log | the store did not extend a lease | +| `queue.lease.lost` | log | a run lost its lease, its result dropped | +| `queue.settle_fenced` | log | a settle the token fenced off: the job may run twice | +| `queue.settle_failed` | log | the store failed a settle: the lease will run out | +| `queue.reserve_failed` | log | the store failed a reserve | +| `queue.close_abandoned` | log | jobs still running past twice the timeout of `close()` | +| `queue.change_stream_failed` | log | the change stream failed, to be opened again | +| `queue.job.lost` | log | BullMQ failed a job itself | +| `queue.job.stalled` | log | BullMQ found a job stalled | +| `MAILER_AUDIT_FAILED` | warning | the audit sink failed | +| `MAILER_QUEUE_HANDLER_THREW` | warning | the handler of a driver threw | +| `MAILER_QUEUE_WORKER_THREW` | warning | an error escaped the steps of the worker | +| `MAILER_QUEUE_RETAIN_FAILED` | warning | BullMQ refused to drop the data of a finished job | + + diff --git a/tests/unit/codes.test.js b/tests/unit/codes.test.js new file mode 100644 index 0000000..3e4bcb8 --- /dev/null +++ b/tests/unit/codes.test.js @@ -0,0 +1,47 @@ +// Not ported: the codes of the queue, its drivers and its worker are those +// the table of docs/guide/queue.md lists, and the table lists no other. +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const ROOT = path.join(__dirname, '../..'); +const SOURCES = [ + 'src/queue', + 'src/bullmq', + 'src/mongodb', + 'src/audit', + 'src/internal/polling-driver.js', + 'src/otel/handler.js', +]; +const CODE = /'((?:queue|render|audit)\.[a-z_.]+|MAILER_(?:QUEUE|AUDIT)[A-Z_]*|delivery_unknown)'/g; + +const filesOf = (entry) => { + const full = path.join(ROOT, entry); + if (!fs.statSync(full).isDirectory()) return [full]; + const files = []; + for (const name of fs.readdirSync(full)) { + if (name.endsWith('.js')) files.push(path.join(full, name)); + } + return files; +}; + +test('Codes: the table of the queue guide lists the codes of the code, and no other', () => { + const inCode = new Set(); + for (const entry of SOURCES) { + for (const file of filesOf(entry)) { + for (const match of fs.readFileSync(file, 'utf8').matchAll(CODE)) inCode.add(match[1]); + } + } + // A state of a message, not a code, where it stands for one. + const guide = fs.readFileSync(path.join(ROOT, 'docs/guide/queue.md'), 'utf8'); + const table = guide.slice( + guide.indexOf(''), + guide.indexOf(''), + ); + const inTable = new Set(); + for (const match of table.matchAll(/^\| `([^`]+)` \|/gm)) inTable.add(match[1]); + const missing = [...inCode].filter((code) => !inTable.has(code)).toSorted(); + const stale = [...inTable].filter((code) => !inCode.has(code)).toSorted(); + assert.deepStrictEqual({ missing, stale }, { missing: [], stale: [] }); +}); From e3537a009b5c2b19cc87cd7e81aff966ce6eca3b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:22:39 +0300 Subject: [PATCH 66/83] docs: the comparison and the security policy say what 1.2 does The cells of @alexify/mailer did not name the MongoDB store of keys, the spans and metrics of the queue's worker, the signed and sealed jobs or the tenants in turn, and its count of tests was that of before the review; SECURITY.md did not say that a store drops the message of a job done with, nor how the id of a job could name a person and what idSecret does. Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 8 ++++++-- docs/comparison.json | 8 ++++---- docs/guide/comparison.md | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 48f5aa7..242bda4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -79,8 +79,12 @@ are next. The principles all of it is designed around: or a reference that the application's `resolveAttachment` reads. A job's data is stored as it is, a token of a reset link among it, unless a `codec` of the application's seals it (an envelope encryption: the store then holds the version, the hash and the signature in clear and the - rest encrypted); the store is the application's to restrict (Redis ACLs and TLS, MongoDB - roles). + rest encrypted); once a job is done with, every driver drops its message (`retainData`, the + failed jobs' kept for a retry by hand), so a reset link does not wait in the store and its + backups. The id of a job is a hash of its key, which goes into the audit and the logs: a key + names what the message tells of, never an address or a secret, and `idSecret` derives the id + with an HMAC so that it cannot be traced back. The store is the application's to restrict + (Redis ACLs and TLS, MongoDB roles). - **Templates are named, never pathed.** A template, include, partial or layout is read only through the loader of the renderer, by a name with no `..` or leading `/`; the file loader compares real paths, so a symbolic link cannot lead out of its directory. No engine reads a file diff --git a/docs/comparison.json b/docs/comparison.json index 32f2f55..fa63a16 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -341,7 +341,7 @@ "queue": { "mark": "✅", "text": "MongoDB outbox (transactional), BullMQ driver, memory queue", - "note": "Permanent failures fail at once, `Retry-After` is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause." + "note": "Permanent failures fail at once, `Retry-After` is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause. Jobs are signed (HMAC) and may be sealed, their message dropped once done with; tenants take turns, and a job that kills its worker is given up." }, "attachments": { "mark": "✅", @@ -400,7 +400,7 @@ }, "idempotency": { "mark": "✅", - "text": "decorator with reserved keys + 5 providers" + "text": "decorator with reserved keys, a MongoDB store + 5 providers" }, "dkim": { "mark": "✅", @@ -466,7 +466,7 @@ }, "otel": { "mark": "✅", - "text": "decorator: spans + metrics" + "text": "decorator: spans + metrics, and those of the queue's worker" }, "hooks": { "mark": "✅", @@ -501,7 +501,7 @@ "text": "2026-10 (this release)" }, "tests": { - "text": "177 / ~2560" + "text": "183 / ~2690" }, "maintainer": { "text": "1 author, Alexis Technologies" diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index 0344175..d721479 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -85,7 +85,7 @@ it. Scheduled sending✅ queue delay; native in 8 providers6❌❌❌⚠️ 8 drivers❌❌❌—✅✅✅ up to 3 days❌⚠️ campaigns only❌ Tags / metadata✅ metadata in 17 providers, tags in 13⚠️ headers; tags on SES only⚠️ headers⚠️ tags in 7 providers✅⚠️ headers only⚠️ headers⚠️ headers⚠️ headers✅✅✅✅✅✅ Open/click tracking✅ per message in 10 providers❌❌⚠️ config only✅ 10 drivers❌❌❌—⚠️ per domain✅✅✅⚠️ per domain✅ per-message override -Idempotency key✅ decorator with reserved keys + 5 providers❌❌⚠️ Resend, Lettermint, Maileroo✅ middleware + providers⚠️ decorator; batch bypasses it❌❌—✅❌❌❌❌❌ +Idempotency key✅ decorator with reserved keys, a MongoDB store + 5 providers❌❌⚠️ Resend, Lettermint, Maileroo✅ middleware + providers⚠️ decorator; batch bypasses it❌❌—✅❌❌❌❌❌ Security and authentication DKIM signing in the library✅ RSA + Ed25519, streaming, over-signing✅ RSA-SHA256❌✅ RSA + Ed25519, streaming✅ RSA + Ed25519, SMTP only⚠️ body canonicalization bug❌✅ RSA❌⚠️ keys through the API⚠️ whitelabel, generic⚠️ keys through the API⚠️ keys through the API⚠️ domain DNS records⚠️ Easy / BYO DKIM OAuth2 for SMTP✅ refresh, OAUTHBEARER✅ refresh + service account⚠️ bearer token only✅ refresh, OAUTHBEARER⚠️ callback, no refresh✅ refresh client❌⚠️ no refresh——————— @@ -105,7 +105,7 @@ it. Provider API coverage—————————18 namespaces, ~117 callsonly mail/send typed; ~251 endpoints in USAGE.md21 clients, ~155 calls85 methods~126 calls116 commands Multi-channel: SMS, WhatsApp, push❌ out of scope❌❌❌❌✅ SMS, WhatsApp, webpush, FCM❌❌—❌❌❌❌❌❌ Observability and extensibility -OpenTelemetry✅ decorator: spans + metrics❌❌✅ decorator: spans + metrics⚠️ withLogger⚠️ hooks + consoleObserver❌❌❌❌❌❌❌❌⚠️ middlewareStack +OpenTelemetry✅ decorator: spans + metrics, and those of the queue's worker❌❌✅ decorator: spans + metrics⚠️ withLogger⚠️ hooks + consoleObserver❌❌❌❌❌❌❌❌⚠️ middlewareStack Hooks / middleware✅ ./hooks, decorators, a pino-like logger✅ use(compile|stream)⚠️ logger function⚠️ decorators, LogTape✅ Middleware pipeline✅ hooks + plugins⚠️ logger✅ onEvent⚠️ dump()⚠️ console.error⚠️ setClient❌❌❌✅ logger + middlewareStack Mock / test mode✅ ./mock✅ jsonTransport, streamTransport, Ethereal⚠️ Message#read()✅ @upyo/mock✅ mock + Mailpit⚠️ preview, Mailpit❌❌—❌✅ sandboxMode✅ testmode⚠️ Sandbox server✅ Sandbox API⚠️ TestRenderEmailTemplate Local audit log / event journal⚠️ state model + sinks, MongoDB among them; crypto-shredding next10❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌ @@ -116,7 +116,7 @@ it. LicenseMITMIT-0MITMITMITMITMITMITMITMITMITMITMITMITApache-2.0 Version (2026-10-07)1.2.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0 Last commit2026-10 (this release)2026-10-072026-08-262026-09-092026-09-022026-08-192025-11-152026-09-082026-01-282026-10-062025-09-192026-10-072026-07-102026-10-072026-10-06 -Tests: files / cases177 / ~256061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo) +Tests: files / cases183 / ~269061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo) Owner / maintenance risk1 author, Alexis Technologies1 maintainerno SECURITY.md1 main author1 author1 author, 0.1 → 1.2 in ~80 days1 author1 author, 0.xno SECURITY.mdvendorvendor, old toolchainvendorvendorvendorvendor @@ -127,7 +127,7 @@ it. 1. Through npm:@alexify/mailer. Deno’s fetch sends a request again when a reused connection closes before its answer, so the docs give it a fetch of its own (known issues). 2. A failure after which the provider may have taken the message is marked deliveryUnknown: ./retry does not retry it and ./pool does not fail over with it. -3. Permanent failures fail at once, Retry-After is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause. +3. Permanent failures fail at once, Retry-After is a minimum, and a delivery of unknown outcome is never sent blindly again; in MongoDB, a job is written in the transaction of its cause. Jobs are signed (HMAC) and may be sealed, their message dropped once done with; tenants take turns, and a job that kills its worker is given up. 4. Nothing is read implicitly: fromPath and fromUrl are called by the application, fromUrl checks every redirect against allowUrl (no private address by default) and bounds time and bytes. 5. Names, never paths; locales, layouts, front-matter subjects, CSS inlined without remote fetches, text derived. 6. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. From 09cea666e4e6858e3c0e9015bb78064a55a10a5d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:23:05 +0300 Subject: [PATCH 67/83] docs(readme): sign the jobs of the queue example, and name what the queue keeps The example of the queue enqueued unsigned jobs, which whoever can write to Redis could forge; it signs them. A paragraph names what 1.2 adds around the queue: the message dropped once done with, sealing, the limit of runs that end unsettled, tenants in turn and the spans and metrics of the worker. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 483e513..4105e18 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,8 @@ const { createEjsEngine, createFileLoader, createTemplateRenderer } = require('@ const queue = createMailQueue({ driver: createBullMQDriver({ bullmq: { Queue, Worker }, connection: { host: 'localhost' } }), + // Whoever can write to Redis cannot make the worker send a message of theirs. + signingKeys: [{ id: '2026-10', secret: process.env.MAIL_JOB_KEY }], }); await queue.enqueue({ kind: 'template', @@ -144,6 +146,11 @@ queue.createWorker({ }); ``` +A job holds its message only until it is done with (`retainData`), and may be sealed by a codec +of yours; a job that kills its worker is given up rather than sent round for ever; tenants can +take turns (`fairness: 'tenant'` on the outbox), and `instrumentHandler` of `/otel` gives the +worker its spans and metrics. See the [queue guide](https://mailerat.vercel.app/guide/queue). + The same names import from ESM (`import { SmtpTransport } from '@alexify/mailer/smtp'`). The [API reference](https://mailerat.vercel.app/api/exports) lists every member, and the [guide](https://mailerat.vercel.app/guide/getting-started) every transport. From 66e153e7f784f27cf739811f6f2e81140a814278 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:24:35 +0300 Subject: [PATCH 68/83] docs(changelog): order the entry of 1.2.0 by subpath, and name every export The fixes of the review were added to the ./queue entry one sentence after another, until it held the audit, the outbox and the drivers in one paragraph; the entry is now by subpath, ./queue in parts (jobs, the security of the store, the worker, sources and drivers, the memory queue, retention and tenants). It names the exports it left out: MAIL_JOB_NAME, MAIL_JOB_VERSION, MailQueueError, QueueLeaseLostError, MAILER_BACKOFF_TYPE, createMemoryLoader, isFinalState, stateOfDeliveryEvent and the root types of the queue and of JSON. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 176 +++++++++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a42cce..392b76a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,93 +18,101 @@ dependency, BullMQ, MongoDB and the template engines being passed in. ### Added - **`./queue`**: `createMailQueue({ driver, audit, logger })` with `enqueue`, `enqueueMany` and - `createWorker({ transport, renderer, resolveAttachment, checkJob, classify })`. A job is a - template with its context, rendered by the worker, or content rendered already, as JSON - checked when it is enqueued (`prepareMailJob`, `parseMailJob`): its fields as `createMessage` - checks them, its context refused with the path to what JSON would change, attachments as bytes - (bounded) or references the worker resolves. Its id is a UUID of the namespace, tenant and - `idempotencyKey`, so the same message enqueued again is one job and another under the key is - refused (`queue.key_reused`, with the `jobIds` refused and, from `enqueueMany`, the `answers` of - the other jobs, all of them enqueued and audited first); its Message-ID and Content-IDs are fixed - for every attempt. The - worker fails a permanent failure, a missing template or a bug at once, retries a temporary one - never sooner than `Retry-After`, an operator's failure (`auth`, `configuration`) after five - minutes, and never sends again a message of unknown delivery. The renderer is given the job's - `signal`; a template that fails to render leaves only its name, phase and engine in the store and - the audit, its text in the log at `debug`. `scheduledAt` is a delay of the - queue and `expiresAt` a deadline. `createMailJobHandler` is the worker for any driver; - `createPollingDriver(source)` makes a driver of a store that lends its jobs (an outbox), and - `createMemoryQueue()` is one in memory for tests, its jobs on heaps so a reserve costs - O(log n) however many wait, its `source` held to the same contract as the outbox. A source's - settling writes give `false` when the token of the lease no longer holds the job, and the driver - logs `queue.settle_fenced`. Without a transaction, the `queued` event is told before the job is - enqueued, so a strict sink that fails refuses the enqueue with nothing enqueued; a driver that - writes no transaction refuses one (`transactional` drivers, the outbox, take it). A run whose lease the store does not extend is aborted before the - lease runs out, not after another worker took the job. A source counts the runs of a job that end unsettled - (`redeliveries`, never counted down, BullMQ's `stalledCounter`), and the worker gives up a job - past `maxRedeliveries` (3) as `delivery_unknown`, so a job that kills or blocks its worker does - not take every worker down in turn for ever. `redelivered: 'delivery_unknown'` makes a run again - of an attempt whose run before died `delivery_unknown` rather than a second send, unless the - audit sink tells (`has`, new on the memory, MongoDB and combined sinks) that it died before - its submit. `AuditSink.appendMany` (memory, MongoDB in one `bulkWrite`, combined) writes the - `queued` events of an `enqueueMany` in one call. The MongoDB sink's - `eventsOfProviderMessage(id)` finds the events of a message by the provider's id of it, where a - webhook's event without `correlationId` is filed. The fields of a job of version 1 are fixed: - a later version, or a field this one does not know, makes the job wait for a worker that reads - it (`queue.unsupported_version`, given back with its attempts not spent, for a week or until it - expires) rather than fail its hash during a rolling deploy. - `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and a worker given them sends - no job they did not sign (`queue.signature_invalid`): the SHA-256 of a job tells a change, but - whoever writes to the store can recompute it. The first key signs and each verifies, for - rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the application's key) seals the - fields of each job in its store, its addresses, context and attachments, leaving the version, - hash, signature and trace context in clear; a worker without the codec gives a sealed job back - (`queue.codec_missing`), and one whose codec fails retries it (`queue.codec_failed`); its - `correlationId` stays in clear too. `idSecret` derives the id of a job from its - key with an HMAC, so an id that goes into the audit and the logs cannot be traced back to a key - that names a person. + `createWorker({ transport, renderer, resolveAttachment, checkJob, classify })`. + - **Jobs.** A job is a template with its context, rendered by the worker, or content rendered + already, as JSON checked when it is enqueued (`prepareMailJob`, `parseMailJob`): its fields as + `createMessage` checks them, its context refused with the path to what JSON would change, + attachments as bytes (bounded) or references the worker resolves. Its id is a UUID of the + namespace, tenant and `idempotencyKey`, so the same message enqueued again is one job, and + another under the key is refused (`MailQueueError` `queue.key_reused`, with the `jobIds` + refused and, from `enqueueMany`, the `answers` of the other jobs, all of them enqueued and + audited first). Its Message-ID and Content-IDs are fixed for every attempt. `scheduledAt` is a + delay of the queue and `expiresAt` a deadline. The fields of a job of version 1 + (`MAIL_JOB_VERSION`, named `MAIL_JOB_NAME`) are fixed: a later version, or a field this one + does not know, makes the job wait for a worker that reads it (`queue.unsupported_version`, + given back with its attempts not spent, for a week or until it expires) rather than fail its + hash during a rolling deploy. + - **Security of the store.** `signingKeys` sign each job with an HMAC-SHA256 (`kid`, `mac`), and + a worker given them sends no job they did not sign (`queue.signature_invalid`): the SHA-256 of + a job tells a change, but whoever writes to the store can recompute it; the first key signs and + each verifies, for rotation. A `codec` (`encode`, `decode`, such as AES-GCM under the + application's key) seals the fields of each job, its addresses, context and attachments, + leaving the version, hash, correlation id, signature and trace context in clear; a worker + without it gives a sealed job back (`queue.codec_missing`), one whose codec fails retries it + (`queue.codec_failed`). `idSecret` derives the id of a job from its key with an HMAC, so an id + in the audit and the logs cannot be traced back to a key that names a person. + - **The worker** (`createMailJobHandler`, for any driver) fails a permanent failure, a missing + template or a bug at once, retries a temporary one never sooner than `Retry-After`, an + operator's failure (`auth`, `configuration`) after five minutes, and never sends again a + message of unknown delivery. It gives up a job whose runs ended unsettled more than + `maxRedeliveries` (3) times, as `delivery_unknown`, so a job that kills or blocks its worker + does not take every worker down in turn for ever; with `redelivered: 'delivery_unknown'`, a + run again of an attempt whose run before died is of unknown delivery rather than a second send, + unless the audit sink tells (`has`) that it died before its submit. `rateLimitScope` makes a + rate limit wait for the job, the queue or the job's tenant. The renderer is given the job's + `signal`; a template that fails to render leaves only its name, phase and engine in the store + and the audit, its text in the log at `debug`. Every failure, retry and escape is logged with + its code and duration (the codes in one table of the queue guide). + - **Sources and drivers.** `createPollingDriver(source)` makes a driver of a store that lends its + jobs (an outbox): it keeps the lease of a run, aborts a run whose lease the store does not + extend before the lease runs out (`QueueLeaseLostError`), logs a settle the token fenced off + (`queue.settle_fenced`; the settling writes of a source give `false`), and wakes one lane per + change of its source. A job request carries its `tenantId` and a job gives back its + `redeliveries`, never counted down. A driver declares `transactional`; one that is not refuses + a transaction, and without one the `queued` event is told before the job, so a strict sink + that fails leaves nothing enqueued. `AuditSink.appendMany` takes the `queued` events of an + `enqueueMany` in one write. `traceContext` gives each job the producer's W3C `traceparent`. + - **The memory queue**, `createMemoryQueue()`, for tests and a single process: its jobs on heaps + (a reserve costs O(log n)), its `source` held to the same contract as the outbox + (`tests/helpers/queue/source-contract.js`), with `stats()`, `failed()`, `retryFailed(id)` and + `resume()`. + - **Retention and tenants**, the same on every driver: a completed job is kept + `dedupeWindowSeconds` (a day, its window of duplicates), a failed one `retainFailedSeconds` (a + week), and the message of a job done with is dropped (`retainData`: `'failed'` keeps the failed + ones, `'all'`, `'none'`), its version and hash kept so a duplicate is still told. + `fairness: 'tenant'` (memory queue, outbox) takes a high job of any tenant first, then the + tenants in turn, so one tenant's backlog holds no other's mail. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases - that spend no attempt. `createBullMQProcessor` and `createBullMQBackoffStrategy` serve a Worker - of your own. A job BullMQ gives up itself (stalled too often) is told to the queue's audit sink - as `delivery_unknown` (`queue.lost`), so its trail does not end at `submitted`. `pnpm test:bullmq` runs it on BullMQ and Valkey. -- **`./mongodb`**: the transactional outbox, `createMongoOutbox({ collection })`, a - `QueueSource` in a collection of the MongoDB driver passed in: a job enqueued with - `{ transaction: session }` is written in that transaction (by an upsert, so a duplicate does - not abort it), lent by one atomic `findOneAndUpdate` under a token every later write is - fenced by, given to another worker as `redelivered` when its lease runs out, with a pause of - the whole queue and an optional change stream to wake the workers; `ensureIndexes()` makes - its indexes and the TTL of finished jobs. `createMongoAuditSink` (each event once by its id, - in the transaction) and `createMongoIdempotencyStore` (each step atomic, for - `IdempotencyTransport` across processes) go with it. `pnpm test:mongodb` runs them on a - replica set. A lease is taken back `clockSkewToleranceMilliseconds` (5 s) after it ran out, so - that a worker whose clock is behind does not run a job beside its holder. Every driver drops the - message of a job done with (`retainData`: `'failed'` keeps the failed ones, `'all'`, `'none'`), - keeping its version and hash, so a duplicate is still told and a reset link does not wait in the - store. Every driver keeps a completed job `dedupeWindowSeconds` (a day, its window of duplicates) - and a failed one `retainFailedSeconds` (a week); the outbox drops them by a TTL on `expireAt`. - An idle poll of the outbox is two round trips (leases taken back once a third of a lease, the - pause read once a poll interval, the next due time from its own index); its change stream wakes - one lane per change after which a job may be ready, and is opened again when it fails. - `fairness: 'tenant'` (memory queue, outbox) takes a high job of any tenant first, then the - tenants in turn, so one tenant's backlog holds no other's mail; `rateLimitScope: 'tenant'` pauses - one tenant's jobs alone, and `resume()` ends a pause early. `stats()` (jobs by state, the age of - the oldest job due, the pauses), `failed({ limit, after })` and `retryFailed(id)` read the outbox - and the memory queue, so a failed job does not go unseen until its retention drops it. + that spend no attempt; a batch read back 32 jobs at once to tell its duplicates. + `createBullMQProcessor` and `createBullMQBackoffStrategy` (of the type `MAILER_BACKOFF_TYPE`) + serve a Worker of your own. A job BullMQ gives up itself (stalled too often) is told to the + queue's audit sink as `delivery_unknown` (`queue.lost`). `pnpm test:bullmq` runs it on BullMQ + and Valkey. +- **`./mongodb`**: the transactional outbox, `createMongoOutbox({ collection })`, a `QueueSource` + in a collection of the MongoDB driver passed in. + - A job enqueued with `{ transaction: session }` is written in that transaction (an upsert, so a + duplicate does not abort it), lent by one atomic `findOneAndUpdate` under a token every later + write is fenced by, and given to another worker as `redelivered` when its lease runs out, + `clockSkewToleranceMilliseconds` (5 s) after by the taker's clock. + - An idle poll is two round trips: leases are taken back once a third of a lease, the pauses read + once a poll interval, and the next due time read from its own index; its optional change + stream wakes one lane per change after which a job may be ready, and is opened again when it + fails. `bench/outbox.js` measures it (2 round trips a job, some 2300 jobs a second for one + worker of ten lanes on a laptop). + - A pause of the queue or of one tenant (`rateLimitScope: 'tenant'`), and `resume()`; `stats()` + (jobs by state, the age of the oldest job due, the pauses), `failed({ limit, after })` and + `retryFailed(id)`; `ensureIndexes()` makes its indexes and the TTL on `expireAt`. + - `createMongoAuditSink` (each event once by its id, in the transaction; `has`, `appendMany` in + one `bulkWrite`, and `eventsOfProviderMessage(id)`, which finds the events a webhook filed + under the provider's id of a message) and `createMongoIdempotencyStore` (each step atomic, for + `IdempotencyTransport` across processes) go with it. `pnpm test:mongodb` runs them on a replica + set. - **`./templates`**: `createTemplateRenderer({ engine, loader, layout, partials, globals, defaultLocale, inlineCss })`, a `Renderer` of templates by name, over nine engines passed in (`createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, `createReactEmailEngine`): names never paths (`createFileLoader` reads nothing outside its - directory), variants by locale and part, the subject in front matter, a layout, includes read - through the loader, each template compiled once, CSS inlined by juice or css-inline without - fetching a stylesheet (`createCssInliner`), and the text derived from the HTML. - `TemplateNotFoundError` and `TemplateRenderError` have a stable `code`. + directory, `createMemoryLoader` holds them by name), variants by locale and part, the subject in + front matter, a layout, includes read through the loader, each template compiled once, CSS inlined + by juice or css-inline without fetching a stylesheet (`createCssInliner`), and the text derived + from the HTML. `TemplateNotFoundError` and `TemplateRenderError` have a stable `code`. - **`./audit`**: the states of a message (`queued`, `rendered`, `submitted`, `accepted`, `delivered`, `bounced`, `complained`, `failed`, `delivery_unknown`) and their sinks: - `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks`, `stateOf`, and - `auditEventOfDeliveryEvent` for the events of `./webhooks`. + `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks` (with `has` and + `appendMany` where a sink of them has them), `stateOf`, `isFinalState`, and + `auditEventOfDeliveryEvent` and `stateOfDeliveryEvent` for the events of `./webhooks`. - **`./otel`**: `instrumentHandler(handler, { tracerProvider, meterProvider, withSpan })`, a span `mailer.queue.process` per run of a job, linked to the producer's trace (the `traceparent` the mail queue's `traceContext` gives each job as `trace`), and the metrics `mailer.queue.jobs` (by @@ -113,16 +121,22 @@ dependency, BullMQ, MongoDB and the template engines being passed in. - **`./render`**: `RenderOutput.subject` and `renderFields(renderer, input)`, which gives `{ subject, content }`. - **The root**: the contracts of the reliability layer as types (`QueueDriver`, `QueueSource`, - `QueueJob`, `QueueOutcome`, `MessageState`, `AuditEvent`, `AuditSink`), with no code. + `QueueJob`, `QueueJobRequest`, `QueueLease`, `QueueOutcome`, `QueueHandler`, `QueueConsumer`, + `QueueConsumeOptions`, `QueueEnqueueOptions`, `MessageState`, `AuditEvent`, `AuditSink`, + `JsonValue`, `JsonObject`), with no code. ### Tooling - **Opt-in suites on the real services:** `pnpm test:bullmq` on BullMQ and Valkey (`tests/bullmq/docker-compose.yml`) and `pnpm test:mongodb` on a MongoDB replica set - (`tests/mongodb/docker-compose.yml`), outside the gates, as `test:mailpit` is. + (`tests/mongodb/docker-compose.yml`), outside the gates, as `test:mailpit` is; + `pnpm test:stores` runs both and fails without their stores, a manual step of every release. +- **`bench/outbox.js`**: the throughput of the outbox on a replica set, by workers and lanes. +- **One table of the codes of the queue** in its guide, which `tests/unit/codes.test.js` keeps in + step with the code. - **devDependencies for the tests:** the nine template engines, juice and css-inline, BullMQ, - ioredis and the MongoDB driver; `src/` requires none of them, as - `tests/unit/dependencies.test.js` checks. + ioredis and the MongoDB driver; `src/` requires none of them, as `tests/unit/dependencies.test.js` + checks. - **Shared internals:** the backoff, the classification of thrown errors, the fingerprint of a message and the path check moved to `src/internal/`, with no change of behavior. From 30b583e00fdd3feb41ba2cf7f079655476826396 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 01:28:35 +0300 Subject: [PATCH 69/83] test(queue): cover resume of the drivers, and drop an unused getter The coverage gate of functions (100) failed: resume() of the memory queue and of the outbox was reached only through their sources, and the size getter of the heap had no caller. The tests of stats() resume a tenant's pause through the drivers, and the getter is gone. Co-Authored-By: Claude Opus 5.5 --- src/internal/heap.js | 4 ---- tests/unit/mongodb.test.js | 2 ++ tests/unit/queue-memory.test.js | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/internal/heap.js b/src/internal/heap.js index 6c1d477..ff875e0 100644 --- a/src/internal/heap.js +++ b/src/internal/heap.js @@ -19,10 +19,6 @@ class Heap { this.#before = before; } - get size() { - return this.#items.length; - } - /** The least item, left on the heap. */ peek() { return this.#items[0]; diff --git a/tests/unit/mongodb.test.js b/tests/unit/mongodb.test.js index 88ee4fb..1986bae 100644 --- a/tests/unit/mongodb.test.js +++ b/tests/unit/mongodb.test.js @@ -622,6 +622,8 @@ test('MongoDB: the outbox tells what it holds, lists its failed jobs and retries ); assert.strictEqual(stats.oldestReadyMilliseconds, 6000); assert.deepStrictEqual(stats.pausedTenants, [{ tenantId: 'acme', until: 20_000 }]); + await outbox.resume({ tenantId: 'acme' }); + assert.strictEqual((await outbox.stats()).pausedTenants, undefined); assert.deepStrictEqual( (await outbox.failed({ limit: 2 })).map((job) => job.id), ['f1', 'f2'], diff --git a/tests/unit/queue-memory.test.js b/tests/unit/queue-memory.test.js index 4d20104..10df1a8 100644 --- a/tests/unit/queue-memory.test.js +++ b/tests/unit/queue-memory.test.js @@ -661,6 +661,8 @@ test('Queue: the memory queue tells what it holds, lists its failed jobs and ret ); assert.strictEqual(stats.oldestReadyMilliseconds, 6000); assert.deepStrictEqual(stats.pausedTenants, [{ tenantId: 'acme', until: 20_000 }]); + await queue.resume({ tenantId: 'acme' }); + assert.strictEqual(queue.stats().pausedTenants, undefined); // In the order of their ids, a page at a time. assert.deepStrictEqual( queue.failed({ limit: 2 }).map((job) => job.id), From 3bda2a8a448e807ecabc2ecf3384d749b014b155 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 09:52:21 +0300 Subject: [PATCH 70/83] chore: raise the size budget of ./templates for its review wave The templates wave of the review, made in a worktree that was not to touch the budgets, takes ./templates from 11.6 to 15.7 KB min+gzip: the bounds of includes, depth and output, the timeout and signal of a render, the structured errors and TemplateLoadError, the shared environments of the engines, the bounded caches of the engines, the locales and the logger. Its budget goes from 12 to 16 KB, the smallest half-KB step that passes. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index c527b97..1da594d 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4.5, "./render": 3, - "./templates": 12, + "./templates": 16, "./webhooks": 15.5, "./smtp": 23.5, "./sendmail": 12, From 9cdb82868ddaf559542a472575a284bead72e57e Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 09:53:43 +0300 Subject: [PATCH 71/83] fix(queue): retry a template failure that says it may pass The templates wave made a render that timed out (TemplateRenderError, phase timeout) and a loader out of reach (TemplateLoadError) retryable, but settleThrown failed every error of a template code at once, before it looked at retryable, so a hung loader failed the job for good. An error that says it is retryable is retried first; a broken template, a template not found and a bug of the loader still fail the job. Co-Authored-By: Claude Opus 5.5 --- docs/guide/queue.md | 2 +- src/queue/handler.js | 6 +++++- tests/unit/queue-handler.test.js | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 202de83..06ad51b 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -195,7 +195,7 @@ rotated, as another secret makes other ids and the same message another job. | a refused message (`validation`, `rejected`, `unsupported`) | failed at once | `failed` | | a failure of the operator (`auth`, `configuration`) | retried after `operatorDelayMilliseconds` (5 minutes) | `queued` | | a template missing or broken, no subject, no renderer | failed at once (`render.*`); a broken one's reason names the template and where it failed, its text (a helper's error, an excerpt, a value of the context) going to the log at `debug` only | `failed` | -| an outage of the template loader or of `resolveAttachment` | retried | `queued` | +| an outage of the template loader (`TemplateLoadError`) or of `resolveAttachment`, a render that timed out | retried | `queued` | | a bug: a `TypeError` thrown by the transport | failed at once (`queue.thrown`), logged with its stack | `failed` | | a job past `expiresAt`, or whose retry would come after it | failed (`queue.expired`) | `failed` | | a job written by a later version of the worker, or sealed for a codec it lacks | given back, its attempts not spent, for a worker that reads it; failed once it expires or has waited a week | — | diff --git a/src/queue/handler.js b/src/queue/handler.js index 9dc9d7c..4293791 100644 --- a/src/queue/handler.js +++ b/src/queue/handler.js @@ -233,7 +233,11 @@ const settleFailure = async (run, receipt, code) => { // A failure thrown by the application's code (a resolver, a renderer's // loader): a bug fails the job, anything else is classified as a send's is. const settleThrown = (run, error, code) => { - if (isProgrammerError(error) || error?.retryable === false || TEMPLATE_CODES.has(error?.code)) { + // What says it may pass is retried before any rule that fails it: a render + // that timed out, a loader out of reach. + const failsAgain = + isProgrammerError(error) || error?.retryable === false || TEMPLATE_CODES.has(error?.code); + if (error?.retryable !== true && failsAgain) { const category = classifyReceiptError(error).category; return fail(run, code, messageOf(error), { error: failureError( diff --git a/tests/unit/queue-handler.test.js b/tests/unit/queue-handler.test.js index caff3eb..16872a0 100644 --- a/tests/unit/queue-handler.test.js +++ b/tests/unit/queue-handler.test.js @@ -707,3 +707,33 @@ test('Queue: what escapes the steps is told, and each outcome logged with its du await new Promise(setImmediate); assert.ok(warnings.includes('MAILER_QUEUE_WORKER_THREW')); }); + +test('Queue: a render that timed out, or a loader out of reach, is retried; a broken template is not', async () => { + const { TemplateLoadError, TemplateRenderError } = require('../../src/templates/index.js'); + const throwing = (error) => ({ + render() { + throw error; + }, + }); + const outcomeOf = async (error, key) => { + const { handler, transport } = handlerOf({ renderer: throwing(error) }); + const outcome = await run(handler, jobOf({ idempotencyKey: key })); + assert.strictEqual(transport.calls.length, 0); + return outcome; + }; + const timedOut = await outcomeOf(new TemplateRenderError('welcome', 'timeout', 'slow'), 't'); + assert.deepStrictEqual([timedOut.type, timedOut.code], ['retry', 'render.failed']); + const unreachable = await outcomeOf( + new TemplateLoadError('welcome', { cause: new Error('connect ECONNREFUSED') }), + 'l', + ); + assert.strictEqual(unreachable.type, 'retry'); + const broken = await outcomeOf(new TemplateRenderError('welcome', 'compile', 'bad tag'), 'b'); + assert.deepStrictEqual([broken.type, broken.code], ['fail', 'render.failed']); + // A bug of the loader fails the job. + const bug = await outcomeOf( + new TemplateLoadError('welcome', { cause: new TypeError('x is undefined') }), + 'g', + ); + assert.strictEqual(bug.type, 'fail'); +}); From 1162ad2f5a82641e50fad297c9656844c8bba711 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 09:55:13 +0300 Subject: [PATCH 72/83] docs(changelog): the templates wave of the review in the entry of 1.2.0 The ./templates entry gains what its wave added and fixed before release: escaping that cannot be turned off silently, Liquid's limits and the bounds of a render, the timeout and signal of a render and TemplateLoadError, one rule of includes in every engine, bounded caches, locales and the logger, and the bugs of Handlebars, Nunjucks, Liquid and the layout. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 392b76a..79f2ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,33 @@ dependency, BullMQ, MongoDB and the template engines being passed in. front matter, a layout, includes read through the loader, each template compiled once, CSS inlined by juice or css-inline without fetching a stylesheet (`createCssInliner`), and the text derived from the HTML. `TemplateNotFoundError` and `TemplateRenderError` have a stable `code`. + - **Safety.** An engine option that turns escaping off (Handlebars `noEscape`, Nunjucks + `autoescape: false`, a Liquid `outputEscape`, Eta `autoEscape: false` or `escapeFunction`, EJS + `escape`) throws unless the engine is given `allowUnescaped: true`. Liquid has `parseLimit` + (1 MiB), `renderLimit` (1000 ms) and `memoryLimit` (1e8) on by default, and the guide says who + may write a template in each engine. A render is bounded: `maxIncludes` (10000), + `maxIncludeDepth` (32), `maxOutputBytes` (10 MiB). A loader is given template names alone, + none with a segment starting with a dot, and the file loader reads a file through the + descriptor its size came from. + - **Time and failures.** `render({ ..., signal })` stops at the abort of its signal, with its + reason, and `renderTimeoutMilliseconds` (30000) fails a render that hangs with a retryable + `TemplateRenderError` of phase `timeout`, what it waited for read or compiled anew; + `loader.load(name, { signal })` is told when to stop. A loader that fails is a + `TemplateLoadError` (`MAILER_TEMPLATE_LOAD`), retryable unless its error is a bug; every error + tells `phase`, `engine`, `template`, `requested` and `locale`. + - **Includes.** One rule in every engine: `/x` from the root, `./x` against the template it is + written in, a bare name from the root then the partials, with the engine's extensions; EJS and + Mustache read nested relative includes against their own template, and a bare Pug include is + read from the root, then the partials (`include ./footer` for the file beside). + - **Caches and telemetry.** What each engine keeps is bounded by the renderer's cache; an EJS + include is compiled once (`ejs.cache`, per renderer, forgotten at `clear()`), and Mustache + parses a template once (it needs `mustache.Context`). `locales` names the locales templates + are written in; `logger` hears `templates.compiled` and `templates.rendered` at `debug`. + - **Fixed before release:** a Handlebars partial that names itself hung the render, and + `./footer` of two directories was one partial; concurrent Nunjucks renders took each other's + missing include, and `ignore missing` failed; Liquid's missing include was a + `TemplateRenderError`; a renderer with a layout and React Email sent every message without the + layout, which now throws when the renderer is made. - **`./audit`**: the states of a message (`queued`, `rendered`, `submitted`, `accepted`, `delivered`, `bounced`, `complained`, `failed`, `delivery_unknown`) and their sinks: `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks` (with `has` and From e5e9b6ac6553168056a2be4fc29344c91be7f34b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 10:09:22 +0300 Subject: [PATCH 73/83] fix(queue): check the clock of a mail queue when it is made createMailQueue took now as given, while every other option of the queue and its worker is checked when it is made, so a clock that was not a function failed at the first enqueue. It is checked with the others. Co-Authored-By: Claude Opus 5.5 --- src/queue/mail-queue.js | 2 +- tests/unit/queue.test.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index 5b9e149..c38fe96 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -120,7 +120,7 @@ const createQueueConfig = (options) => { if (typeof auditStrict !== 'boolean') throw new TypeError('auditStrict must be a boolean.'); keyringOf(options.signingKeys); checkIdSecret(options.idSecret); - const now = options.now ?? Date.now; + const now = checkOptionalFunction(options.now, 'now') ?? Date.now; const jobOptions = { namespace: options.namespace, maxAttempts: options.maxAttempts, diff --git a/tests/unit/queue.test.js b/tests/unit/queue.test.js index 247e53e..f190a2f 100644 --- a/tests/unit/queue.test.js +++ b/tests/unit/queue.test.js @@ -372,3 +372,10 @@ test('Queue: enqueueMany tells the sink its batch in one write where it can', as // The duplicate is not told again. assert.deepStrictEqual(told, [[2, 'session']]); }); + +test('Queue: the clock of a queue is checked when the queue is made', () => { + assert.throws( + () => createMailQueue({ driver: createMemoryQueue(), now: 1 }), + /now must be a function/, + ); +}); From 63b38396b5e9d217163fbfac821964019dad933a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 10:10:31 +0300 Subject: [PATCH 74/83] fix(bullmq): count a delay from the clock the driver is given The driver counted the delay of a scheduled job, and the processor the time of a release with a delay, from Date.now(), while the request's enqueuedAt and availableAt came from the queue's own clock, so a queue with a clock of its own (tests, a clock kept in step otherwise) scheduled by another. createBullMQDriver and createBullMQProcessor take now, the driver passing its own to the processors it makes. Co-Authored-By: Claude Opus 5.5 --- src/bullmq/bullmq-driver.js | 14 ++++++++++---- src/bullmq/index.d.ts | 4 ++++ src/bullmq/processor.js | 10 ++++++---- tests/unit/bullmq.test.js | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/bullmq/bullmq-driver.js b/src/bullmq/bullmq-driver.js index 123cf5b..babb681 100644 --- a/src/bullmq/bullmq-driver.js +++ b/src/bullmq/bullmq-driver.js @@ -8,7 +8,7 @@ const { warnThrown } = require('../internal/callback.js'); const { createLog } = require('../internal/log.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { toUuid } = require('../internal/uuid.js'); -const { checkPositiveInteger } = require('../internal/options.js'); +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); const { checkRetainData, retentionOf } = require('../internal/queue-options.js'); const { settledWithin } = require('../internal/settle.js'); const { @@ -91,6 +91,7 @@ const createDriverConfig = (options) => { priorities: checkPriorities(options.priorities), detectDuplicates, retainData: checkRetainData(options.retainData), + now: checkOptionalFunction(options.now, 'now') ?? Date.now, // Kept by time, not by count: the window of duplicates does not shrink // under load. retention: retentionOf(options), @@ -104,7 +105,7 @@ const optionsOf = (config, request) => ({ jobId: request.id, timestamp: request.enqueuedAt, priority: config.priorities[request.priority] ?? config.priorities.normal, - delay: request.availableAt === undefined ? 0 : Math.max(0, request.availableAt - Date.now()), + delay: request.availableAt === undefined ? 0 : Math.max(0, request.availableAt - config.now()), attempts: request.maxAttempts, backoff: { type: MAILER_BACKOFF_TYPE }, }); @@ -170,6 +171,7 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { rateLimit, signal: consumeOptions.signal, retainData: config.retainData, + now: config.now, }); const worker = new Worker(config.queueName, processor, { ...workerOptions, @@ -228,7 +230,7 @@ const startWorker = (config, queue, handler, consumeOptions, log) => { * `lockDuration`; `priorities` (`{ high: 1, normal: 2, low: 3 }`); * `detectDuplicates` (true); `retainData` ('failed'), the finished jobs whose * data is kept; `dedupeWindowSeconds` (a day) and `retainFailedSeconds` (a - * week), BullMQ's `removeOnComplete` and `removeOnFail`; `logger`. + * week), BullMQ's `removeOnComplete` and `removeOnFail`; `now`; `logger`. * @returns The driver, with `queue` and `createProcessor(handler)`. * @throws {TypeError} If an option is invalid. */ @@ -284,7 +286,11 @@ const createBullMQDriver = (options = {}) => { consume: (handler, consumeOptions = {}) => startWorker(config, queue, handler, consumeOptions, log), createProcessor: (handler, processorOptions) => - createBullMQProcessor(handler, { retainData: config.retainData, ...processorOptions }), + createBullMQProcessor(handler, { + retainData: config.retainData, + now: config.now, + ...processorOptions, + }), async close() { if (config.owned) await queue.close(); }, diff --git a/src/bullmq/index.d.ts b/src/bullmq/index.d.ts index 149b170..3242598 100644 --- a/src/bullmq/index.d.ts +++ b/src/bullmq/index.d.ts @@ -131,6 +131,8 @@ export interface BullMQDriverOptions number; /** Hears the events of the workers: errors, jobs BullMQ failed itself, stalls. */ readonly logger?: Logger; } @@ -149,6 +151,8 @@ export interface BullMQProcessorOptions { readonly signal?: AbortSignal; /** The finished jobs whose data is kept, through `job.updateData`. @default 'failed' */ readonly retainData?: 'all' | 'failed' | 'none'; + /** The clock a release with a delay counts from. */ + readonly now?: () => number; } /** A consumer of a BullMQ driver: its worker. */ diff --git a/src/bullmq/processor.js b/src/bullmq/processor.js index 6f593c1..923d393 100644 --- a/src/bullmq/processor.js +++ b/src/bullmq/processor.js @@ -4,6 +4,7 @@ // and a job moved back for a release, which spends no attempt. const { warnThrown } = require('../internal/callback.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { checkRetainData } = require('../internal/queue-options.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { namedError, rateLimitError, retryError } = require('./backoff.js'); @@ -20,7 +21,7 @@ const jobOf = (job) => ({ }); // A job given back, by the means its version of BullMQ has. -const release = async (job, token, outcome) => { +const release = async (job, token, outcome, now) => { const delay = outcome.delayMilliseconds; if ( token !== undefined && @@ -28,7 +29,7 @@ const release = async (job, token, outcome) => { delay > 0 && typeof job.moveToDelayed === 'function' ) { - await job.moveToDelayed(Date.now() + delay, token); + await job.moveToDelayed(now() + delay, token); throw namedError('DelayedError', 'Released to the queue.'); } if (token !== undefined && typeof job.moveToWait === 'function') { @@ -66,7 +67,7 @@ const dropData = async (job) => { * @param handler The handler, such as `createMailJobHandler()`. * @param options `rateLimit(ms)`, the queue's `rateLimit`, for a retry of the * whole queue (it needs the worker's `limiter`); `signal`; `retainData` - * ('failed'), the finished jobs whose data is kept. + * ('failed'), the finished jobs whose data is kept; `now`. * @returns The processor, with `shutdown(reason)`, which aborts the jobs it * runs. * @throws {TypeError} If the handler is not a function. @@ -78,6 +79,7 @@ const createBullMQProcessor = (handler, options = {}) => { throw new TypeError('rateLimit must be a function.'); } const retainData = checkRetainData(options.retainData); + const now = checkOptionalFunction(options.now, 'now') ?? Date.now; const shutdown = new AbortController(); const signals = [shutdown.signal]; if (options.signal !== undefined) signals.push(options.signal); @@ -115,7 +117,7 @@ const createBullMQProcessor = (handler, options = {}) => { redactAddresses(`${outcome.code}: ${outcome.reason}`), ); case 'release': - return release(job, token, outcome); + return release(job, token, outcome, now); default: throw namedError( 'UnrecoverableError', diff --git a/tests/unit/bullmq.test.js b/tests/unit/bullmq.test.js index fa9fc8c..fcc2a3e 100644 --- a/tests/unit/bullmq.test.js +++ b/tests/unit/bullmq.test.js @@ -523,3 +523,18 @@ test('BullMQ: enqueueMany reads its jobs back many at once to tell duplicates', // At most 32 at once, more than one. assert.ok(most > 1 && most <= 32, String(most)); }); + +test('BullMQ: a delay is counted from the clock the driver is given', async () => { + const bullmq = fakeBullMQ(); + const driver = createBullMQDriver({ bullmq, now: () => 1_000_000 }); + await driver.enqueue(requestOf({ id: 'later', availableAt: 1_060_000 })); + assert.strictEqual(bullmq.queues[0].added[0].opts.delay, 60_000); + const job = fakeJob(); + const processor = driver.createProcessor(async () => ({ + type: 'release', + delayMilliseconds: 5000, + })); + await processor(job, 'tok').catch(() => {}); + assert.deepStrictEqual(job.moves, [['delayed', 1_005_000, 'tok']]); + assert.throws(() => createBullMQDriver({ bullmq, now: 'clock' }), /now must be a function/); +}); From 123c010b1bfdd2c843794f30cb5685e0635b8d52 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 10:11:01 +0300 Subject: [PATCH 75/83] docs: the defaults of the queue, its worker and its drivers in one table The defaults (attempts, backoff, the delay of an operator's failure, redeliveries, leases and polls, retention and data, fairness, the skew of clocks, BullMQ's priorities, the lease of IdempotencyTransport and the timeout of a render) were each in the docs of their option, some only in the code. The queue guide lists them in one table, each read from the code. Co-Authored-By: Claude Opus 5.5 --- docs/guide/queue.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 06ad51b..83ea006 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -287,6 +287,37 @@ token, releases that spend no attempt, and the pause. The repository's `tests/helpers/queue/source-contract.js` is that contract as tests; copy it, and run it on your source (`createMemoryQueue().source` is one to compare). +## Defaults + +What the queue, its worker and its drivers do when not told otherwise, in one +place: + +| Option | Of | Default | +| --- | --- | --- | +| `namespace` | the queue | `'mail'` | +| `maxAttempts` | the queue, a job | 5 | +| `maxJobBytes`, `maxInlineAttachmentBytes` | the queue | 256 KiB each | +| `backoff` | the worker | 30 seconds, doubled per attempt, an hour at most, full jitter | +| `operatorDelayMilliseconds` | the worker | 5 minutes (`auth`, `configuration`, a job it cannot read yet) | +| `maxRedeliveries` | the worker | 3 | +| `redelivered` | the worker | `'send'` | +| `rateLimitScope` | the worker | `'job'` | +| `concurrency` | a worker | 1 | +| `pollIntervalMilliseconds` | the memory queue, the outbox | 1 second | +| `leaseMilliseconds` | the memory queue, the outbox | 1 minute, extended every third of it | +| `timeoutMilliseconds` of `close()` | every driver | 30 seconds, then the jobs are aborted | +| `dedupeWindowSeconds` | every driver | a day | +| `retainFailedSeconds` | every driver | a week | +| `retainData` | every driver | `'failed'` | +| `fairness` | the memory queue, the outbox | `'none'` | +| `clockSkewToleranceMilliseconds` | the outbox | 5 seconds | +| `changeStream` | the outbox | `false` | +| `maxJobs`, `retainFinished` | the memory queue | 100000, 10000 | +| `priorities` | BullMQ | `high` 1, `normal` 2, `low` 3 | +| `detectDuplicates` | BullMQ | `true` | +| `leaseMs` | `IdempotencyTransport` | 10 minutes, longer than a lease, so a run that died is not sent again | +| `renderTimeoutMilliseconds` | a template renderer | 30 seconds | + ## Codes The codes the queue, its drivers and its worker give: in the `failure` of a job From 1ea8f0d68922103ee5ce69cfb3c2944ba50e7d03 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 10:11:12 +0300 Subject: [PATCH 76/83] docs(changelog): name the clock of the BullMQ driver Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f2ffc..0830221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,8 @@ dependency, BullMQ, MongoDB and the template engines being passed in. - **`./bullmq`**: `createBullMQDriver({ bullmq: { Queue, Worker }, connection })`, with BullMQ passed in: jobs under their id, priorities from 1, retention by time, the delay of each retry read back by a backoff strategy of the worker's, failures as `UnrecoverableError` and releases - that spend no attempt; a batch read back 32 jobs at once to tell its duplicates. + that spend no attempt, delays counted from the clock it is given (`now`); a batch read back 32 + jobs at once to tell its duplicates. `createBullMQProcessor` and `createBullMQBackoffStrategy` (of the type `MAILER_BACKOFF_TYPE`) serve a Worker of your own. A job BullMQ gives up itself (stalled too often) is told to the queue's audit sink as `delivery_unknown` (`queue.lost`). `pnpm test:bullmq` runs it on BullMQ From a1989dc54ac71c74d40e8f9c8c9109c2d8b81f1a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Mon, 19 Oct 2026 10:25:24 +0300 Subject: [PATCH 77/83] chore(release): date v1.2.0 and name its new exports in the docs The entry of 1.2.0 is dated the day it is prepared for release (to move to the day of the merge if that is another). The API page names instrumentHandler of ./otel and TemplateLoadError of ./templates, and llms.txt the spans and metrics of the worker, the signed and sealed jobs and the tenants in turn. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- docs/api/exports.md | 4 ++-- docs/public/llms.txt | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0830221..e9aa91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.2.0] - 2026-10-18 +## [1.2.0] - 2026-10-19 The reliability layer of the roadmap, on the transport of 1.1: a persistent queue, on BullMQ or a transactional outbox in MongoDB, whose workers render templates of the engine the application diff --git a/docs/api/exports.md b/docs/api/exports.md index aa5410c..b6b509a 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -53,7 +53,7 @@ model of delivery events. | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | | `@alexify/mailer/render` | `renderContent`, `renderFields`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | -| `@alexify/mailer/templates` | `createTemplateRenderer`, `createFileLoader`, `createMemoryLoader`, `createCssInliner`, `createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, `createReactEmailEngine`, `TemplateNotFoundError`, `TemplateRenderError` | [Templates by name](/messages/templates) | +| `@alexify/mailer/templates` | `createTemplateRenderer`, `createFileLoader`, `createMemoryLoader`, `createCssInliner`, `createEjsEngine`, `createHandlebarsEngine`, `createPugEngine`, `createMjmlEngine`, `createLiquidEngine`, `createNunjucksEngine`, `createEtaEngine`, `createMustacheEngine`, `createReactEmailEngine`, `TemplateNotFoundError`, `TemplateRenderError`, `TemplateLoadError` | [Templates by name](/messages/templates) | | `@alexify/mailer/queue` | `createMailQueue`, `createMailJobHandler`, `createMemoryQueue`, `createPollingDriver`, `prepareMailJob`, `parseMailJob`, `MailQueueError`, `QueueLeaseLostError`, `MAIL_JOB_NAME`, `MAIL_JOB_VERSION` | [The queue](/guide/queue) | | `@alexify/mailer/bullmq` | `createBullMQDriver`, `createBullMQProcessor`, `createBullMQBackoffStrategy`, `MAILER_BACKOFF_TYPE` | [BullMQ](/transports/bullmq) | | `@alexify/mailer/mongodb` | `createMongoOutbox`, `createMongoAuditSink`, `createMongoIdempotencyStore` | [MongoDB outbox](/guide/outbox) | @@ -83,7 +83,7 @@ model of delivery events. | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | | `@alexify/mailer/retry` | `RetryTransport`, `createRetryTransport` | [Retry](/transports/retry) | -| `@alexify/mailer/otel` | `OpenTelemetryTransport`, `createOpenTelemetryTransport`, `createEmailAttributeExtractor`, `createErrorClassifier`, `defaultErrorClassifier` | [OpenTelemetry](/transports/otel) | +| `@alexify/mailer/otel` | `OpenTelemetryTransport`, `createOpenTelemetryTransport`, `createEmailAttributeExtractor`, `createErrorClassifier`, `defaultErrorClassifier`, `instrumentHandler` | [OpenTelemetry](/transports/otel) | | `@alexify/mailer/logger` | `LoggerTransport` | [Logger](/transports/logger) | | `@alexify/mailer/breaker` | `CircuitBreakerTransport` | [Circuit breaker](/transports/breaker) | | `@alexify/mailer/ratelimit` | `RateLimitTransport` | [Rate limit](/transports/ratelimit) | diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 8e3d563..1b5e881 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -10,7 +10,9 @@ Key facts: - Status: 1.0 is the transport layer, ported from upyo 0.7.0 (MIT); 1.1 ports what nodemailer, unemail and sently did better and adds delivery events with verified webhooks; 1.2 adds the - queue (`/queue`, `/bullmq`, `/mongodb`), templates (`/templates`) and the audit (`/audit`) + queue (`/queue`, `/bullmq`, `/mongodb`), templates (`/templates`), the audit (`/audit`) and + the spans and metrics of the queue's worker (`instrumentHandler` of `/otel`); jobs are signed + (HMAC) and may be sealed by a codec, dropped once done with, and tenants can take turns - Package (npm): `@alexify/mailer`, installed with `npm install @alexify/mailer` - Node.js >= 22, tested on Bun and Deno too; CommonJS with ESM named-import interop; hand-written d.ts per entry From a6600a51010b10e3bf145eae131188d51e90d81e Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 19:05:16 +0300 Subject: [PATCH 78/83] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9aa91e..7ae0e04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.2.0] - 2026-10-19 +## [1.2.0] - 2026-10-20 The reliability layer of the roadmap, on the transport of 1.1: a persistent queue, on BullMQ or a transactional outbox in MongoDB, whose workers render templates of the engine the application From 491467504e430a76f2459906b8fd7c348e32fa3d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 19:15:28 +0300 Subject: [PATCH 79/83] test(queue): date the stale sealed job by the clock of the test The test of a sealed job that waited past a week for a worker with its codec dated the job by Date.now() and the handler by a fixed clock, so it passed only while the real day was within a week of the fixed one. The job is dated by the same fixed clock. Co-Authored-By: Claude Opus 5.5 --- tests/unit/queue-security.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index 2036a0f..932ae70 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -211,7 +211,7 @@ test('Queue: a sealed job waits for a worker with its codec, and fails one that const waiting = await plain(runOf(sealed, { attempt: 5 }), new AbortController().signal); assert.strictEqual(waiting.type, 'release'); const stale = await plain( - runOf(sealed, { enqueuedAt: Date.now() - 8 * 24 * 60 * 60 * 1000 }), + runOf(sealed, { enqueuedAt: NOW - 8 * 24 * 60 * 60 * 1000 }), new AbortController().signal, ); assert.deepStrictEqual([stale.type, stale.code], ['fail', 'queue.codec_missing']); From 9480f351799eb84837bb5fdff3cea306fb9b0a0a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 19:19:20 +0300 Subject: [PATCH 80/83] feat(queue): derive the ids of a job your way with deriveId A job's id, and the Message-ID and Content-IDs of a job enqueued under a key, are derived from that key so that the same key is the same job, and were UUIDs of its hash whatever generateId the queue was given: an application whose Message-IDs are ULIDs got UUIDs from its keyed jobs. deriveId(kind, seed), on the queue and prepareMailJob, makes them of a seed, a SHA-256 in hexadecimal of what they derive from (an HMAC under idSecret for the job id), in a format of the application's. It is a name of its own, not generateId: a generator of new ids (ulid()) would make the same message enqueued again another job, sent twice. The queue checks that the same seed gives the same id, and that a job id is one every driver takes (1 to 128 printable characters, no colon, not all digits) and the other ids the left half of a Message-ID. By default the ids are the UUIDs of before. ./queue goes from 18.9 to 19.3 KB min+gzip: its budget goes from 19 to 19.5 KB. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++-- docs/guide/queue.md | 19 +++++++++ docs/messages/compose.md | 4 +- scripts/size-budgets.json | 2 +- src/queue/derive.js | 77 ++++++++++++++++++++++++++++++++++++ src/queue/index.d.ts | 21 ++++++++++ src/queue/job.js | 28 ++++++++----- src/queue/mail-queue.js | 4 +- tests/types/queue.test-d.ts | 5 +++ tests/unit/queue-job.test.js | 67 +++++++++++++++++++++++++++++++ 10 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 src/queue/derive.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ae0e04..c25e93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,9 +22,11 @@ dependency, BullMQ, MongoDB and the template engines being passed in. - **Jobs.** A job is a template with its context, rendered by the worker, or content rendered already, as JSON checked when it is enqueued (`prepareMailJob`, `parseMailJob`): its fields as `createMessage` checks them, its context refused with the path to what JSON would change, - attachments as bytes (bounded) or references the worker resolves. Its id is a UUID of the - namespace, tenant and `idempotencyKey`, so the same message enqueued again is one job, and - another under the key is refused (`MailQueueError` `queue.key_reused`, with the `jobIds` + attachments as bytes (bounded) or references the worker resolves. Its id is derived from the + namespace, tenant and `idempotencyKey`, a UUID of their hash or what `deriveId(kind, seed)` + makes of it (as for the Message-ID and Content-IDs of a job under a key; `generateId` makes + those it draws), so the same message enqueued again is one job, and another under the key is + refused (`MailQueueError` `queue.key_reused`, with the `jobIds` refused and, from `enqueueMany`, the `answers` of the other jobs, all of them enqueued and audited first). Its Message-ID and Content-IDs are fixed for every attempt. `scheduledAt` is a delay of the queue and `expiresAt` a deadline. The fields of a job of version 1 diff --git a/docs/guide/queue.md b/docs/guide/queue.md index 83ea006..c504c5d 100644 --- a/docs/guide/queue.md +++ b/docs/guide/queue.md @@ -184,6 +184,25 @@ again by hashing the addresses one knows. `idSecret` (32 bytes at least) derives the id with an HMAC instead; it is the same on every producer and never rotated, as another secret makes other ids and the same message another job. +The identifiers a job draws come from `generateId`, as everywhere in the +library: its key when you give none, and its Message-ID and Content-IDs then. +Those it derives, the same for the same key, are UUIDs of a hash: its id and, +under a key of yours, its Message-ID and Content-IDs. `deriveId(kind, seed)` +makes them your way; `seed` is a SHA-256 in hexadecimal of what they derive +from (an HMAC under `idSecret` for the id of the job), and it must give the same +identifier for the same seed, which the queue checks: + +```js +const queue = createMailQueue({ + driver, + generateId: () => ulid(), // drawn: new at each call + deriveId: (kind, seed) => (kind === 'job-id' ? `job_${seed.slice(0, 26)}` : `m${seed.slice(0, 32)}`), +}); +``` + +A generator such as `ulid()` does not fit `deriveId`: the same message enqueued +again would get another id, and so be another job, sent twice. + ## What the queue does with an attempt | The attempt | The queue | The state | diff --git a/docs/messages/compose.md b/docs/messages/compose.md index db37feb..5937c4e 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -423,7 +423,9 @@ const transport = new SmtpTransport({ host: 'smtp.example.com', generateId }); ``` `composeMessage()` of `./mime` takes `generateId` in its options too, and every -transport in its configuration. +transport in its configuration. A [queue](/guide/queue#once-per-key) derives +the id of a job, and the Message-ID of a job under a key, from that key rather +than drawing it, so that the same key is the same job: `deriveId` makes those. ### Dropping an inherited header diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 1da594d..5f80ffb 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -37,7 +37,7 @@ "./hooks": 3, "./tee": 3.5, "./audit": 2, - "./queue": 19, + "./queue": 19.5, "./bullmq": 4.5, "./mongodb": 7.5 } diff --git a/src/queue/derive.js b/src/queue/derive.js new file mode 100644 index 0000000..930e138 --- /dev/null +++ b/src/queue/derive.js @@ -0,0 +1,77 @@ +// Not ported: the identifiers a job derives rather than draws: its id, from +// its namespace, tenant and key, and the Message-ID and Content-IDs of a job +// enqueued under a key, from its id. They are UUIDs of a hash by default, or +// what `deriveId(kind, seed)` of the application makes of that hash. Unlike +// `generateId`, which gives a new identifier at each call (a ULID, a cuid2), +// it must give the same for the same seed: the same message enqueued again is +// then the same job, with the same Message-ID, rather than another job (sent +// twice) or a message refused as another under its key. + +const { Buffer } = require('node:buffer'); +const { createHash, createHmac } = require('node:crypto'); +const { isDotAtomText } = require('../core/text.js'); +const { toKeyedUuid, toUuid } = require('../internal/uuid.js'); + +const MAX_JOB_ID_LENGTH = 128; +const MAX_ID_LENGTH = 256; +const ALL_DIGITS = /^\d+$/; + +const isPrintableAscii = (value) => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code < 0x21 || code > 0x7e) return false; + } + return true; +}; + +// What every driver takes as the id of a job, and what a header field takes +// as the left half of a Message-ID or a Content-ID. +const isValid = (kind, value) => { + if (typeof value !== 'string' || value.length === 0 || !isPrintableAscii(value)) return false; + if (kind === 'job-id') { + return value.length <= MAX_JOB_ID_LENGTH && !value.includes(':') && !ALL_DIGITS.test(value); + } + return value.length <= MAX_ID_LENGTH && isDotAtomText(value); +}; + +// The seed given to `deriveId`: a SHA-256 in hexadecimal of what the id +// derives from, or its HMAC under `idSecret`, so that it holds no key as given. +const seedOf = (text, secret) => + (secret === undefined ? createHash('sha256') : createHmac('sha256', secret)) + .update(Buffer.from(text, 'utf8')) + .digest('hex'); + +/** + * Derives an identifier of a job. + * + * @param kind `job-id`, `message-id` or `content-id`. + * @param text What it derives from: the namespace, tenant and key of the job + * for its id; its id and what is named for the others. + * @param config `deriveId` and `idSecret`, both optional. + * @returns The identifier: a UUID of the hash by default. + * @throws {TypeError} If `deriveId` gives an identifier that is not one, or + * another one for the same seed. + * @internal + */ +const deriveJobId = (kind, text, config) => { + const secret = kind === 'job-id' ? config.idSecret : undefined; + if (config.deriveId === undefined) { + return secret === undefined ? toUuid(text) : toKeyedUuid(text, secret); + } + const seed = seedOf(text, secret); + const value = config.deriveId(kind, seed); + if (!isValid(kind, value)) { + throw new TypeError( + `Invalid ${kind} from deriveId: ${JSON.stringify(String(value).slice(0, 100))}.`, + ); + } + // A generator of new identifiers given by mistake breaks every duplicate. + if (config.deriveId(kind, seed) !== value) { + throw new TypeError( + `deriveId gave another ${kind} for the same seed: it must derive, not generate.`, + ); + } + return value; +}; + +module.exports = { deriveJobId }; diff --git a/src/queue/index.d.ts b/src/queue/index.d.ts index 5de8634..7775d88 100644 --- a/src/queue/index.d.ts +++ b/src/queue/index.d.ts @@ -197,6 +197,19 @@ export interface MailJobCodec { decode(text: string): string | Promise; } +/** The identifiers a job derives rather than draws. */ +export type MailJobDerivedIdKind = 'job-id' | 'message-id' | 'content-id'; + +/** + * Makes an identifier a job derives of a seed, a SHA-256 in hexadecimal (an + * HMAC under `idSecret` for the job id) of what it derives from. It must + * give the same identifier for the same seed: unlike `generateId`, which + * gives a new one at each call. A `job-id` is 1 to 128 printable ASCII + * characters, no `:`, not all digits; the others are the left half of a + * Message-ID or Content-ID (dot-atom-text, at most 256 characters). + */ +export type MailJobIdDeriver = (kind: MailJobDerivedIdKind, seed: string) => string; + /** The options of the jobs of a queue. */ export interface MailJobOptions { /** The scope of the keys of the jobs. @default 'mail' */ @@ -219,6 +232,14 @@ export interface MailJobOptions { * another job. */ readonly idSecret?: string | Uint8Array; + /** + * Makes the id of each job, and the Message-ID and Content-IDs of a job + * enqueued under a key, of the hash they derive from, in place of a UUID + * of it: an id of your format, the same for the same key. The identifiers + * a job draws (its key and Message-ID without a key of yours) come from + * `generateId`. + */ + readonly deriveId?: MailJobIdDeriver; } /** A job of a message: its data, and the request a driver takes. */ diff --git a/src/queue/job.js b/src/queue/job.js index e463c35..d3085b1 100644 --- a/src/queue/job.js +++ b/src/queue/job.js @@ -11,7 +11,7 @@ const { createMessage } = require('../core/message.js'); const { generateSenderMessageId } = require('../core/message-id.js'); const { parseAddress } = require('../core/address.js'); const { toBase64 } = require('../internal/base64.js'); -const { toKeyedUuid, toUuid } = require('../internal/uuid.js'); +const { deriveJobId } = require('./derive.js'); const { hashJson } = require('./hash.js'); const { toJson } = require('./json.js'); const { keyringOf, signJob } = require('./signing.js'); @@ -128,7 +128,7 @@ const prepareAttachments = (given, sender, config, idOf) => { const inline = attachment.inline ?? false; const contentId = attachment.contentId ?? - `${generateId('content-id', idOf(`content-id\u0000${attachments.length}`))}@${domainOf(sender)}`; + `${generateId('content-id', idOf('content-id', `content-id\u0000${attachments.length}`))}@${domainOf(sender)}`; if ( typeof contentType !== 'string' || typeof inline !== 'boolean' || @@ -224,6 +224,9 @@ const resolveConfig = (options) => { if (!isText(config.namespace, 128)) throw new TypeError('namespace must be a short string.'); config.keyring = keyringOf(config.signingKeys); checkIdSecret(config.idSecret); + if (config.deriveId !== undefined && typeof config.deriveId !== 'function') { + throw new TypeError('deriveId must be a function.'); + } return config; }; @@ -261,7 +264,9 @@ const fieldsOf = (input, probe, attachments, messageId) => { * @param options `namespace` ('mail'), `generateId`, `now`, `maxAttempts` * (5), `maxJobBytes` (256 KiB), `maxInlineAttachmentBytes` (256 KiB); * `signingKeys`, whose first key signs the job (`kid`, `mac`); `idSecret`, - * under which its id is derived from its key. + * under which its id is derived from its key; `deriveId(kind, seed)`, which + * makes its id, and the Message-ID and Content-IDs of a job under a key, of + * the hash they derive from. * @returns `{ data, request }`. * @throws {TypeError} If a field is invalid, as `createMessage` finds it, or * a signing key or `idSecret`. @@ -276,11 +281,11 @@ const prepareMailJob = (input, options = {}) => { const tenantId = checkKey(input.tenantId, 'tenantId'); const keyed = checkKey(input.idempotencyKey, 'idempotencyKey'); const idempotencyKey = keyed ?? generateId('idempotency-key', config.generateId); - // A UUID of the key: under `idSecret`, one that cannot be traced back to a - // key that names a person. + // Derived from the key, so that the same key is the same job: a UUID of + // its hash (under `idSecret`, one no key can be found from), or what + // `deriveId` makes of it. const scoped = `${config.namespace}\u0000${tenantId ?? ''}\u0000${idempotencyKey}`; - const jobId = - config.idSecret === undefined ? toUuid(scoped) : toKeyedUuid(scoped, config.idSecret); + const jobId = deriveJobId('job-id', scoped, config); const correlationId = correlationIdOf(input, jobId); const scheduledAt = checkDateField(input.scheduledAt, 'scheduledAt'); const expiresAt = checkDateField(input.expiresAt, 'expiresAt'); @@ -291,8 +296,10 @@ const prepareMailJob = (input, options = {}) => { // Under a key of the caller's, the identifiers the job makes derive from // it, so that the same message enqueued again is the same job; otherwise // they are drawn from the generator. - const idOf = (what) => - keyed === undefined ? config.generateId : () => toUuid(`${jobId}\u0000${what}`); + const idOf = (kind, what) => + keyed === undefined + ? config.generateId + : () => deriveJobId(kind, `${jobId}\u0000${what}`, config); const sender = typeof input.from === 'string' ? parseAddress(input.from) : input.from; const { attachments, probes } = prepareAttachments( input.attachments, @@ -312,7 +319,8 @@ const prepareMailJob = (input, options = {}) => { throw new TypeError('A job takes its Message-ID as messageId, not as a header field.'); } const messageId = - probe.messageId ?? generateSenderMessageId(probe.sender.address, idOf('message-id')); + probe.messageId ?? + generateSenderMessageId(probe.sender.address, idOf('message-id', 'message-id')); const now = (config.now ?? Date.now)(); const data = { v: MAIL_JOB_VERSION, ...body, ...fieldsOf(input, probe, attachments, messageId) }; data.idempotencyKey = idempotencyKey; diff --git a/src/queue/mail-queue.js b/src/queue/mail-queue.js index c38fe96..dd1837d 100644 --- a/src/queue/mail-queue.js +++ b/src/queue/mail-queue.js @@ -129,6 +129,7 @@ const createQueueConfig = (options) => { generateId: options.generateId, signingKeys: options.signingKeys, idSecret: options.idSecret, + deriveId: checkOptionalFunction(options.deriveId, 'deriveId'), now, }; for (const name of Object.keys(jobOptions)) { @@ -167,7 +168,8 @@ const prepareJob = async (queue, input) => { * `maxInlineAttachmentBytes` (256 KiB); `signingKeys`, which sign each job * and which its workers verify; `codec`, which seals the fields of each job * in its store and opens them in its workers; `idSecret`, under which the id - * of a job is derived from its key; `traceContext`, which gives the W3C + * of a job is derived from its key; `deriveId(kind, seed)`, which makes the + * identifiers a job derives of their hash; `traceContext`, which gives the W3C * traceparent of the producer each job carries; `logger`, `generateId`, * `now`. * @returns `{ driver, enqueue, enqueueMany, createWorker }`. diff --git a/tests/types/queue.test-d.ts b/tests/types/queue.test-d.ts index 2c60ebf..396d898 100644 --- a/tests/types/queue.test-d.ts +++ b/tests/types/queue.test-d.ts @@ -159,6 +159,11 @@ prepareMailJob( { idSecret: new Uint8Array(32) }, ); expectError(createMailQueue({ driver, idSecret: 42 })); +createMailQueue({ + driver, + deriveId: (kind, seed) => (kind === 'job-id' ? `job_${seed.slice(0, 26)}` : seed.slice(0, 32)), +}); +expectError(createMailQueue({ driver, deriveId: (kind: number) => String(kind) })); const codec = { encode: (text: string) => text, decode: async (text: string) => text }; createMailQueue({ driver, codec }); createMailJobHandler({ transport, codec, maxRedeliveries: 0, redelivered: 'delivery_unknown' }); diff --git a/tests/unit/queue-job.test.js b/tests/unit/queue-job.test.js index b0d0640..e5bda8e 100644 --- a/tests/unit/queue-job.test.js +++ b/tests/unit/queue-job.test.js @@ -405,3 +405,70 @@ test('Queue: a field a later version adds makes the job wait, not fail its hash' (error) => error.code === 'queue.invalid_job', ); }); + +test('Queue: deriveId makes the ids a job derives, the same for the same key', async () => { + const { createMailQueue, createMemoryQueue } = require('../../src/queue/index.js'); + const seeds = []; + // A deterministic format of the application's: a prefix and part of the hash. + const deriveId = (kind, seed) => { + seeds.push([kind, seed]); + return kind === 'job-id' ? `job_${seed.slice(0, 26)}` : `m${seed.slice(0, 24)}`; + }; + const keyed = prepare( + { + idempotencyKey: 'welcome:42', + attachments: [{ filename: 'a.png', content: new Uint8Array([1]), inline: true }], + }, + { deriveId }, + ); + assert.match(keyed.request.id, /^job_[0-9a-f]{26}$/); + assert.match(keyed.data.messageId, /^m[0-9a-f]{24}@example\.com$/); + assert.match(keyed.data.attachments[0].contentId, /^m[0-9a-f]{24}@example\.com$/); + assert.strictEqual(keyed.data.correlationId, keyed.request.id); + // The seed is a hash: it holds no key as given. + for (const [, seed] of seeds) { + assert.match(seed, /^[0-9a-f]{64}$/); + assert.ok(!seed.includes('welcome')); + } + // Without a key, the job id is derived from the generated key, and the + // Message-ID drawn from generateId, as before. + let drawn = 0; + const generateId = (kind) => `g${kind.replace('-', '')}${++drawn}`; + const unkeyed = prepare({}, { deriveId, generateId }); + assert.match(unkeyed.request.id, /^job_/); + assert.match(unkeyed.data.messageId, /^gmessageid\d+@example\.com$/); + // The same message under its key is the same job, a duplicate. + const queue = createMailQueue({ driver: createMemoryQueue(), deriveId, now: () => NOW }); + const input = { ...base, idempotencyKey: 'welcome:42' }; + const first = await queue.enqueue(input); + const again = await queue.enqueue(input); + assert.deepStrictEqual([first.duplicate, again.duplicate, again.id], [false, true, first.id]); + // Under idSecret, the seed of the job id is its HMAC: another id. + const secret = prepare({ idempotencyKey: 'welcome:42' }, { deriveId, idSecret: 's'.repeat(32) }); + assert.notStrictEqual(secret.request.id, keyed.request.id); + // By default, the UUIDs of before. + assert.strictEqual( + prepare({ idempotencyKey: 'welcome:42' }).request.id, + prepare({ idempotencyKey: 'welcome:42' }, { deriveId: undefined }).request.id, + ); +}); + +test('Queue: deriveId must derive, and give ids a driver and a header field take', () => { + let count = 0; + const generating = () => `job_${++count}`; + assert.throws( + () => prepare({ idempotencyKey: 'k' }, { deriveId: generating }), + /deriveId gave another job-id for the same seed: it must derive, not generate/, + ); + const refusals = [ + [(kind, seed) => (kind === 'job-id' ? `job:${seed}` : seed), /Invalid job-id/], + [(kind, seed) => (kind === 'job-id' ? '12345' : seed), /Invalid job-id/], + [(kind, seed) => (kind === 'job-id' ? 'x'.repeat(129) : seed), /Invalid job-id/], + [(kind, seed) => (kind === 'job-id' ? `j${seed}` : `a b${seed}`), /Invalid message-id/], + [() => 42, /Invalid job-id/], + ]; + for (const [deriveId, pattern] of refusals) { + assert.throws(() => prepare({ idempotencyKey: 'k' }, { deriveId }), pattern); + } + assert.throws(() => prepare({}, { deriveId: 'uuid' }), /deriveId must be a function/); +}); From 363c654ff5264ffefc40ee2e1ca3dd1bbf1a042b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 22:32:36 +0300 Subject: [PATCH 81/83] fix(templates): resolve a template's real path the same way in loadSync and load loadSync resolved the directory and the file with fs.realpathSync, which keeps the spelling it is given (its case, a short name of Windows such as the RUNNER~1 of a CI runner), and load with fsp.realpath, the system's. Whichever ran first fixed the root, so a load after a loadSync was refused as leading out of the directory. The sync path uses fs.realpathSync.native. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++- src/templates/loader.js | 7 +++++-- tests/unit/templates-loader.test.js | 22 +++++++++++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c25e93b..4ee3bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,7 +137,9 @@ dependency, BullMQ, MongoDB and the template engines being passed in. `./footer` of two directories was one partial; concurrent Nunjucks renders took each other's missing include, and `ignore missing` failed; Liquid's missing include was a `TemplateRenderError`; a renderer with a layout and React Email sent every message without the - layout, which now throws when the renderer is made. + layout, which now throws when the renderer is made; the file loader refused every `load` after + a `loadSync` when its `dir` was spelled otherwise than on disk (its case, a short name of + Windows). - **`./audit`**: the states of a message (`queued`, `rendered`, `submitted`, `accepted`, `delivered`, `bounced`, `complained`, `failed`, `delivery_unknown`) and their sinks: `createMemoryAuditSink`, `createLoggerAuditSink`, `combineAuditSinks` (with `has` and diff --git a/src/templates/loader.js b/src/templates/loader.js index d1883ab..70cfa09 100644 --- a/src/templates/loader.js +++ b/src/templates/loader.js @@ -167,12 +167,15 @@ const loadFile = async (files, name, signal) => { } }; +// The real path of the system, as `fsp.realpath` gives it: `fs.realpathSync` +// keeps the spelling it is given (its case, a short name of Windows), so a +// root it resolved held no path `load` resolved, and the other way round. const loadFileSync = (files, name) => { if (!accepts(files, name)) return undefined; - files.root ??= fs.realpathSync(files.dir); + files.root ??= fs.realpathSync.native(files.dir); let real; try { - real = fs.realpathSync(path.join(files.root, name)); + real = fs.realpathSync.native(path.join(files.root, name)); } catch (error) { if (isMissing(error)) return undefined; throw error; diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index e8f33e3..ec84f54 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -50,7 +50,7 @@ test('Templates: the file loader reads a name under its directory, its BOM dropp const welcome = await loader.load('welcome.ejs'); assert.strictEqual(welcome.name, 'welcome.ejs'); assert.strictEqual(welcome.source, '

    Hi <%= name %>

    '); - assert.strictEqual(welcome.path, fs.realpathSync(path.join(root, 'welcome.ejs'))); + assert.strictEqual(welcome.path, fs.realpathSync.native(path.join(root, 'welcome.ejs'))); assert.strictEqual((await loader.load('auth/reset.ejs')).source, '

    reset

    '); assert.strictEqual(loader.loadSync('auth/reset.ejs').source, '

    reset

    '); // Missing, a directory, another extension and a name that is not one: none. @@ -80,6 +80,26 @@ test('Templates: the file loader reads a name under its directory, its BOM dropp ); }); +test('Templates: the file loader reads one real path, sync or not, however its directory is spelled', async (t) => { + // Before: loadSync resolved with the realpath of JavaScript, which keeps + // the case it is given and the short names of Windows (the temporary + // directory of a runner), and load with the system's, so a load after a + // loadSync was refused as leading out of the directory. + const { root } = makeTree(t); + const spelled = root.replace('mailer-templates-', 'MAILER-TEMPLATES-'); + if (!fs.existsSync(spelled)) { + t.skip('the file system tells case apart'); + return; + } + const real = fs.realpathSync.native(path.join(root, 'welcome.ejs')); + const syncFirst = createFileLoader({ dir: spelled }); + assert.strictEqual(syncFirst.loadSync('welcome.ejs').path, real); + assert.strictEqual((await syncFirst.load('welcome.ejs')).path, real); + const asyncFirst = createFileLoader({ dir: spelled }); + assert.strictEqual((await asyncFirst.load('welcome.ejs')).path, real); + assert.strictEqual(asyncFirst.loadSync('welcome.ejs').path, real); +}); + test('Templates: the file loader reads no more than maxBytes of a file that grew since its size', async (t) => { // Before: the size was taken by path and the file read whole by path // after, so a file that grew in between, or was replaced, was read whole. From 49b27feea9f3968e92095428b2b711e259a9400a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 22:33:21 +0300 Subject: [PATCH 82/83] test(queue): close the workers with t.after, as Node 22 parses no await using Three tests declared their worker with `await using`, which Node 22 does not parse, so queue.test.js and queue-security.test.js failed whole on Node 22 in CI (Linux, Windows and the coverage job) while passing on Node 24. CLAUDE.md says so, and how a test compares a path with the template loader's. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 +++++- tests/unit/queue-security.test.js | 12 ++++++------ tests/unit/queue.test.js | 6 +++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 415d725..7c40699 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1128,7 +1128,11 @@ change. **This file wins** if they ever disagree. catches a new one, and Docker (`node:22-bookworm-slim`) reproduces Linux. The JMAP test of a 401 to a raw message of unknown size, which streams, is `todo` on Node 24 (known upstream bug 14). `tests/unit/export-parity.test.js` also checks that every d.ts using `AsyncDisposable` - references `esnext.disposable` itself. + references `esnext.disposable` itself. Node 22 parses no `await using` (Node 24 does), so no + file it loads may hold one: a test closes what it opens with `t.after`, and a file with one + fails whole on Node 22 alone, in CI, never on a Node 24 at hand. A path compared with what the + loader of `./templates` gives is resolved with `fs.realpathSync.native`, as the plain one keeps + the short names of Windows (the temporary directory of a runner). - **`tests/runtime/`** runs the package as a consumer does, by its name, with no mock: every entry required and imported by name; SMTP over plain TCP, implicit TLS, STARTTLS and an HTTP CONNECT proxy, with DKIM checked by `tests/helpers/mime/verify-dkim.js`; an HTTP provider and diff --git a/tests/unit/queue-security.test.js b/tests/unit/queue-security.test.js index 932ae70..84cecde 100644 --- a/tests/unit/queue-security.test.js +++ b/tests/unit/queue-security.test.js @@ -104,16 +104,16 @@ test('Queue: signing keys are checked', () => { } }); -test('Queue: a worker with signing keys sends a signed job and fails a forged one unsent', async () => { +test('Queue: a worker with signing keys sends a signed job and fails a forged one unsent', async (t) => { const clock = manualClock(NOW); const driver = createMemoryQueue({ now: clock.now, sleep: clock.sleep }); const queue = createMailQueue({ driver, now: clock.now, signingKeys: [KEY_A] }); const transport = fakeTransport(); await queue.enqueue(input); - await using worker = queue.createWorker({ transport }); + const worker = queue.createWorker({ transport }); + t.after(() => worker.close()); await driver.drain(); assert.strictEqual(transport.calls.length, 1); - assert.ok(worker); const records = []; const logger = { error: (record, message) => records.push([message, record]) }; @@ -163,7 +163,7 @@ const runOf = (data, extra = {}) => ({ ...extra, }); -test('Queue: with a codec the store holds no address, and the worker opens the job', async () => { +test('Queue: with a codec the store holds no address, and the worker opens the job', async (t) => { const clock = manualClock(NOW); const driver = createMemoryQueue({ now: clock.now, sleep: clock.sleep }); const codec = aesCodec(); @@ -190,13 +190,13 @@ test('Queue: with a codec the store holds no address, and the worker opens the j } // A duplicate is still told by the hash in clear. assert.strictEqual((await queue.enqueue(withFile)).duplicate, true); - await using worker = queue.createWorker({ transport }); + const worker = queue.createWorker({ transport }); + t.after(() => worker.close()); await driver.drain(); assert.strictEqual(transport.calls.length, 1); const { message } = transport.calls[0]; assert.strictEqual(message.recipients[0].address, 'ada@example.com'); assert.strictEqual(message.messageId, enqueued.messageId); - assert.ok(worker); assert.throws(() => parseMailJob(stored.data), isCode('queue.codec_missing')); }); diff --git a/tests/unit/queue.test.js b/tests/unit/queue.test.js index f190a2f..85c5470 100644 --- a/tests/unit/queue.test.js +++ b/tests/unit/queue.test.js @@ -42,13 +42,14 @@ const setup = (options = {}) => { return { clock, driver, audit, queue }; }; -test('Queue: a message enqueued is rendered and sent by a worker, its states told', async () => { +test('Queue: a message enqueued is rendered and sent by a worker, its states told', async (t) => { const { driver, audit, queue } = setup(); const transport = fakeTransport((message) => sent(`p-${message.subject.length}`)); const enqueued = await queue.enqueue(resetJob()); assert.strictEqual(enqueued.duplicate, false); assert.match(enqueued.messageId, /@example\.com$/); - await using worker = queue.createWorker({ transport, renderer: templates, random: () => 0 }); + const worker = queue.createWorker({ transport, renderer: templates, random: () => 0 }); + t.after(() => worker.close()); await driver.drain(); assert.strictEqual(transport.calls.length, 1); const { message } = transport.calls[0]; @@ -71,7 +72,6 @@ test('Queue: a message enqueued is rendered and sent by a worker, its states tol ); assert.strictEqual(audit.stateOf(enqueued.correlationId), 'accepted'); assert.strictEqual(driver.get(enqueued.id).state, 'completed'); - assert.ok(worker); }); test('Queue: the same message enqueued twice is sent once, another under its key refused', async () => { From 4bd8d73d7ea1de78c5028adfea8127e704707a1e Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Tue, 20 Oct 2026 22:33:56 +0300 Subject: [PATCH 83/83] test(templates): restore the replaced node:fs after each test, as Deno does not Deno keeps a method mocked with t.mock.method after its test, so the open that threw EACCES in one test of the file loader threw in the three after it. The helper that replaces node:fs restores every mock in t.after; CLAUDE.md says so next to the timers Deno keeps. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 ++-- tests/unit/templates-loader.test.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7c40699..46c3b05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1141,8 +1141,8 @@ change. **This file wins** if they ever disagree. they use `node:test` and `node:assert` alone, never `t.mock`. - **Bun and Deno** (`pnpm test:bun`, `pnpm test:deno`) run each file in a fresh global (`--parallel`), as `node --test` runs each in a process of its own; in one global, the mocks of - one file reach the next. Deno keeps mocked timers enabled after a test (`t.after` resets - them), a TLS server reads no SNI as `null` on Deno and `undefined` on Bun (`false` on Node), the + one file reach the next. Deno keeps mocked timers enabled and mocked methods in place after a + test (`t.after` resets and restores them, `t.mock.restoreAll()`), a TLS server reads no SNI as `null` on Deno and `undefined` on Bun (`false` on Node), the JMAP tests of a lost answer are skipped on Deno, whose `fetch` sends it again, and the import by name of `export-parity` spawns the node CLI, which Deno is not. diff --git a/tests/unit/templates-loader.test.js b/tests/unit/templates-loader.test.js index ec84f54..67e7949 100644 --- a/tests/unit/templates-loader.test.js +++ b/tests/unit/templates-loader.test.js @@ -15,8 +15,10 @@ const { const ejs = require('ejs'); // Replaces functions of node:fs for a test, and tells whether it could: a -// runtime whose node:fs is not writable leaves the test to the others. +// runtime whose node:fs is not writable leaves the test to the others. Node +// restores them after the test; Deno keeps them for the tests that follow. const replace = (t, pairs) => { + t.after(() => t.mock.restoreAll()); for (const [object, name, implementation] of pairs) { const before = object[name]; try {