From dbc44faeb1c7d2f4b5ae7be5b45487f25057e222 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 19:55:23 +0300 Subject: [PATCH 001/143] chore: prepare the ports of nodemailer, unemail and sently 1.1 takes the features of three libraries of the comparison where they lead this package. LICENSE carries the notices of nodemailer 10.0.16 (MIT-0), unemail 0.7.0 and sently 1.2.2 (MIT). The attribution test reads a table of sources, each with its version, path pattern and notice, and a second source of a file needs its own notice; the marker of code that is not a port becomes "// Not ported:". CLAUDE.md gains the rules of these ports: the feature is ported onto the contracts of the package, and a known upstream bug is fixed in the feat commit, with a test and an entry in the list. The decision on providers allows the ten most used of unemail and sently. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 48 +++++++++- LICENSE | 79 ++++++++++++++++ src/core/id.js | 2 +- src/core/text.js | 2 +- src/internal/base64.js | 2 +- src/internal/batch.js | 2 +- src/internal/body.js | 2 +- src/internal/config.js | 2 +- src/internal/delivery-unknown.js | 2 +- src/internal/headers.js | 2 +- src/internal/http.js | 2 +- src/internal/log.js | 2 +- src/internal/loopback.js | 2 +- src/internal/provider.js | 2 +- src/internal/redact-addresses.js | 2 +- src/internal/redact.js | 2 +- src/internal/sleep.js | 2 +- src/internal/unsent.js | 2 +- src/internal/validate-ssl.js | 2 +- src/version.js | 2 +- tests/unit/attribution.test.js | 150 +++++++++++++++++++++++++++---- 21 files changed, 276 insertions(+), 37 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 94f92e8..1e57c8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,8 +108,8 @@ Copyright 2025 Hong Minhee; the notice is in `LICENSE`): one subpath per upyo pa A d.ts names the directory of its package (`// Types ported from upyo v0.7.0 (packages/smtp/src)`), as its types come from several files, and lines that say how a file was adapted may come between the two. A file that is not a port starts with - `// Not ported from upyo:` and what it is. `tests/unit/attribution.test.js` checks one or the - other in every file the package ships. + `// Not ported:` and what it is. `tests/unit/attribution.test.js` checks one or the other in + every file the package ships. - **The port keeps upstream's behavior.** What changes is the platform: CommonJS and the code style below, `node:crypto` instead of `@noble/hashes`, the OpenTelemetry providers and the logger passed in, fallbacks for runtimes older than Node 22 dropped, and the `@since` tags of @@ -429,6 +429,43 @@ fails without it; ported tests that pinned the old behavior say "Changed after t a message that went out through one transport again through the others. It is gone from the type, and `continueOnSuccess: true` throws a `TypeError` when the pool is made. +## Code ported from nodemailer, unemail and sently + +From 1.1 the package also takes the features of three libraries of `docs/comparison.json` where +they lead it: [nodemailer](https://github.com/nodemailer/nodemailer) 10.0.16 (MIT-0, Copyright +2011-2023 Andris Reinman), [unemail](https://github.com/productdevbook/unemail) 0.7.0 (MIT, +Copyright 2025 Wind) and [sently](https://github.com/omqkhafi/sently) 1.2.2 (MIT, Copyright 2026 +sendx contributors). Their notices are in `LICENSE`. + +- **What is ported is the feature, onto the contracts of this package:** the mapping of a + provider's fields, an algorithm (a token bucket, a circuit breaker), the formatting of header + fields, and the test cases. The code lands on `Receipt`, `src/internal/http.js` and the injected + options; the HTTP paths, retries and error models of the sources are not kept. +- **Known upstream bugs are fixed in the port,** unlike upyo's: in the same `feat:` commit, each + with a test that fails on upstream's behavior and an entry in the list below (N for nodemailer, + U for unemail, S for sently). A ported test that pinned upstream's behavior says "Changed in the + port". +- A ported file starts with the header of its source, as for upyo: + ```js + // Ported from nodemailer v10.0.16 (src/sendmail-transport/index.ts) + // https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman + ``` + The notice of unemail is `// https://github.com/productdevbook/unemail — MIT, Copyright 2025 + Wind`, that of sently `// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx + contributors`. A file with a second source names it further down, `// Also ported from + v ()` followed by its notice. Ported test suites keep their `describe`/`it`, as + upyo's do; the `expect` of vitest and bun:test becomes `node:assert`. +- **Not ported, on purpose:** the DKIM, SES over the AWS SDK, `addressparser`, `fetch`, `punycode` + and plugin pipeline of nodemailer (this package does each better or has it); the SMTP client, + MIME builder, DKIM, fallback, round-robin and retry of unemail (`./smtp`, `./mime`, `./pool` and + `./retry` are safer); the `{{var}}` engine, SMS, WhatsApp and push, socket adapters, pool, retry, + fallback, rate limiter and idempotency of sently (unemail's are the base where one is needed). + +**Upstream bugs fixed in the port** (each in the `feat:` commit of the feature, with a test that +fails on upstream's behavior): + +None yet. + ## Repository layout ``` @@ -542,8 +579,11 @@ optional React Email adapter (no template engine of its own); more providers. ported from upyo"), and the correctness suite of Phase 1 still applies to it. - **BullMQ is one driver, not the base:** it ties the queue to Redis/Valkey; the MongoDB outbox gives a queue with no new infrastructure. -- **Few new providers.** The eight HTTP providers of upyo come with the port. Each provider adds - its webhooks and API drift to maintain, so another one waits for a user who needs it. +- **Eighteen providers, then few new ones.** The eight HTTP providers of upyo came with the port; + 1.1 adds the ten most used of unemail and sently (Postmark, Brevo, Mailjet, MailerSend, + SparkPost, ZeptoMail, SMTP2GO, Azure Communication Services, Scaleway and Cloudflare). Each + provider adds its webhooks and API drift to maintain, so another one waits for a user who needs + it. **What the competitors appear to get wrong, to not repeat** (findings from reading their code in October 2026, not reproduced except for emailjs, which was run against fake SMTP servers): no diff --git a/LICENSE b/LICENSE index 40405b4..aae508f 100644 --- a/LICENSE +++ b/LICENSE @@ -45,3 +45,82 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +Portions of this software are derived from Nodemailer +(https://github.com/nodemailer/nodemailer), version 10.0.16, which is +distributed under the following license (MIT No Attribution): + +Copyright (c) 2011-2023 Andris Reinman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- + +Portions of this software are derived from unemail +(https://github.com/productdevbook/unemail), version 0.7.0, which is +distributed under the following license: + +MIT License + +Copyright (c) 2025 Wind + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- + +Portions of this software are derived from sently +(https://github.com/omqkhafi/sently), version 1.2.2, which is distributed +under the following license: + +MIT License + +Copyright (c) 2026 sendx contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/core/id.js b/src/core/id.js index 8d8dacc..b3a6be9 100644 --- a/src/core/id.js +++ b/src/core/id.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the identifiers the library generates, from a generator +// Not ported: the identifiers the library generates, from a generator // the application may pass in (cuid2, ulid, nanoid, ...) to correlate them with // its own records, checked before they reach a header field or a request. diff --git a/src/core/text.js b/src/core/text.js index 1fbe8ef..c86485a 100644 --- a/src/core/text.js +++ b/src/core/text.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the character classes RFC 5322 builds atoms and address +// Not ported: the character classes RFC 5322 builds atoms and address // literals from, as tables indexed by char code, and a scan of them in one pass. const DOT = 0x2e; diff --git a/src/internal/base64.js b/src/internal/base64.js index d712727..80d9906 100644 --- a/src/internal/base64.js +++ b/src/internal/base64.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the one base64 encoder of the HTTP providers, which +// Not ported: the one base64 encoder of the HTTP providers, which // carried seven, from String.fromCharCode with btoa to string concatenation // byte by byte, all far slower than Buffer, which Node always has. diff --git a/src/internal/batch.js b/src/internal/batch.js index b154527..def24af 100644 --- a/src/internal/batch.js +++ b/src/internal/batch.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the messages of a sendMany() read as they are needed +// Not ported: the messages of a sendMany() read as they are needed // rather than all at once, so that it holds a batch in memory, not its input. const noop = () => {}; diff --git a/src/internal/body.js b/src/internal/body.js index 8b14e85..0f6cc8c 100644 --- a/src/internal/body.js +++ b/src/internal/body.js @@ -1,4 +1,4 @@ -// Not ported from upyo: reads the body of a response within a bound, so that +// Not ported: reads the body of a response within a bound, so that // a broken or hostile server cannot fill the memory of a sender. // Far above any answer of a provider (a batch of receipts, a JMAP session or diff --git a/src/internal/config.js b/src/internal/config.js index 599c5b7..a3050fb 100644 --- a/src/internal/config.js +++ b/src/internal/config.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the checks of the options that every HTTP provider +// Not ported: the checks of the options that every HTTP provider // shares, made once, when its configuration is made. const { isLoopbackHost } = require('./loopback.js'); diff --git a/src/internal/delivery-unknown.js b/src/internal/delivery-unknown.js index 19091a6..fa62337 100644 --- a/src/internal/delivery-unknown.js +++ b/src/internal/delivery-unknown.js @@ -1,4 +1,4 @@ -// Not ported from upyo: marks the failures after which a message may have +// Not ported: marks the failures after which a message may have // been accepted, so that a transport reports them as of unknown delivery // rather than as retryable, whatever class of error carries them. diff --git a/src/internal/headers.js b/src/internal/headers.js index 8d4a332..cd73e5b 100644 --- a/src/internal/headers.js +++ b/src/internal/headers.js @@ -1,4 +1,4 @@ -// Not ported from upyo: small helpers of header names that the converters of +// Not ported: small helpers of header names that the converters of // the HTTP providers share. /** diff --git a/src/internal/http.js b/src/internal/http.js index a6f7b08..1ff78f3 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the retry loop that the HTTP providers carried as +// Not ported: the retry loop that the HTTP providers carried as // copies. A provider keeps what is its own (the request, its errors, what it // retries and how long it waits) and passes it in. diff --git a/src/internal/log.js b/src/internal/log.js index 89144ca..5f07b72 100644 --- a/src/internal/log.js +++ b/src/internal/log.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the logging of what happens inside a transport, through +// Not ported: the logging of what happens inside a transport, through // the pino-like logger the application passes in. Without one a transport // writes nothing, except the warnings of warnDegradation. diff --git a/src/internal/loopback.js b/src/internal/loopback.js index 01624bd..806cca2 100644 --- a/src/internal/loopback.js +++ b/src/internal/loopback.js @@ -1,4 +1,4 @@ -// Not ported from upyo: whether a host is this machine, the one place where +// Not ported: whether a host is this machine, the one place where // credentials may go over plain text: SMTP AUTH without TLS, an OAuth 2.0 token // endpoint or an HTTP provider over http:, to a local fake server. One rule // for all three, which upstream wrote three times, differently. diff --git a/src/internal/provider.js b/src/internal/provider.js index bdc0886..1c36ed5 100644 --- a/src/internal/provider.js +++ b/src/internal/provider.js @@ -1,4 +1,4 @@ -// Not ported from upyo: what the message converters of the HTTP providers +// Not ported: what the message converters of the HTTP providers // share, once in place of a copy in each provider. const { combineSignals } = require('../core/abort-signal.js'); diff --git a/src/internal/redact-addresses.js b/src/internal/redact-addresses.js index 91d4e44..f17ae0d 100644 --- a/src/internal/redact-addresses.js +++ b/src/internal/redact-addresses.js @@ -1,4 +1,4 @@ -// Not ported from upyo: keeps email addresses out of error text that goes to +// Not ported: keeps email addresses out of error text that goes to // telemetry and logs, such as an SMTP rejection naming its recipient. const REDACTED = '[REDACTED]'; diff --git a/src/internal/redact.js b/src/internal/redact.js index 8916465..c3c1345 100644 --- a/src/internal/redact.js +++ b/src/internal/redact.js @@ -1,4 +1,4 @@ -// Not ported from upyo: keeps credentials out of a transport that is logged, +// Not ported: keeps credentials out of a transport that is logged, // inspected or serialized. The configuration stays readable as it is. const { EventEmitter } = require('node:events'); diff --git a/src/internal/sleep.js b/src/internal/sleep.js index 8743ea1..ebc46fa 100644 --- a/src/internal/sleep.js +++ b/src/internal/sleep.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the wait of a retry or a delay, which upstream wrote in +// Not ported: the wait of a retry or a delay, which upstream wrote in // five ways, some deaf to the AbortSignal or rejecting with an error of their // own instead of its reason. diff --git a/src/internal/unsent.js b/src/internal/unsent.js index af1e1cf..11b60c8 100644 --- a/src/internal/unsent.js +++ b/src/internal/unsent.js @@ -1,4 +1,4 @@ -// Not ported from upyo: tells a request that never left from one that may +// Not ported: tells a request that never left from one that may // have reached the server, for a request that must not be sent twice. // System error codes of a connection that was never made, so no request went diff --git a/src/internal/validate-ssl.js b/src/internal/validate-ssl.js index 9533298..4726c1f 100644 --- a/src/internal/validate-ssl.js +++ b/src/internal/validate-ssl.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the check of `validateSsl`, in a module of its own so +// Not ported: the check of `validateSsl`, in a module of its own so // that only the providers that take the option bundle it. /** diff --git a/src/version.js b/src/version.js index 7b424aa..3914683 100644 --- a/src/version.js +++ b/src/version.js @@ -1,4 +1,4 @@ -// Not ported from upyo: the package's own name and version, for the +// Not ported: the package's own name and version, for the // instrumentation scope of the OpenTelemetry tracer and meter: a relative // require of package.json would bundle the whole manifest into ./otel. // tests/unit/version.test.js keeps it equal to package.json, and the release diff --git a/tests/unit/attribution.test.js b/tests/unit/attribution.test.js index a73e109..1cefec5 100644 --- a/tests/unit/attribution.test.js +++ b/tests/unit/attribution.test.js @@ -1,16 +1,77 @@ // Every file the package ships says where it comes from: the attribution header -// of upyo, whose code most of the transport is a port of (MIT, the notice is in -// LICENSE), or that it is not ported (CLAUDE.md, "Code ported from upyo"). +// of the project it is a port of (upyo for most of the transport; nodemailer, +// unemail and sently for the features of 1.1; the notices are in LICENSE), or +// that it is not ported (CLAUDE.md, "Code ported from upyo" and "Code ported +// from nodemailer, unemail and sently"). const { test } = require('node:test'); const assert = require('node:assert'); const fs = require('node:fs'); const path = require('node:path'); const ROOT = path.join(__dirname, '../..'); -const NOTICE = '// https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee'; -const PORTED = /^\/\/ Ported from upyo v0\.7\.0 \(packages\/[a-z-]+\/src\/[\w./-]+\.ts[),]/; -const PORTED_TYPES = /^\/\/ Types ported from upyo v0\.7\.0 \(packages\/[a-z-]+\/src[/)]/; -const NOT_PORTED = '// Not ported from upyo: '; + +// The path a header names: a source file, or for a d.ts, whose types come from +// several files, a directory. +const UPYO_FILE = 'packages\\/[a-z-]+\\/src\\/[\\w./-]+\\.ts'; +const UPYO_TYPES = 'packages\\/[a-z-]+\\/src(?:\\/[\\w./-]+)?'; +const SRC_FILE = 'src\\/[\\w./-]+\\.ts'; +const SRC_TYPES = 'src(?:\\/[\\w./-]+)?'; + +const SOURCES = [ + { + name: 'upyo', + version: '0.7.0', + notice: '// https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee', + license: /derived from Upyo \(https:\/\/github\.com\/dahlia\/upyo\)[\s\S]*?2025 Hong Minhee/, + file: UPYO_FILE, + types: UPYO_TYPES, + }, + { + name: 'nodemailer', + version: '10.0.16', + notice: + '// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman', + license: + /derived from Nodemailer\s+\(https:\/\/github\.com\/nodemailer\/nodemailer\), version 10\.0\.16[\s\S]*?2011-2023 Andris Reinman/, + file: SRC_FILE, + types: SRC_TYPES, + }, + { + name: 'unemail', + version: '0.7.0', + notice: '// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind', + license: + /derived from unemail\s+\(https:\/\/github\.com\/productdevbook\/unemail\), version 0\.7\.0[\s\S]*?2025 Wind/, + file: SRC_FILE, + types: SRC_TYPES, + }, + { + name: 'sently', + version: '1.2.2', + notice: '// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors', + license: + /derived from sently\s+\(https:\/\/github\.com\/omqkhafi\/sently\), version 1\.2\.2[\s\S]*?2026 sendx contributors/, + file: SRC_FILE, + types: SRC_TYPES, + }, +]; + +const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +// The header lines that name a source: the first line of a port (`Ported +// from`, or `Types ported from` in a d.ts), and `Also ported from` for a +// second source further down. +const patternsOf = (source) => { + const prefix = `${source.name} v${escape(source.version)} \\(`; + return { + ported: new RegExp(`^\\/\\/ Ported from ${prefix}${source.file}[),]`), + types: new RegExp(`^\\/\\/ Types ported from ${prefix}${source.types}[),]`), + also: new RegExp(`^\\/\\/ Also ported from ${prefix}(?:${source.file}|${source.types})[),]`), + }; +}; + +const PATTERNS = SOURCES.map((source) => ({ source, ...patternsOf(source) })); +const NOT_PORTED = '// Not ported: '; const shippedFiles = () => { const files = ['index.d.ts']; @@ -39,15 +100,23 @@ const problemOf = (file, text) => { const header = headerOf(text); if (header.length === 0) return `${file} has no header`; if (header[0].startsWith(NOT_PORTED)) return undefined; - const pattern = file.endsWith('.d.ts') ? PORTED_TYPES : PORTED; - if (!pattern.test(header[0])) { - return `${file} starts with neither the upyo header nor "${NOT_PORTED.trim()}"`; + const first = file.endsWith('.d.ts') ? 'types' : 'ported'; + const named = PATTERNS.filter( + (pattern) => + pattern[first].test(header[0]) || header.slice(1).some((line) => pattern.also.test(line)), + ); + if (!PATTERNS.some((pattern) => pattern[first].test(header[0]))) { + return `${file} starts with neither a ported header nor "${NOT_PORTED.trim()}"`; + } + for (const { source } of named) { + if (!header.includes(source.notice)) { + return `${file} has the ${source.name} header without its notice`; + } } - if (!header.includes(NOTICE)) return `${file} has the upyo header without its notice`; return undefined; }; -test('Attribution: every shipped file has the upyo header or says it is not ported', () => { +test('Attribution: every shipped file has a ported header or says it is not ported', () => { // The root index.js is a one-line shim over src/index.js. const problems = []; for (const file of shippedFiles()) { @@ -58,7 +127,8 @@ test('Attribution: every shipped file has the upyo header or says it is not port }); test('Attribution: the check refuses a file without a header or without the notice', () => { - const notice = `${NOTICE}\n`; + const upyo = SOURCES[0]; + const notice = `${upyo.notice}\n`; const ported = '// Ported from upyo v0.7.0 (packages/smtp/src/smtp-connection.ts)\n'; assert.strictEqual(problemOf('src/a.js', `${ported}${notice}code`), undefined); assert.strictEqual(problemOf('src/a.js', `${NOT_PORTED}a helper.\ncode`), undefined); @@ -73,10 +143,60 @@ test('Attribution: the check refuses a file without a header or without the noti ); assert.match(problemOf('src/a.js', '// Some code.\ncode'), /starts with neither/); assert.match(problemOf('src/a.d.ts', `${ported}${notice}`), /starts with neither/); + // The marker of upyo's port alone is no longer enough. + assert.match( + problemOf('src/a.js', '// Not ported from upyo: a helper.\n'), + /starts with neither/, + ); +}); + +test('Attribution: every source has its own header, version and notice', () => { + const nodemailer = '// Ported from nodemailer v10.0.16 (src/sendmail-transport/index.ts)\n'; + const unemail = '// Ported from unemail v0.7.0 (src/middleware/rate-limit.ts)\n'; + const sently = '// Ported from sently v1.2.2 (src/webhooks/resend.ts)\n'; + const [, nodemailerSource, unemailSource, sentlySource] = SOURCES; + assert.strictEqual( + problemOf('src/a.js', `${nodemailer}${nodemailerSource.notice}\ncode`), + undefined, + ); + assert.strictEqual(problemOf('src/a.js', `${unemail}${unemailSource.notice}\n`), undefined); + assert.strictEqual(problemOf('src/a.js', `${sently}${sentlySource.notice}\n`), undefined); + assert.strictEqual( + problemOf( + 'src/a.d.ts', + `// Types ported from unemail v0.7.0 (src/drivers)\n${unemailSource.notice}\n`, + ), + undefined, + ); + // Another version, or another source's notice, is refused. + assert.match( + problemOf('src/a.js', `// Ported from unemail v0.6.0 (src/a.ts)\n${unemailSource.notice}\n`), + /starts with neither/, + ); + assert.strictEqual( + problemOf('src/a.js', `${unemail}${sentlySource.notice}\n`), + 'src/a.js has the unemail header without its notice', + ); +}); + +test('Attribution: a second source needs its own notice', () => { + const [, , unemail, sently] = SOURCES; + const header = [ + '// Ported from unemail v0.7.0 (src/drivers/postmark.ts)', + unemail.notice, + '// Also ported from sently v1.2.2 (src/transports/postmark.ts)', + ]; + assert.strictEqual( + problemOf('src/a.js', `${header.join('\n')}\n${sently.notice}\ncode`), + undefined, + ); + assert.strictEqual( + problemOf('src/a.js', `${header.join('\n')}\ncode`), + 'src/a.js has the sently header without its notice', + ); }); -test('Attribution: LICENSE carries the notice of upyo', () => { +test('Attribution: LICENSE carries the notice of every source', () => { const license = fs.readFileSync(path.join(ROOT, 'LICENSE'), 'utf8'); - assert.match(license, /derived from Upyo \(https:\/\/github\.com\/dahlia\/upyo\)/); - assert.match(license, /Copyright 2025 Hong Minhee/); + for (const source of SOURCES) assert.match(license, source.license, source.name); }); From 83e5752410fc100589ca155d7a816fe9271c9ab4 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:00:17 +0300 Subject: [PATCH 002/143] feat(core): write IDN domains as A-labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Unicode domain under an ASCII local part needed SMTPUTF8, so mail to user@bücher.de failed on any server without it, while nodemailer sends it as user@xn--bcher-kva.de. toAsciiAddress, after nodemailer's normalizeDomain, maps the domain with domainToASCII (IDNA with the mapping of UTS #46) and leaves a domain at which the host parser would cut, an address literal and a local part beyond ASCII as given. It is applied where a message is written: header fields, SMTP envelopes (overrides and raw messages too), JMAP and the provider converters. The Message keeps the address as the application gave it. Co-Authored-By: Claude Opus 5.5 --- docs/messages/compose.md | 9 +- docs/transports/smtp.md | 16 +- src/core/address.js | 51 +++++- src/jmap/message-converter.js | 3 +- src/jmap/raw-delivery.js | 5 +- src/lettermint/message-converter.js | 5 +- src/maileroo/message-converter.js | 5 +- src/mailtrap/message-converter.js | 5 +- src/mime/index.js | 5 +- src/mime/message.js | 7 +- src/plunk/message-converter.js | 14 +- src/sendgrid/message-converter.js | 3 +- src/smtp/envelope.js | 23 ++- src/smtp/raw-message.js | 7 +- tests/unit/idn.test.js | 150 ++++++++++++++++++ ...smtp-message-converter.integration.test.js | 5 +- 16 files changed, 277 insertions(+), 36 deletions(-) create mode 100644 tests/unit/idn.test.js diff --git a/docs/messages/compose.md b/docs/messages/compose.md index e205466..9f3e5b7 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -89,7 +89,14 @@ The local part is limited to 64 UTF-8 octets. @alexify/mailer preserves the spe Unicode normalization supplied by the caller because changing a local part can change the mailbox it identifies. -Provider support still depends on the selected transport. The +A Unicode domain under an ASCII local part needs no extension: when the message is +written, into its header fields, an SMTP envelope, a JMAP request or a provider's API, +the domain goes as its A-label (IDNA with the mapping of UTS #46, as browsers and DNS +resolvers apply it), so `user@bücher.de` is sent as `user@xn--bcher-kva.de`. The +`Message` keeps the address as you gave it. An address literal, and a local part beyond +ASCII with its domain, are written as given. + +Provider support for a UTF-8 local part still depends on the selected transport. The [SMTP transport](/transports/smtp#internationalized-addresses) negotiates the `SMTPUTF8` extension automatically and reports a failed receipt when the server cannot accept an internationalized address. diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index e4af661..3774193 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -386,11 +386,15 @@ override does not alter message headers or DKIM signatures. @alexify/mailer automatically uses the `SMTPUTF8` extension defined by [RFC 6531] when the effective SMTP envelope or a visible From, To, Cc, or Reply-To mailbox contains -a non-ASCII character. The default envelope includes the message's Bcc +a non-ASCII local part. The default envelope includes the message's Bcc addresses. The transport requires the server to advertise both `SMTPUTF8` and `8BITMIME`, then adds `BODY=8BITMIME SMTPUTF8` to `MAIL FROM`. This supports -UTF-8 local parts and Unicode domain labels without another configuration -option. +UTF-8 local parts without another configuration option. + +A Unicode domain under an ASCII local part does not require SMTPUTF8: the envelope +(including an `envelope` override and the envelope of a raw message) and the header +fields carry its A-label, so `user@bücher.de` goes as `RCPT TO:` +to any server. If either required extension is missing, @alexify/mailer returns a non-retryable failed receipt with the code `smtp.smtputf8-unsupported` before sending `MAIL FROM`. @@ -398,9 +402,9 @@ The connection remains available for a later ASCII-only message. Unicode display names and subjects do not by themselves require SMTPUTF8. @alexify/mailer continues to encode those values as RFC 2047 encoded words, so an address -such as `José ` follows the ordinary ASCII SMTP flow. An -ASCII A-label domain such as `xn--r8jz45g.xn--zckzah` likewise does not require -SMTPUTF8, while its Unicode U-label form does. +such as `José ` follows the ordinary ASCII SMTP flow, and so does +`support@例え.テスト`, written as `support@xn--r8jz45g.xn--zckzah`. A verbatim custom +header with a Unicode address in it is written as given and does require SMTPUTF8. [RFC 6531]: https://www.rfc-editor.org/rfc/rfc6531 diff --git a/src/core/address.js b/src/core/address.js index 1b829df..0ebe349 100644 --- a/src/core/address.js +++ b/src/core/address.js @@ -1,7 +1,11 @@ // Ported from upyo v0.7.0 (packages/core/src/address.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the A-label of +// a domain, `toAsciiAddress`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const net = require('node:net'); +const { domainToASCII } = require('node:url'); const { ATEXT, isDotAtomText } = require('./text.js'); /** @@ -74,9 +78,9 @@ const checkDisplayName = (name) => { * @internal */ const formatMailbox = (address) => { - if (!address.name) return address.address; + if (!address.name) return toAsciiAddress(address.address); checkDisplayName(address.name); - return `${formatPhrase(address.name)} <${address.address}>`; + return `${formatPhrase(address.name)} <${toAsciiAddress(address.address)}>`; }; /** @@ -89,9 +93,9 @@ const formatMailbox = (address) => { * @internal */ const formatQuotedMailbox = (address) => { - if (!address.name) return address.address; + if (!address.name) return toAsciiAddress(address.address); checkDisplayName(address.name); - return `${quoteString(address.name)} <${address.address}>`; + return `${quoteString(address.name)} <${toAsciiAddress(address.address)}>`; }; /** @@ -342,6 +346,44 @@ const isEmailAddress = (email) => { return typeof email === 'string' && isValidEmail(email); }; +// domainToASCII is a WHATWG host parser rather than a plain IDNA mapper: it +// cuts the host at "/", "\\", "?" and "#", drops controls and percent-decodes, +// so "evil.example/mail.corp.example" would come back as the deliverable +// "evil.example". None of these belongs in a domain, and a domain holding one +// is left as it is. +// oxlint-disable-next-line no-control-regex -- the controls are what it finds +const URL_PARSER_UNSAFE = /[/\\?#%\x00-\x20\x7f]/; + +const hasNonAscii = (text) => { + for (let i = 0; i < text.length; i++) { + if (text.charCodeAt(i) > 0x7f) return true; + } + return false; +}; + +/** + * Writes the domain of an address as an A-label (IDNA with the mapping of + * UTS #46, as browsers and resolvers do) when its local part is ASCII, so that + * `user@bücher.de` goes as `user@xn--bcher-kva.de` and needs no SMTPUTF8. A + * local part beyond ASCII needs SMTPUTF8 anyway, and its domain stays as given; + * so does an address literal, and a domain the mapper does not take. + * + * @param address A valid address. + * @returns The address to write into a header field, an envelope or a request. + * @internal + */ +const toAsciiAddress = (address) => { + if (!hasNonAscii(address)) return address; + const at = findLocalPartEnd(address); + if (at === -1) return address; + const localPart = address.slice(0, at); + const domain = address.slice(at + 1); + if (hasNonAscii(localPart) || domain.charCodeAt(0) === 0x5b) return address; + if (URL_PARSER_UNSAFE.test(domain)) return address; + const ascii = domainToASCII(domain); + return ascii === '' ? address : `${localPart}@${ascii}`; +}; + module.exports = { formatAddress, formatMailbox, @@ -349,4 +391,5 @@ module.exports = { formatQuotedMailbox, parseAddress, isEmailAddress, + toAsciiAddress, }; diff --git a/src/jmap/message-converter.js b/src/jmap/message-converter.js index 92bc534..534cd30 100644 --- a/src/jmap/message-converter.js +++ b/src/jmap/message-converter.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { resolveCalendarContent } = require('../core/calendar.js'); +const { toAsciiAddress } = require('../core/address.js'); const { parseMessageId } = require('../core/message-id.js'); /** @@ -11,7 +12,7 @@ const { parseMessageId } = require('../core/message-id.js'); */ const formatAddress = (address) => { const result = { - email: address.address, + email: toAsciiAddress(address.address), }; if (address.name) { diff --git a/src/jmap/raw-delivery.js b/src/jmap/raw-delivery.js index 690507e..5f6752b 100644 --- a/src/jmap/raw-delivery.js +++ b/src/jmap/raw-delivery.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/jmap/src/raw-delivery.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { toAsciiAddress } = require('../core/address.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { RawMessageValidationError } = require('../core/raw-message.js'); const { JMAP_ERROR_TYPES, JmapApiError } = require('./errors.js'); @@ -228,7 +229,7 @@ const existingEmailId = async (execute, accountId, existingId, blobId) => { */ const submitRawEmail = async (execute, accountId, emailId, identityId, envelope) => { const rcptTo = []; - for (const email of envelope.to) rcptTo.push({ email }); + for (const email of envelope.to) rcptTo.push({ email: toAsciiAddress(email) }); const submitted = methodResult( await execute({ using, @@ -241,7 +242,7 @@ const submitRawEmail = async (execute, accountId, emailId, identityId, envelope) raw: { emailId, identityId, - envelope: { mailFrom: { email: envelope.from ?? '' }, rcptTo }, + envelope: { mailFrom: { email: toAsciiAddress(envelope.from ?? '') }, rcptTo }, }, }, }, diff --git a/src/lettermint/message-converter.js b/src/lettermint/message-converter.js index 9847238..23ae256 100644 --- a/src/lettermint/message-converter.js +++ b/src/lettermint/message-converter.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { generateId } = require('../core/id.js'); +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { @@ -120,9 +121,9 @@ const convertSettings = (config) => { const formatAddress = (address) => { if (address.name) { const escapedName = address.name.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); - return `"${escapedName}" <${address.address}>`; + return `"${escapedName}" <${toAsciiAddress(address.address)}>`; } - return address.address; + return toAsciiAddress(address.address); }; const convertAttachment = async (attachment, signal) => { diff --git a/src/maileroo/message-converter.js b/src/maileroo/message-converter.js index 4d47883..bdf1516 100644 --- a/src/maileroo/message-converter.js +++ b/src/maileroo/message-converter.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { generateId } = require('../core/id.js'); +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { @@ -119,8 +120,8 @@ const hashReference = (value) => { const convertAddress = (address) => { return address.name == null || address.name === '' - ? { address: address.address } - : { address: address.address, display_name: address.name }; + ? { address: toAsciiAddress(address.address) } + : { address: toAsciiAddress(address.address), display_name: address.name }; }; const convertAddressList = (addresses) => { diff --git a/src/mailtrap/message-converter.js b/src/mailtrap/message-converter.js index 29b35c4..cda6bd3 100644 --- a/src/mailtrap/message-converter.js +++ b/src/mailtrap/message-converter.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/mailtrap/src/message-converter.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { @@ -119,9 +120,9 @@ const buildCustomVariables = (tags, metadata) => { const formatAddress = (address) => { if (address.name) { - return { email: address.address, name: address.name }; + return { email: toAsciiAddress(address.address), name: address.name }; } - return { email: address.address }; + return { email: toAsciiAddress(address.address) }; }; const convertAttachment = async (attachment, signal) => { diff --git a/src/mime/index.js b/src/mime/index.js index a352c1c..937bd22 100644 --- a/src/mime/index.js +++ b/src/mime/index.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/mime/src/index.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { toAsciiAddress } = require('../core/address.js'); const { createRawMessagePlan } = require('../core/raw-message.js'); const { combineSignals } = require('../core/abort-signal.js'); const { prepareMimeMessage } = require('./message.js'); @@ -27,7 +28,7 @@ const { MimeAttachmentReplayError } = require('./stream.js'); const envelopeRecipients = (message) => { const addresses = []; for (const list of [message.recipients, message.ccRecipients, message.bccRecipients]) { - for (let i = 0; i < list.length; i++) addresses.push(list[i].address); + for (let i = 0; i < list.length; i++) addresses.push(toAsciiAddress(list[i].address)); } return addresses; }; @@ -38,7 +39,7 @@ const composeMessage = async (message, options = {}) => { const plan = prepareMimeMessage(message, options.dkim, options); const { envelope } = createRawMessagePlan({ envelope: { - from: message.sender.address, + from: toAsciiAddress(message.sender.address), to: envelopeRecipients(message), }, content: new Uint8Array(), diff --git a/src/mime/message.js b/src/mime/message.js index 0cbc436..3de609c 100644 --- a/src/mime/message.js +++ b/src/mime/message.js @@ -2,7 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { Buffer } = require('node:buffer'); -const { formatPhrase } = require('../core/address.js'); +const { formatPhrase, toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { checkAttachment } = require('../core/message.js'); const { @@ -541,9 +541,10 @@ const encodeAddress = (address) => { if (/[\r\n]/.test(address.address)) { throw new TypeError('Address must not contain a carriage return or line feed.'); } + const email = toAsciiAddress(address.address); if (address.name == null) { // No display name, just return the email address - return address.address; + return email; } // Encode only the display name part, leave email address as-is. An @@ -552,7 +553,7 @@ const encodeAddress = (address) => { const encodedDisplayName = encodeHeaderValue(address.name, true); const displayName = encodedDisplayName === address.name ? formatPhrase(address.name) : encodedDisplayName; - return `${displayName} <${address.address}>`; + return `${displayName} <${email}>`; }; // Whether a word of the value, between whitespace, is longer than `limit`. diff --git a/src/plunk/message-converter.js b/src/plunk/message-converter.js index 08e1af3..2783b2d 100644 --- a/src/plunk/message-converter.js +++ b/src/plunk/message-converter.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/plunk/src/message-converter.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { collectCustomHeaders, withCalendarAttachment } = require('../internal/provider.js'); @@ -27,21 +28,26 @@ const isDedicatedHeader = (name) => DEDICATED_HEADERS.has(name.toLowerCase()); const addressesOf = (message) => { const addresses = []; - for (const recipient of message.recipients) addresses.push(recipient.address); + for (const recipient of message.recipients) addresses.push(toAsciiAddress(recipient.address)); return addresses; }; const convertMessage = async (message, _config, signal) => { // Convert recipients - Plunk accepts either string or array - const to = message.recipients.length === 1 ? message.recipients[0].address : addressesOf(message); + const to = + message.recipients.length === 1 + ? toAsciiAddress(message.recipients[0].address) + : addressesOf(message); // Convert sender information const senderName = message.sender.name; - const senderEmail = message.sender.address; + const senderEmail = toAsciiAddress(message.sender.address); // Convert reply-to (use first reply recipient) const replyTo = - message.replyRecipients.length > 0 ? message.replyRecipients[0].address : undefined; + message.replyRecipients.length > 0 + ? toAsciiAddress(message.replyRecipients[0].address) + : undefined; // Convert content - HTML takes precedence over text let body; diff --git a/src/sendgrid/message-converter.js b/src/sendgrid/message-converter.js index 8e07415..c30be7e 100644 --- a/src/sendgrid/message-converter.js +++ b/src/sendgrid/message-converter.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/sendgrid/src/message-converter.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { @@ -121,7 +122,7 @@ const trackingSettingsOf = (config) => { */ const formatAddress = (address) => { const result = { - email: address.address, + email: toAsciiAddress(address.address), }; if (address.name) { diff --git a/src/smtp/envelope.js b/src/smtp/envelope.js index ebee135..218fb53 100644 --- a/src/smtp/envelope.js +++ b/src/smtp/envelope.js @@ -1,7 +1,7 @@ // Ported from upyo v0.7.0 (packages/smtp/src/envelope.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -const { isEmailAddress } = require('../core/address.js'); +const { isEmailAddress, toAsciiAddress } = require('../core/address.js'); /** * Error produced when an effective SMTP envelope is invalid. @@ -68,7 +68,7 @@ const resolveSmtpEnvelope = (message, override) => { } } - return { from, to: [...recipients] }; + return toAsciiEnvelope({ from, to: recipients }); }; /** @@ -121,9 +121,28 @@ const hasNonAsciiAddress = (envelope) => { return false; }; +/** + * Writes the domains of an envelope as A-labels where the local part is ASCII + * (`toAsciiAddress`), so that only a local part beyond ASCII asks for SMTPUTF8. + * + * @param envelope A validated envelope. + * @returns A new envelope. + * @internal + */ +const toAsciiEnvelope = (envelope) => { + const to = []; + for (let i = 0; i < envelope.to.length; i++) to.push(toAsciiAddress(envelope.to[i])); + return { + ...envelope, + from: envelope.from == null ? envelope.from : toAsciiAddress(envelope.from), + to, + }; +}; + module.exports = { SmtpEnvelopeValidationError, checkSmtpPath, hasNonAsciiAddress, resolveSmtpEnvelope, + toAsciiEnvelope, }; diff --git a/src/smtp/raw-message.js b/src/smtp/raw-message.js index 6a0883a..3a57c11 100644 --- a/src/smtp/raw-message.js +++ b/src/smtp/raw-message.js @@ -3,7 +3,7 @@ const { analyzeRawMessage, iterateRawMessage } = require('../core/raw-message.js'); const { readAttachmentContent } = require('../core/attachment.js'); -const { hasNonAsciiAddress } = require('./envelope.js'); +const { hasNonAsciiAddress, toAsciiEnvelope } = require('./envelope.js'); /** Missing support for explicitly declared 8-bit MIME bodies. @since 0.6.0 */ class Smtp8BitMimeUnsupportedError extends TypeError { @@ -16,10 +16,11 @@ class Smtp8BitMimeUnsupportedError extends TypeError { /** @internal */ const prepareRawSmtpMessage = (plan, dsn) => { + const envelope = { ...toAsciiEnvelope(plan.envelope), dsn }; return { rawMessage: plan, - envelope: { ...plan.envelope, dsn }, - requiresSmtpUtf8: plan.encoding === 'utf8' || hasNonAsciiAddress(plan.envelope), + envelope, + requiresSmtpUtf8: plan.encoding === 'utf8' || hasNonAsciiAddress(envelope), }; }; diff --git a/tests/unit/idn.test.js b/tests/unit/idn.test.js new file mode 100644 index 0000000..2d95e1a --- /dev/null +++ b/tests/unit/idn.test.js @@ -0,0 +1,150 @@ +// Not ported: the A-label of a domain beyond ASCII, written into header +// fields, SMTP envelopes, JMAP and provider requests (nodemailer converts it, +// upyo wrote the domain as given and asked for SMTPUTF8). +const { test } = require('node:test'); +const assert = require('node:assert'); +const { domainToUnicode } = require('node:url'); +const PostalMime = require('postal-mime'); +const { createMessage } = require('../../index.js'); +const { toAsciiAddress } = require('../../src/core/address.js'); +const { readAttachmentContent } = require('../../src/core/attachment.js'); +const { composeMessage } = require('../../src/mime/index.js'); +const { SmtpTransport } = require('../../src/smtp/index.js'); +const { convertMessage: convertSendGrid } = require('../../src/sendgrid/message-converter.js'); +const { createSendGridConfig } = require('../../src/sendgrid/config.js'); +const { formatAddress: formatJmapAddress } = require('../../src/jmap/message-converter.js'); +const { convertMessage: convertPlunk } = require('../../src/plunk/message-converter.js'); +const { MockSmtpServer } = require('../helpers/smtp/mock-smtp-server.js'); + +test('IDN: a domain beyond ASCII under an ASCII local part becomes its A-label', () => { + assert.strictEqual(toAsciiAddress('user@bücher.de'), 'user@xn--bcher-kva.de'); + assert.strictEqual(toAsciiAddress('support@例え.テスト'), 'support@xn--r8jz45g.xn--zckzah'); + assert.strictEqual(toAsciiAddress('info@пример.рф'), 'info@xn--e1afmkfd.xn--p1ai'); + assert.strictEqual(toAsciiAddress('"a@b"@bücher.de'), '"a@b"@xn--bcher-kva.de'); + // UTS #46 maps as browsers and resolvers do: a soft hyphen is ignored, so + // the domain mailed is the one a validator reads. + assert.strictEqual(toAsciiAddress('user@compa­ny.com'), 'user@company.com'); +}); + +test('IDN: ASCII, a local part beyond ASCII and address literals stay as given', () => { + assert.strictEqual(toAsciiAddress('User@Example.COM'), 'User@Example.COM'); + assert.strictEqual(toAsciiAddress('josé@bücher.de'), 'josé@bücher.de'); + assert.strictEqual(toAsciiAddress('用户@例子.广告'), '用户@例子.广告'); + assert.strictEqual(toAsciiAddress('user@[192.0.2.1]'), 'user@[192.0.2.1]'); + assert.strictEqual(toAsciiAddress('no-at-sign-ü'), 'no-at-sign-ü'); +}); + +test('IDN: a domain the URL parser would cut is left as it is', () => { + // domainToASCII('bücher.de/evil.example') gives 'xn--bcher-kva.de', which + // would mail a domain the sender never named. + assert.strictEqual(toAsciiAddress('user@bücher.de/evil.example'), 'user@bücher.de/evil.example'); + assert.strictEqual(toAsciiAddress('user@bücher.de?x'), 'user@bücher.de?x'); + assert.strictEqual(toAsciiAddress('user@bü%63her.de'), 'user@bü%63her.de'); +}); + +const idnMessage = () => + createMessage({ + from: { name: 'Jörg', address: 'jorg@bücher.de' }, + to: 'leser@例え.テスト', + cc: 'info@пример.рф', + bcc: 'blind@bücher.de', + replyTo: 'support@bücher.de', + subject: 'Bücher', + content: { text: 'Hallo' }, + }); + +test('IDN: composeMessage writes A-labels into the header fields and the envelope', async () => { + const message = idnMessage(); + // The message keeps the addresses as the application gave them. + assert.strictEqual(message.recipients[0].address, 'leser@例え.テスト'); + const raw = await composeMessage(message); + assert.deepStrictEqual(raw.envelope, { + from: 'jorg@xn--bcher-kva.de', + to: ['leser@xn--r8jz45g.xn--zckzah', 'info@xn--e1afmkfd.xn--p1ai', 'blind@xn--bcher-kva.de'], + }); + assert.strictEqual(raw.encoding, '7bit'); + const bytes = await readAttachmentContent(raw.content); + const text = new TextDecoder().decode(bytes); + assert.match(text, /^To: leser@xn--r8jz45g\.xn--zckzah\r$/m); + const parsed = await PostalMime.parse(bytes); + assert.strictEqual(parsed.from.address, 'jorg@xn--bcher-kva.de'); + assert.strictEqual(parsed.from.name, 'Jörg'); + assert.strictEqual(domainToUnicode(parsed.to[0].address.split('@')[1]), '例え.テスト'); + assert.strictEqual(parsed.cc[0].address, 'info@xn--e1afmkfd.xn--p1ai'); + assert.strictEqual(parsed.replyTo[0].address, 'support@xn--bcher-kva.de'); +}); + +test('IDN: SMTP sends to a domain beyond ASCII to a server without SMTPUTF8', async () => { + const server = new MockSmtpServer(); + const port = await server.start(); + try { + const transport = new SmtpTransport({ host: 'localhost', port, pool: false }); + const receipt = await transport.send(idnMessage()); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + const commands = server.getReceivedCommands(); + assert.ok(commands.includes('MAIL FROM:'), commands.join('\n')); + assert.ok(commands.includes('RCPT TO:')); + assert.ok(commands.includes('RCPT TO:')); + } finally { + await server.stop(); + } +}); + +test('IDN: an envelope override and a raw message go as A-labels too', async () => { + const server = new MockSmtpServer(); + const port = await server.start(); + try { + const transport = new SmtpTransport({ host: 'localhost', port, pool: false }); + const overridden = await transport.send(idnMessage(), { + envelope: { from: 'bounce@bücher.de', to: ['other@bücher.de'] }, + }); + assert.strictEqual(overridden.successful, true, overridden.errorMessages?.join()); + const raw = await transport.sendRaw({ + envelope: { from: 'a@bücher.de', to: ['b@пример.рф'] }, + content: new TextEncoder().encode( + 'From: a@xn--bcher-kva.de\r\nTo: b@xn--e1afmkfd.xn--p1ai\r\n\r\nHi\r\n', + ), + }); + assert.strictEqual(raw.successful, true, raw.errorMessages?.join()); + const commands = server.getReceivedCommands(); + assert.ok(commands.includes('MAIL FROM:'), commands.join('\n')); + assert.ok(commands.includes('RCPT TO:')); + assert.ok(commands.includes('MAIL FROM:')); + assert.ok(commands.includes('RCPT TO:')); + } finally { + await server.stop(); + } +}); + +test('IDN: a local part beyond ASCII still asks for SMTPUTF8', async () => { + const server = new MockSmtpServer(); + const port = await server.start(); + try { + const transport = new SmtpTransport({ host: 'localhost', port, pool: false }); + const receipt = await transport.send( + createMessage({ + from: 'a@example.com', + to: 'josé@bücher.de', + subject: 'S', + content: { text: 'T' }, + }), + ); + assert.strictEqual(receipt.successful, false); + assert.match(receipt.errorMessages.join(), /SMTPUTF8/); + } finally { + await server.stop(); + } +}); + +test('IDN: providers and JMAP get the A-label', async () => { + const message = idnMessage(); + const sendgrid = await convertSendGrid(message, createSendGridConfig({ apiKey: 'key' })); + assert.strictEqual(sendgrid.from.email, 'jorg@xn--bcher-kva.de'); + assert.strictEqual(sendgrid.personalizations[0].to[0].email, 'leser@xn--r8jz45g.xn--zckzah'); + const plunk = await convertPlunk(message, {}); + assert.strictEqual(plunk.to, 'leser@xn--r8jz45g.xn--zckzah'); + assert.strictEqual(plunk.from, 'jorg@xn--bcher-kva.de'); + assert.deepStrictEqual(formatJmapAddress({ address: 'leser@例え.テスト' }), { + email: 'leser@xn--r8jz45g.xn--zckzah', + }); +}); diff --git a/tests/unit/smtp-message-converter.integration.test.js b/tests/unit/smtp-message-converter.integration.test.js index cd4de2d..ac842bc 100644 --- a/tests/unit/smtp-message-converter.integration.test.js +++ b/tests/unit/smtp-message-converter.integration.test.js @@ -279,7 +279,10 @@ describe('Message Converter Integration Tests', () => { assert.equal(sender.requiresSmtpUtf8, true); assert.equal(recipient.requiresSmtpUtf8, true); - assert.equal(replyTo.requiresSmtpUtf8, true); + // Changed in the port: a domain beyond ASCII under an ASCII local part + // goes as its A-label, which needs no SMTPUTF8. + assert.equal(replyTo.requiresSmtpUtf8, false); + assert.ok(replyTo.raw.includes('Reply-To: support@xn--r8jz45g.xn--zckzah')); assert.ok(sender.raw.includes('From: josé@example.com')); }); From a3ef8537d16727dbae78a01bfc41637a0860b85a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:07:45 +0300 Subject: [PATCH 003/143] feat(mime): write custom field names capitalized Headers keeps names in lower case, so every custom field went out as x-custom-header or list-unsubscribe-post. The MIME writer capitalizes each word of the name, as nodemailer's _normalizeHeaderKey does, with MIME, DKIM, ARC and BIMI at the start, -ID, -SPF, -FBL and -MD5 at the end and X-SMTPAPI upper-cased, and Content-features as RFC 2912 spells it. Names are case-insensitive, so readers and DKIM verification see no difference; ported tests that pinned lower case say so. Co-Authored-By: Claude Opus 5.5 --- docs/messages/compose.md | 6 +++ src/mime/message.js | 15 ++++++- tests/unit/mime-compose.test.js | 3 +- tests/unit/mime.test.js | 44 ++++++++++++++++++- ...smtp-message-converter.integration.test.js | 17 ++++--- tests/unit/smtp-transport.dkim.test.js | 5 ++- tests/unit/smtp-transport.integration.test.js | 5 ++- 7 files changed, 81 insertions(+), 14 deletions(-) diff --git a/docs/messages/compose.md b/docs/messages/compose.md index 9f3e5b7..652b3fa 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -218,6 +218,12 @@ The `createMessage()` function handles all the complexity of email message construction, ensuring that your messages are properly formatted and valid before sending them through your chosen transport. +`Headers` keeps names in lower case, so a transport that composes the message +writes each custom field name capitalized, as nodemailer does: `X-Custom-Header`, +with `MIME`, `DKIM`, `ARC` and `BIMI` at the start and `-ID`, `-SPF`, `-FBL` and +`-MD5` at the end upper-cased (`X-Entity-Ref-ID`, `Received-SPF`), and `X-SMTPAPI` +as SendGrid spells it. + [`Headers`]: https://developer.mozilla.org/en-US/docs/Web/API/Headers ### Headers the transport owns diff --git a/src/mime/message.js b/src/mime/message.js index 3de609c..70db7f4 100644 --- a/src/mime/message.js +++ b/src/mime/message.js @@ -1,5 +1,8 @@ // Ported from upyo v0.7.0 (packages/mime/src/message.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the case of +// a custom field name, `_normalizeHeaderKey`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const { Buffer } = require('node:buffer'); const { formatPhrase, toAsciiAddress } = require('../core/address.js'); @@ -239,10 +242,20 @@ const pushHeaderFields = (lines, message, generateId) => { const name = key.toLowerCase(); if (COMPOSED_HEADERS.has(name) || reservedHeaders.has(name)) continue; if (hasPriority && PRIORITY_HEADERS.has(name)) continue; - lines.push(foldHeader(key, encodeHeaderValue(value))); + lines.push(foldHeader(canonicalHeaderName(name), encodeHeaderValue(value))); } }; +// Each word of a field name capitalized, as nodemailer writes it, the +// abbreviations its readers know upper-cased: `Headers` keeps names in lower +// case, and `list-unsubscribe-post` reads as a field nobody named. +const UPPER_CASE_PARTS = + /^x-smtpapi$|^(?:mime|dkim|arc|bimi)\b|^[a-z]|-(?:spf|fbl|id|md5)$|-[a-z]/g; +const canonicalHeaderName = (name) => + name === 'content-features' + ? 'Content-features' + : name.replace(UPPER_CASE_PARTS, (part) => part.toUpperCase()); + const encodeAddressList = (addresses) => { let list = ''; for (let i = 0; i < addresses.length; i++) { diff --git a/tests/unit/mime-compose.test.js b/tests/unit/mime-compose.test.js index 277cf32..6527f3f 100644 --- a/tests/unit/mime-compose.test.js +++ b/tests/unit/mime-compose.test.js @@ -264,7 +264,8 @@ test('snapshots metadata and signing configuration before awaiting bytes', async signatures[0].headerFields.push('date'); resolve(new Uint8Array()); const wire = await collect(await composing); - assert.ok(wire.includes('x-test: before')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(wire.includes('X-Test: before')); assert.ok(wire.includes('2026')); assert.ok(wire.includes('s=test;')); assert.ok(wire.includes('h=from:subject;')); diff --git a/tests/unit/mime.test.js b/tests/unit/mime.test.js index f3667c2..8a9b88a 100644 --- a/tests/unit/mime.test.js +++ b/tests/unit/mime.test.js @@ -126,11 +126,53 @@ test('MIME: long headers fold at whitespace and unfold to the original', async ( assert.strictEqual((await PostalMime.parse(text)).subject, subject); }); +test('MIME: a custom field name is written capitalized, as nodemailer writes it', async () => { + const names = [ + 'x-custom-header', + 'list-unsubscribe-post', + 'list-id', + 'x-entity-ref-id', + 'received-spf', + 'x-report-fbl', + 'content-md5', + 'x-smtpapi', + 'arc-seal', + 'dkim-filter', + 'bimi-selector', + 'mime-extra', + 'archived-at', + 'content-features', + ]; + const headers = {}; + for (const name of names) headers[name] = 'v'; + const text = await wire(createMessage({ ...base, subject: 'a', headers })); + const written = text + .split('\r\n') + .filter((line) => line.endsWith(': v')) + .map((line) => line.slice(0, -3)); + assert.deepStrictEqual(written.sort(), [ + 'ARC-Seal', + 'Archived-At', + 'BIMI-Selector', + 'Content-MD5', + 'Content-features', + 'DKIM-Filter', + 'List-ID', + 'List-Unsubscribe-Post', + 'MIME-Extra', + 'Received-SPF', + 'X-Custom-Header', + 'X-Entity-Ref-ID', + 'X-Report-FBL', + 'X-SMTPAPI', + ]); +}); + test('MIME: a token too long to fold is refused', async () => { const message = createMessage({ ...base, subject: 'a', headers: { 'X-Long': 'y'.repeat(1200) } }); await assert.rejects(wire(message), { name: 'RangeError', - message: 'Header field x-long contains a token too long to fold.', + message: 'Header field X-Long contains a token too long to fold.', }); }); diff --git a/tests/unit/smtp-message-converter.integration.test.js b/tests/unit/smtp-message-converter.integration.test.js index ac842bc..0c43d9a 100644 --- a/tests/unit/smtp-message-converter.integration.test.js +++ b/tests/unit/smtp-message-converter.integration.test.js @@ -461,9 +461,10 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); // Should include all custom headers without encoding (ASCII only) - assert.ok(result.raw.includes('x-custom-header: Custom Value')); - assert.ok(result.raw.includes('x-mailer: Test Mailer')); - assert.ok(result.raw.includes('x-custom-info: ASCII Info')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(result.raw.includes('X-Custom-Header: Custom Value')); + assert.ok(result.raw.includes('X-Mailer: Test Mailer')); + assert.ok(result.raw.includes('X-Custom-Info: ASCII Info')); }); test('should preserve long structured ASCII header values', async () => { @@ -689,9 +690,10 @@ describe('Message Converter Integration Tests', () => { const message = createTestMessage({ headers }); const result = await convertMessage(message); - assert.ok(result.raw.includes('x-custom-header: Custom Value')); - assert.ok(result.raw.includes('x-mailer: Test Mailer')); - assert.ok(result.raw.includes('x-ascii-header: ASCII Value')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(result.raw.includes('X-Custom-Header: Custom Value')); + assert.ok(result.raw.includes('X-Mailer: Test Mailer')); + assert.ok(result.raw.includes('X-Ascii-Header: ASCII Value')); }); test('should handle reply-to addresses', async () => { @@ -886,7 +888,8 @@ describe('Message Converter Integration Tests', () => { }); const result = await convertMessage(createTestMessage({ headers })); - assert.ok(result.raw.includes('x-mailer: Test Mailer')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(result.raw.includes('X-Mailer: Test Mailer')); assert.ok(result.raw.includes('In-Reply-To: <788@example.com>')); }); }); diff --git a/tests/unit/smtp-transport.dkim.test.js b/tests/unit/smtp-transport.dkim.test.js index 6478860..ef8756f 100644 --- a/tests/unit/smtp-transport.dkim.test.js +++ b/tests/unit/smtp-transport.dkim.test.js @@ -339,8 +339,9 @@ describe('DKIM Integration Tests', () => { const result = await convertMessage(message, testDkimConfig); assert.ok(result.raw.includes('DKIM-Signature:')); - assert.ok(result.raw.includes('x-custom-header: Custom Value')); - assert.ok(result.raw.includes('x-mailer: Upyo Test')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(result.raw.includes('X-Custom-Header: Custom Value')); + assert.ok(result.raw.includes('X-Mailer: Upyo Test')); }); test('should sign email with multiple recipients', async () => { diff --git a/tests/unit/smtp-transport.integration.test.js b/tests/unit/smtp-transport.integration.test.js index 589245e..4a6f6b2 100644 --- a/tests/unit/smtp-transport.integration.test.js +++ b/tests/unit/smtp-transport.integration.test.js @@ -1195,8 +1195,9 @@ describe('SmtpTransport Integration Tests', () => { const receivedMessages = server.getReceivedMessages(); const receivedMessage = receivedMessages[0]; - assert.ok(receivedMessage.data.includes('x-custom-header: Custom Value')); - assert.ok(receivedMessage.data.includes('x-mailer: Test Mailer')); + // Changed in the port: a custom field name is written capitalized, as nodemailer writes it. + assert.ok(receivedMessage.data.includes('X-Custom-Header: Custom Value')); + assert.ok(receivedMessage.data.includes('X-Mailer: Test Mailer')); } finally { await teardownTest(server, transport); } From dcd706187ad4eef5e90e3bef17e3b17cc82d2d78 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:08:55 +0300 Subject: [PATCH 004/143] feat(core): List-* header fields and one-click unsubscription createMessage takes `list`: the List-Id of RFC 2919, the URL fields of RFC 2369 (help, unsubscribe, subscribe, post, owner, archive) and, with oneClick, the List-Unsubscribe-Post of RFC 8058 that Gmail and Yahoo ask of bulk senders. The fields go into Message.headers, so every transport that sends custom header fields sends them. The formatting is nodemailer's (` (comment)`, a bare address as mailto) and the one-click field unemail's. Fixed in the port: unemail built List-Unsubscribe after its line-break check, so a CRLF in the URL added header fields (U1), and turned one-click on for any URL, http included (U2); nodemailer wrote no List-Unsubscribe-Post, and neither signed it (N1). A URL with whitespace, controls or angle brackets, another scheme or none is refused rather than stripped or guessed, oneClick is opt-in and needs an https URL, and the default DKIM h= covers List-Unsubscribe and List-Unsubscribe-Post. The root budget goes to 9.5 KB. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 11 +- docs/messages/compose.md | 44 +++++++ docs/transports/smtp.md | 5 +- index.d.ts | 56 +++++++++ scripts/size-budgets.json | 2 +- src/core/list.js | 181 +++++++++++++++++++++++++++ src/core/message.js | 8 +- src/mime/dkim/types.js | 5 +- tests/types/index.test-d.ts | 32 +++++ tests/unit/list.test.js | 196 ++++++++++++++++++++++++++++++ tests/unit/mime-dkim-sign.test.js | 5 +- 11 files changed, 535 insertions(+), 10 deletions(-) create mode 100644 src/core/list.js create mode 100644 tests/unit/list.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 1e57c8f..2836073 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -464,7 +464,16 @@ sendx contributors). Their notices are in `LICENSE`. **Upstream bugs fixed in the port** (each in the `feat:` commit of the feature, with a test that fails on upstream's behavior): -None yet. +- N1. nodemailer: nothing wrote `List-Unsubscribe-Post` (a `list.unsubscribePost` key became + `List-Unsubscribepost`), and the default DKIM `h=` of neither nodemailer nor unemail covered it + or `List-Unsubscribe`, as RFC 8058 §4 requires. `oneClick` of `list` writes it, and the default + `h=` of `./mime` ends with `list-unsubscribe:list-unsubscribe-post`. +- U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL + after the check for line breaks in header values, so a CR or LF in the URL added header fields. + `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and + writes what `URL` serializes. +- U2. unemail: one-click went with any `unsubscribe.url`, `http:` included, and was on by + default. RFC 8058 §3.1 asks for an https URI; `oneClick` is opt-in and needs one. ## Repository layout diff --git a/docs/messages/compose.md b/docs/messages/compose.md index 652b3fa..b439868 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -241,6 +241,50 @@ described in the next section. They differ from the fields above in that a custom header is still honored when the dedicated field is left unset, so the header form keeps working. +## Mailing lists and one-click unsubscribe + +The `list` field writes the List-* header fields of [RFC 2369] and [RFC 2919], +which mail clients use to show an unsubscribe button, and, with `oneClick`, the +`List-Unsubscribe-Post` field of [RFC 8058] that Gmail and Yahoo ask of bulk +senders: + +```js +const message = createMessage({ + from: 'news@example.com', + to: 'reader@example.com', + subject: 'This week', + content: { html: '

…

' }, + list: { + id: { id: 'news.example.com', name: 'Weekly News' }, + unsubscribe: ['https://example.com/unsubscribe?u=42', 'unsubscribe@example.com'], + oneClick: true, + }, +}); +// List-ID: Weekly News +// List-Unsubscribe: , +// +// List-Unsubscribe-Post: List-Unsubscribe=One-Click +``` + +`help`, `unsubscribe`, `subscribe`, `post`, `owner` and `archive` each take one +URL or several, which go into one field: an `http:`, `https:` or `mailto:` URL, +or a bare address for a `mailto:` one. An entry may be `{ url, comment }`, with +a comment of printable ASCII. `id` is a dot-atom with a namespace, with an +optional `name`. A URL with whitespace, a control character or an angle bracket, +a URL of another scheme or without one, and an unknown key are refused with a +`TypeError`, never stripped or guessed. + +Set `oneClick: true` only when a POST to the https URL of `unsubscribe` +unsubscribes at once, without a page to confirm; it needs such a URL. RFC 8058 +also asks for a DKIM signature over both fields, which the default signature of +`./mime` and `./smtp` gives. The fields go into `Message.headers`, so every +transport that sends custom header fields sends them; a field given in `list` +may not also be given in `headers`. + +[RFC 2369]: https://www.rfc-editor.org/rfc/rfc2369 +[RFC 2919]: https://www.rfc-editor.org/rfc/rfc2919 +[RFC 8058]: https://www.rfc-editor.org/rfc/rfc8058 + ## Message identity and reply threading Applications that correlate replies, such as a helpdesk matching an incoming diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index 3774193..0df1662 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -753,8 +753,9 @@ Each signature in the `signatures` array can have the following options: | `headerFields` | `string[]` | see below | Headers to sign (h= tag) | By default the signature covers From, Reply-To, Subject, To, Cc, Date, -Message-ID, In-Reply-To, References, MIME-Version and Content-Type, present or -not, since a field absent when signed cannot be added afterwards without +Message-ID, In-Reply-To, References, MIME-Version, Content-Type, +List-Unsubscribe and List-Unsubscribe-Post (which one-click unsubscription +needs signed, RFC 8058 §4), present or not, since a field absent when signed cannot be added afterwards without breaking the signature; From, Reply-To and Subject are listed twice, so that a second instance cannot be added either (over-signing, RFC 6376 §8.15). A name listed more than once in `headerFields` signs the instances of that field from diff --git a/index.d.ts b/index.d.ts index b23e4c0..02b8e4e 100644 --- a/index.d.ts +++ b/index.d.ts @@ -594,6 +594,13 @@ export interface MessageConstructor { * @default `{}` */ readonly headers?: ImmutableHeaders | Record; + /** + * The List-* header fields of a mailing list or a newsletter (RFC 2369, + * RFC 2919), and one-click unsubscription (RFC 8058). They are added to + * {@link Message.headers}, so every transport that sends custom header + * fields sends them; a field given here may not also be in `headers`. + */ + readonly list?: ListFields; /** * An idempotency key to ensure that the same message is not sent multiple * times. This is useful for retrying failed send operations without @@ -646,6 +653,55 @@ export interface MessageConstructor { */ readonly calendar?: CalendarConstructor; } +/** + * A URL of a List-* field: an `http:`, `https:` or `mailto:` URL, or a bare + * address for a `mailto:` one, with an optional comment of printable ASCII. + */ +export type ListUrl = string | { readonly url: string; readonly comment?: string }; +/** + * The List-* header fields of a message. + * + * @example + * ```js + * createMessage({ + * // ... + * list: { + * id: { id: 'news.example.com', name: 'Weekly News' }, + * unsubscribe: ['https://example.com/unsubscribe?u=42', 'unsubscribe@example.com'], + * oneClick: true, + * }, + * }); + * ``` + */ +export interface ListFields { + /** + * `List-Id` (RFC 2919): a dot-atom with a namespace, such as + * `news.example.com`, with an optional name of printable ASCII. + */ + readonly id?: string | { readonly id: string; readonly name?: string }; + /** `List-Help`. */ + readonly help?: ListUrl | readonly ListUrl[]; + /** `List-Unsubscribe`: one field, its URLs separated by commas. */ + readonly unsubscribe?: ListUrl | readonly ListUrl[]; + /** `List-Subscribe`. */ + readonly subscribe?: ListUrl | readonly ListUrl[]; + /** `List-Post`. */ + readonly post?: ListUrl | readonly ListUrl[]; + /** `List-Owner`. */ + readonly owner?: ListUrl | readonly ListUrl[]; + /** `List-Archive`. */ + readonly archive?: ListUrl | readonly ListUrl[]; + /** + * Adds `List-Unsubscribe-Post: List-Unsubscribe=One-Click` (RFC 8058), + * which tells a mailbox provider that a POST to the https URL of + * `unsubscribe` unsubscribes at once, without a page to confirm. Gmail and + * Yahoo ask bulk senders for it. Set it only when that endpoint does so; + * it needs an https URL in `unsubscribe`. The DKIM signature of `./mime` + * and `./smtp` covers both fields by default. + * @default false + */ + readonly oneClick?: boolean; +} /** * Creates a new email {@link Message} based on the provided constructor * parameters. This function provides a more convenient API for creating diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 2bf6d94..9df7052 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -1,5 +1,5 @@ { - ".": 8.5, + ".": 9.5, "./mime": 7, "./smtp": 20, "./jmap": 12.5, diff --git a/src/core/list.js b/src/core/list.js new file mode 100644 index 0000000..da30439 --- /dev/null +++ b/src/core/list.js @@ -0,0 +1,181 @@ +// Ported from nodemailer v10.0.16 (src/mailer/mail-message.ts): the List-* +// header fields of RFC 2369 and RFC 2919, `_getListHeaders` and `_formatListUrl`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Also ported from unemail v0.7.0 (src/core/message.ts): List-Unsubscribe-Post, +// the one-click unsubscription of RFC 8058. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a URL, comment or list ID nodemailer would strip, guess a +// scheme for or encode is refused, the fields of one list go as one field each, +// and one-click needs an https URL. + +const { formatPhrase } = require('./address.js'); +const { isDotAtomText } = require('./text.js'); + +// The fields of RFC 2369 a list may carry URLs in, by the key that names them. +const URL_FIELDS = new Map([ + ['help', 'List-Help'], + ['unsubscribe', 'List-Unsubscribe'], + ['subscribe', 'List-Subscribe'], + ['post', 'List-Post'], + ['owner', 'List-Owner'], + ['archive', 'List-Archive'], +]); + +const URL_SCHEMES = new Set(['http:', 'https:', 'mailto:']); +const SCHEME = /^[a-z][a-z\d+.-]*:/i; +const BARE_ADDRESS = /^[^@]+@[^@]+$/; + +const throwTypeError = (message) => { + throw new TypeError(message); +}; + +// Printable ASCII only: a value beyond it would make the MIME writer encode +// the whole field, angle brackets and all. +const isPrintableAscii = (text) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code < 0x20 || code > 0x7e) return false; + } + return true; +}; + +// Whitespace, controls and angle brackets would end the bracketed URL, and +// nothing in a URL needs them unencoded. +const hasUrlBreak = (text) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code <= 0x20 || code === 0x7f || code === 0x3c || code === 0x3e) return true; + } + return false; +}; + +// A bare address is a mailto URL, as nodemailer reads one; anything else +// needs its scheme, which nodemailer would have guessed as http. +const listUrlOf = (field, text) => { + const invalid = () => throwTypeError(`Invalid ${field} URL: ${JSON.stringify(text)}`); + if (typeof text !== 'string' || text === '' || hasUrlBreak(text)) invalid(); + let candidate; + if (SCHEME.test(text)) candidate = text; + else if (BARE_ADDRESS.test(text)) candidate = `mailto:${text}`; + else invalid(); + let url; + try { + url = new URL(candidate); + } catch { + invalid(); + } + if (!URL_SCHEMES.has(url.protocol)) { + throwTypeError(`A ${field} URL must be http, https or mailto: ${JSON.stringify(text)}`); + } + return url.href; +}; + +// A comment as ctext, with the specials that would end or nest it escaped. +const formatComment = (field, comment) => { + if (typeof comment !== 'string' || !isPrintableAscii(comment)) { + throwTypeError(`A ${field} comment must be printable ASCII: ${JSON.stringify(comment)}`); + } + return comment.replace(/[()\\]/g, '\\$&'); +}; + +// The URLs of one field, comma-separated, and whether one of them is https. +const formatEntries = (field, entries) => { + let text = ''; + let hasHttps = false; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + const isObject = entry !== null && typeof entry === 'object'; + const href = listUrlOf(field, isObject ? entry.url : entry); + if (href.startsWith('https:')) hasHttps = true; + const comment = + isObject && entry.comment != null ? ` (${formatComment(field, entry.comment)})` : ''; + text += `${i === 0 ? '' : ', '}<${href}>${comment}`; + } + return { text, hasHttps }; +}; + +// RFC 2919: a list ID is a dot-atom with a namespace, `list.example.com`. +const formatListId = (entry) => { + const isObject = entry !== null && typeof entry === 'object'; + const id = isObject ? entry.id : entry; + const isId = + typeof id === 'string' && isPrintableAscii(id) && isDotAtomText(id) && id.includes('.'); + if (!isId) throwTypeError(`Invalid list ID: ${JSON.stringify(id)}`); + const name = isObject ? entry.name : undefined; + if (name == null) return `<${id}>`; + if (typeof name !== 'string' || name === '' || !isPrintableAscii(name)) { + throwTypeError(`A list name must be printable ASCII: ${JSON.stringify(name)}`); + } + return `${formatPhrase(name)} <${id}>`; +}; + +/** + * Turns the `list` of a message into its header fields: `List-Id`, the URL + * fields of RFC 2369 and, with `oneClick`, `List-Unsubscribe-Post` (RFC 8058). + * + * @param list The list fields, or `null`/`undefined` for none. + * @returns The fields, `{ name, value }` each, in the order of the keys. + * @throws {TypeError} If a key is unknown, a URL is not an http, https or + * mailto URL, a comment or name is not printable ASCII, the list ID is not a + * dot-atom with a namespace, or `oneClick` comes without an https URL to + * unsubscribe. + * @internal + */ +const listFieldsOf = (list) => { + const fields = []; + if (list == null) return fields; + if (typeof list !== 'object' || Array.isArray(list)) { + throwTypeError('The list fields must be an object.'); + } + let canUnsubscribeByPost = false; + for (const key in list) { + if (!Object.hasOwn(list, key)) continue; + const entry = list[key]; + if (entry == null || key === 'oneClick') continue; + if (key === 'id') { + fields.push({ name: 'List-Id', value: formatListId(entry) }); + continue; + } + const name = URL_FIELDS.get(key) ?? throwTypeError(`Unknown list field: ${key}`); + const entries = Array.isArray(entry) ? entry : [entry]; + if (entries.length === 0) continue; + const formatted = formatEntries(name, entries); + if (key === 'unsubscribe') canUnsubscribeByPost = formatted.hasHttps; + fields.push({ name, value: formatted.text }); + } + const { oneClick } = list; + if (oneClick != null && typeof oneClick !== 'boolean') { + throwTypeError('list.oneClick must be a boolean.'); + } + if (oneClick === true) { + if (!canUnsubscribeByPost) { + throwTypeError('One-click unsubscription needs an https URL in list.unsubscribe.'); + } + fields.push({ name: 'List-Unsubscribe-Post', value: 'List-Unsubscribe=One-Click' }); + } + return fields; +}; + +/** + * Adds the list fields of a message to its headers. + * + * @param headers The headers given with the message. + * @param list The list fields. + * @returns The same headers. + * @throws {TypeError} If a list field is invalid, or the headers already carry + * one of the fields the list sets. + * @internal + */ +const addListHeaders = (headers, list) => { + const fields = listFieldsOf(list); + for (let i = 0; i < fields.length; i++) { + const field = fields[i]; + if (headers.has(field.name)) { + throwTypeError(`${field.name} is given both in list and in headers.`); + } + headers.set(field.name, field.value); + } + return headers; +}; + +module.exports = { addListHeaders, listFieldsOf }; diff --git a/src/core/message.js b/src/core/message.js index 6058d93..e2d7bfd 100644 --- a/src/core/message.js +++ b/src/core/message.js @@ -6,6 +6,7 @@ const { parseAddress } = require('./address.js'); const { isAttachment } = require('./attachment.js'); const { resolveCalendarContent } = require('./calendar.js'); const { parseMessageId } = require('./message-id.js'); +const { addListHeaders } = require('./list.js'); /** * Creates a new email {@link Message} based on the provided constructor @@ -35,8 +36,9 @@ const { parseMessageId } = require('./message-id.js'); * object is invalid, when a message identifier is not * valid, when the date is not one RFC 5322 can express, or * when the calendar content is not a single well-formed - * iCalendar object declaring one supported method, or when - * `generateId` returns no dot-atom-text. + * iCalendar object declaring one supported method, when a + * field of `list` is invalid or also given in `headers`, + * or when `generateId` returns no dot-atom-text. */ const createMessage = (constructor, options) => { const sender = checkAddress('sender', constructor.from); @@ -51,7 +53,7 @@ const createMessage = (constructor, options) => { content: constructor.content, priority: constructor.priority ?? 'normal', tags: ensureArray(constructor.tags), - headers: new Headers(constructor.headers ?? {}), + headers: addListHeaders(new Headers(constructor.headers ?? {}), constructor.list), idempotencyKey: constructor.idempotencyKey, messageId: constructor.messageId == null ? undefined : checkMessageId('', constructor.messageId), diff --git a/src/mime/dkim/types.js b/src/mime/dkim/types.js index f099ff8..fa88eb3 100644 --- a/src/mime/dkim/types.js +++ b/src/mime/dkim/types.js @@ -79,7 +79,8 @@ const refuse = (what, value) => { /** * Default header fields to sign if not specified: those a reader trusts or a * client renders, present or not, since a field absent when signed cannot be - * added afterwards without breaking the signature. From, Reply-To and + * added afterwards without breaking the signature. The two of one-click + * unsubscription are among them, as RFC 8058 §4 asks. From, Reply-To and * Subject are listed once more than they can appear, so that a second * instance cannot be added either (over-signing, RFC 6376 §8.15). */ @@ -98,6 +99,8 @@ const DEFAULT_SIGNED_HEADERS = [ 'references', 'mime-version', 'content-type', + 'list-unsubscribe', + 'list-unsubscribe-post', ]; /** diff --git a/tests/types/index.test-d.ts b/tests/types/index.test-d.ts index ced8a21..cbf2017 100644 --- a/tests/types/index.test-d.ts +++ b/tests/types/index.test-d.ts @@ -34,6 +34,38 @@ expectError(createMessage({ from: 'alex@example.com', subject: 'Hello' })); expectError(createMessage({ from: 'alex', to: 'bob@example.com', subject: 'x', content: {} })); expectType
(parseAddress('Alex ')); +// The List-* fields of a newsletter. +createMessage({ + from: 'news@example.com', + to: 'reader@example.com', + subject: 'Weekly', + content: { text: 'Hi' }, + list: { + id: { id: 'news.example.com', name: 'Weekly News' }, + unsubscribe: ['https://example.com/u', { url: 'unsub@example.com', comment: 'mail' }], + help: 'https://example.com/help', + oneClick: true, + }, +}); +expectError( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, + list: { unsubscribe: 42 }, + }), +); +expectError( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, + list: { oneClick: 'yes' }, + }), +); + // A receipt narrows on `successful`. const receipt = {} as Receipt<'smtp'>; if (receipt.successful) { diff --git a/tests/unit/list.test.js b/tests/unit/list.test.js new file mode 100644 index 0000000..8f37706 --- /dev/null +++ b/tests/unit/list.test.js @@ -0,0 +1,196 @@ +// Not ported: tests of the List-* header fields of RFC 2369 and RFC 2919 and +// of one-click unsubscription (RFC 8058), ported from nodemailer and unemail. +const { test } = require('node:test'); +const assert = require('node:assert'); +const PostalMime = require('postal-mime'); +const { createMessage } = require('../../index.js'); +const { readAttachmentContent } = require('../../src/core/attachment.js'); +const { listFieldsOf } = require('../../src/core/list.js'); +const { composeMessage } = require('../../src/mime/index.js'); +const { convertMessage: convertSes } = require('../../src/ses/message-converter.js'); +const { createSesConfig } = require('../../src/ses/config.js'); +const { + TEST_DKIM_PRIVATE_KEY, + TEST_DKIM_PUBLIC_KEY, +} = require('../helpers/mime/dkim-test-keys.js'); +const { verifyDkim } = require('../helpers/mime/verify-dkim.js'); + +const fieldsOf = (list) => { + const fields = {}; + for (const field of listFieldsOf(list)) fields[field.name] = field.value; + return fields; +}; + +const message = (list, headers) => + createMessage({ + from: 'news@example.com', + to: 'reader@example.com', + subject: 'Weekly', + content: { text: 'Hello' }, + headers, + list, + }); + +test('List: URLs go in angle brackets, a bare address as mailto, one field each', () => { + assert.deepStrictEqual( + fieldsOf({ + help: 'https://example.com/help', + unsubscribe: ['https://example.com/u?id=1', 'unsub@example.com?subject=stop'], + subscribe: 'mailto:join@example.com', + post: 'list@example.com', + owner: { url: 'mailto:owner@example.com', comment: 'Ann (owner)' }, + archive: 'http://example.com/archive', + }), + { + 'List-Help': '', + 'List-Unsubscribe': ', ', + 'List-Subscribe': '', + 'List-Post': '', + 'List-Owner': ' (Ann \\(owner\\))', + 'List-Archive': '', + }, + ); +}); + +test('List: List-Id is a dot-atom with a namespace and an optional name', () => { + assert.deepStrictEqual(fieldsOf({ id: 'news.example.com' }), { + 'List-Id': '', + }); + assert.deepStrictEqual(fieldsOf({ id: { id: 'news.example.com', name: 'Weekly News' } }), { + 'List-Id': 'Weekly News ', + }); + assert.deepStrictEqual(fieldsOf({ id: { id: 'news.example.com', name: 'News, weekly' } }), { + 'List-Id': '"News, weekly" ', + }); + for (const id of ['news', 'news..example.com', 'news example.com', 'новини.example.com']) { + assert.throws(() => listFieldsOf({ id }), /Invalid list ID/, id); + } + assert.throws( + () => listFieldsOf({ id: { id: 'news.example.com', name: 'Новини' } }), + /list name must be printable ASCII/, + ); +}); + +test('List: one-click adds List-Unsubscribe-Post and needs an https URL', () => { + assert.deepStrictEqual( + fieldsOf({ unsubscribe: ['mailto:u@example.com', 'https://example.com/u'], oneClick: true }), + { + 'List-Unsubscribe': ', ', + 'List-Unsubscribe-Post': 'List-Unsubscribe=One-Click', + }, + ); + // unemail turned one-click on for any URL; RFC 8058 §3.1 asks for https. + assert.throws( + () => listFieldsOf({ unsubscribe: 'http://example.com/u', oneClick: true }), + /needs an https URL/, + ); + assert.throws(() => listFieldsOf({ oneClick: true }), /needs an https URL/); + assert.throws( + () => listFieldsOf({ unsubscribe: 'https://example.com/u', oneClick: 'yes' }), + /must be a boolean/, + ); + assert.deepStrictEqual(fieldsOf({ unsubscribe: 'https://example.com/u', oneClick: false }), { + 'List-Unsubscribe': '', + }); +}); + +test('List: a URL that could end the field or the brackets is refused', () => { + // unemail built these fields after its line-break check, so a CRLF in the + // URL added header fields; nodemailer stripped the characters silently. + const broken = [ + 'https://example.com/u\r\nBcc: victim@example.com', + 'https://example.com/u>, listFieldsOf({ unsubscribe: url }), /Invalid List-Unsubscribe URL/, url); + } + assert.throws(() => listFieldsOf({ help: 'javascript:alert(1)' }), /http, https or mailto/); + assert.throws(() => listFieldsOf({ help: 'ftp://example.com/help' }), /http, https or mailto/); + // nodemailer guessed http:// for a URL without a scheme. + assert.throws(() => listFieldsOf({ help: 'example.com/help' }), /Invalid List-Help URL/); + assert.throws(() => listFieldsOf({ help: { comment: 'no url' } }), /Invalid List-Help URL/); +}); + +test('List: a comment beyond printable ASCII, an unknown key or a non-object is refused', () => { + assert.throws( + () => listFieldsOf({ help: { url: 'https://example.com', comment: 'line\r\nbreak' } }), + /comment must be printable ASCII/, + ); + assert.throws( + () => listFieldsOf({ help: { url: 'https://example.com', comment: 'Довідка' } }), + /comment must be printable ASCII/, + ); + assert.throws(() => listFieldsOf({ unsubcribe: 'https://example.com' }), /Unknown list field/); + assert.throws(() => listFieldsOf('https://example.com'), /must be an object/); + assert.throws(() => listFieldsOf(['https://example.com']), /must be an object/); + assert.deepStrictEqual(listFieldsOf(undefined), []); + assert.deepStrictEqual(listFieldsOf({ help: [], unsubscribe: null }), []); +}); + +test('List: URLs are written as the URL parser serializes them', () => { + assert.deepStrictEqual(fieldsOf({ help: 'HTTPS://Bücher.example/Hilfe?ä=1' }), { + 'List-Help': '', + }); +}); + +test('List: createMessage puts the fields into the headers, once', () => { + const list = { unsubscribe: 'https://example.com/u', oneClick: true }; + const created = message(list); + assert.strictEqual(created.headers.get('List-Unsubscribe'), ''); + assert.strictEqual(created.headers.get('List-Unsubscribe-Post'), 'List-Unsubscribe=One-Click'); + assert.throws( + () => message(list, { 'List-Unsubscribe': '' }), + /List-Unsubscribe is given both in list and in headers/, + ); + // A field the list leaves alone may still come through the headers. + const custom = message({ id: 'news.example.com' }, { 'List-Unsubscribe': '' }); + assert.strictEqual(custom.headers.get('List-Unsubscribe'), ''); +}); + +test('List: the composed message carries the fields, and DKIM signs both of one-click', async () => { + const raw = await composeMessage( + message({ + id: { id: 'news.example.com', name: 'Weekly News' }, + unsubscribe: ['https://example.com/u?id=1', 'mailto:unsub@example.com'], + oneClick: true, + }), + { + dkim: { + signatures: [ + { signingDomain: 'example.com', selector: 'test', privateKey: TEST_DKIM_PRIVATE_KEY }, + ], + }, + }, + ); + const bytes = await readAttachmentContent(raw.content); + const wire = new TextDecoder().decode(bytes); + verifyDkim(wire, TEST_DKIM_PUBLIC_KEY); + const signature = /^DKIM-Signature:[\s\S]*?\r\n(?![ \t])/m.exec(wire)[0].replace(/\r\n\s/g, ''); + assert.match(signature, /h=[^;]*:list-unsubscribe:list-unsubscribe-post;/); + const parsed = await PostalMime.parse(bytes); + const headers = Object.create(null); + for (const header of parsed.headers) headers[header.key] = header.value; + assert.strictEqual( + headers['list-unsubscribe'], + ', ', + ); + assert.strictEqual(headers['list-unsubscribe-post'], 'List-Unsubscribe=One-Click'); + assert.strictEqual(headers['list-id'], 'Weekly News '); +}); + +test('List: a provider that takes custom headers gets the fields', async () => { + const config = createSesConfig({ + authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, + }); + const request = await convertSes( + message({ unsubscribe: 'https://example.com/u', oneClick: true }), + config, + ); + const headers = Object.create(null); + for (const header of request.Content.Simple.Headers) headers[header.Name] = header.Value; + assert.strictEqual(headers['list-unsubscribe'], ''); + assert.strictEqual(headers['list-unsubscribe-post'], 'List-Unsubscribe=One-Click'); +}); diff --git a/tests/unit/mime-dkim-sign.test.js b/tests/unit/mime-dkim-sign.test.js index 24912b0..e250e48 100644 --- a/tests/unit/mime-dkim-sign.test.js +++ b/tests/unit/mime-dkim-sign.test.js @@ -110,11 +110,12 @@ describe('DKIM Signing', () => { }); // Changed after the port: the default signs more fields, and over-signs - // From, Reply-To and Subject (upstream signed from, to, subject, date). + // From, Reply-To and Subject (upstream signed from, to, subject, date), + // and the two fields of one-click unsubscription (RFC 8058 §4). assert.ok( result.signature.includes( 'h=from:from:reply-to:reply-to:subject:subject:to:cc:date:message-id:in-reply-to:' + - 'references:mime-version:content-type;', + 'references:mime-version:content-type:list-unsubscribe:list-unsubscribe-post;', ), ); }); From 85086073b37cdbe155751f589e71de273dd2c14a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:13:20 +0300 Subject: [PATCH 005/143] feat(mime): choose 7bit, quoted-printable or base64 by content Every text and HTML body went quoted-printable, so Cyrillic or CJK text took about three times its UTF-8 bytes, plain ASCII was escaped for nothing, and a bare LF went as =0A on one long line. As nodemailer does, a body is first given CRLF line breaks, then written 7bit when it is ASCII with lines of at most 76 columns, quoted-printable when it has more Latin letters than other characters, and base64 otherwise. A line that ends in whitespace goes quoted-printable too, so an MTA that trims it cannot break a DKIM signature over the simple body. The budgets of ./mime and ./smtp go to 7.5 and 20.5 KB. Co-Authored-By: Claude Opus 5.5 --- docs/messages/mime.md | 10 +- scripts/size-budgets.json | 4 +- src/mime/message.js | 77 ++++++++--- tests/unit/mime-encoding.test.js | 123 ++++++++++++++++++ ...smtp-message-converter.integration.test.js | 78 ++++++----- tests/unit/smtp-mime-regression.test.js | 14 +- 6 files changed, 246 insertions(+), 60 deletions(-) create mode 100644 tests/unit/mime-encoding.test.js diff --git a/docs/messages/mime.md b/docs/messages/mime.md index 5a208b6..7f95cea 100644 --- a/docs/messages/mime.md +++ b/docs/messages/mime.md @@ -28,7 +28,15 @@ Typed identity and threading fields follow the same precedence as [structured messages](/messages/compose). The bytes use CRLF and include the final line ending. They contain neither SMTP -dot-stuffing nor the DATA terminator. Collecting them as shown above uses memory +dot-stuffing nor the DATA terminator. + +A text or HTML body is written with CRLF line breaks, a bare LF or CR included, +and in the transfer encoding its content needs, as nodemailer chooses it: +`7bit` for ASCII whose lines fit in 76 columns and do not end in whitespace, +`quoted-printable` for text with more Latin letters than other characters, and +`base64` otherwise, which keeps Cyrillic or CJK text at about 4/3 of its UTF-8 +bytes instead of three times. Attachments are Base64, and an iCalendar part is +Base64 so its CRLF structure survives. Collecting them as shown above uses memory proportional to the complete message. For large messages, consume the chunks: ```js diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 9df7052..daabffa 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -1,7 +1,7 @@ { ".": 9.5, - "./mime": 7, - "./smtp": 20, + "./mime": 7.5, + "./smtp": 20.5, "./jmap": 12.5, "./ses": 6.5, "./resend": 5.5, diff --git a/src/mime/message.js b/src/mime/message.js index 70db7f4..9b2dd2a 100644 --- a/src/mime/message.js +++ b/src/mime/message.js @@ -1,7 +1,8 @@ // Ported from upyo v0.7.0 (packages/mime/src/message.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee // Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the case of -// a custom field name, `_normalizeHeaderKey`. +// a custom field name, `_normalizeHeaderKey`, and the transfer encoding of a +// text body, `getTransferEncoding` and `_getTextEncoding`. // https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const { Buffer } = require('node:buffer'); @@ -344,19 +345,11 @@ const buildAlternatives = (message) => { const alternatives = []; if ('text' in message.content && message.content.text !== undefined) { - alternatives.push({ - contentType: 'text/plain; charset=utf-8', - encoding: 'quoted-printable', - body: encodeQuotedPrintable(message.content.text), - }); + alternatives.push(textAlternative('text/plain; charset=utf-8', message.content.text)); } if ('html' in message.content) { - alternatives.push({ - contentType: 'text/html; charset=utf-8', - encoding: 'quoted-printable', - body: encodeQuotedPrintable(message.content.html), - }); + alternatives.push(textAlternative('text/html; charset=utf-8', message.content.html)); } if (message.calendar != null) { @@ -375,16 +368,68 @@ const buildAlternatives = (message) => { } if (alternatives.length < 1) { - alternatives.push({ - contentType: 'text/plain; charset=utf-8', - encoding: 'quoted-printable', - body: '', - }); + alternatives.push(textAlternative('text/plain; charset=utf-8', '')); } return alternatives; }; +// A text body in the canonical form of RFC 2046 §4.1.1, every line ended by +// CRLF: a bare LF or CR is a line break the writer of the text meant, which +// quoted-printable would otherwise carry as an escaped =0A. +const BARE_LINE_BREAK = /\r(?!\n)|(? (BARE_LINE_BREAK.test(text) ? text.replace(/\r\n|\r|\n/g, '\r\n') : text); + +/** + * Chooses the transfer encoding of a text body, as nodemailer does: 7bit for + * ASCII text whose lines fit in 76 columns, otherwise quoted-printable when it + * has more Latin letters than other characters, and Base64 when it has not, + * which keeps Cyrillic or CJK text at a third more than its bytes instead of + * three times. A line that ends in whitespace goes quoted-printable too, as + * an MTA that trims it would break a DKIM signature over the simple body. + * + * @param text A body with CRLF line breaks. + * @returns `7bit`, `quoted-printable` or `base64`. + */ +const textEncodingOf = (text) => { + let latin = 0; + let other = 0; + let isPlain = true; + let column = 0; + let endsInSpace = false; + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code === 0x0d || code === 0x0a) { + if (endsInSpace) isPlain = false; + endsInSpace = false; + column = 0; + continue; + } + column++; + if (column > 76) isPlain = false; + if ((code < 0x20 && code !== 0x09) || code > 0x7f) { + other++; + isPlain = false; + } else if ((code | 0x20) >= 0x61 && (code | 0x20) <= 0x7a) { + latin++; + } + endsInSpace = code === 0x20 || code === 0x09; + } + if (endsInSpace) isPlain = false; + if (isPlain) return '7bit'; + return other < latin ? 'quoted-printable' : 'base64'; +}; + +const textAlternative = (contentType, text) => { + const body = toCrlf(text); + const encoding = textEncodingOf(body); + if (encoding === 'quoted-printable') { + return { contentType, encoding, body: encodeQuotedPrintable(body) }; + } + if (encoding === 'base64') return { contentType, encoding, body: encodeBase64Body(body) }; + return { contentType, encoding, body }; +}; + /** * Encodes a body as Base64, wrapped at the 76 columns RFC 2045 §6.8 allows. * diff --git a/tests/unit/mime-encoding.test.js b/tests/unit/mime-encoding.test.js new file mode 100644 index 0000000..539ed05 --- /dev/null +++ b/tests/unit/mime-encoding.test.js @@ -0,0 +1,123 @@ +// Not ported: the transfer encoding of a text body, chosen by its content as +// nodemailer chooses it (upyo wrote every body quoted-printable). Each body is +// read back with postal-mime, an independent parser. +const { test } = require('node:test'); +const assert = require('node:assert'); +const PostalMime = require('postal-mime'); +const { createMessage, readAttachmentContent } = require('../../index.js'); +const { composeMessage } = require('../../src/mime/index.js'); +const { + TEST_DKIM_PRIVATE_KEY, + TEST_DKIM_PUBLIC_KEY, +} = require('../helpers/mime/dkim-test-keys.js'); +const { verifyDkim } = require('../helpers/mime/verify-dkim.js'); + +const compose = async (content, options) => { + const message = createMessage({ + from: 'alex@example.com', + to: 'bob@example.com', + subject: 'Encoding', + content, + }); + const composed = await composeMessage(message, options); + const bytes = await readAttachmentContent(composed.content); + return { bytes, wire: new TextDecoder().decode(bytes) }; +}; + +// postal-mime gives the text with LF line breaks and the line break the +// writer ends a part with. +const lines = (text) => text.replace(/\r\n|\r|\n/g, '\n').replace(/\n+$/, ''); + +const encodingsOf = (wire) => { + const encodings = []; + for (const match of wire.matchAll(/^Content-Transfer-Encoding: (\S+)\r$/gm)) { + encodings.push(match[1]); + } + return encodings; +}; + +test('Encoding: short ASCII lines go 7bit, as written', async () => { + const text = 'Hello Bob,\r\n\r\nSee you at 10:00. A = B, 50% off.\r\n'; + const { bytes, wire } = await compose({ text }); + assert.deepStrictEqual(encodingsOf(wire), ['7bit']); + assert.ok(wire.endsWith(`\r\n\r\n${text}\r\n`)); + assert.strictEqual(lines((await PostalMime.parse(bytes)).text), lines(text)); +}); + +test('Encoding: a line past 76 columns, a control or trailing space is not 7bit', async () => { + for (const text of ['a'.repeat(77), 'tab\tand bell\u0007', 'trailing space \r\nnext', 'end\t']) { + const { bytes, wire } = await compose({ text }); + assert.deepStrictEqual(encodingsOf(wire), ['quoted-printable'], JSON.stringify(text)); + for (const line of wire.split('\r\n')) assert.ok(line.length <= 998); + assert.strictEqual(lines((await PostalMime.parse(bytes)).text), lines(text)); + } + // 76 columns still fit. + assert.deepStrictEqual(encodingsOf((await compose({ text: 'a'.repeat(76) })).wire), ['7bit']); +}); + +test('Encoding: mostly Latin text goes quoted-printable, the rest Base64', async () => { + const french = 'Le cœur a ses raisons que la raison ne connaît point.'; + const ukrainian = 'Серце має свої причини, яких розум не знає.'; + const japanese = '心には理性の知らない理由がある。'; + const cases = [ + [french, 'quoted-printable'], + [ukrainian, 'base64'], + [japanese, 'base64'], + ['Hi 👋🌍💬', 'base64'], + ]; + for (const [text, encoding] of cases) { + const { bytes, wire } = await compose({ text, html: `

${text}

` }); + assert.deepStrictEqual(encodingsOf(wire), [encoding, encoding], text); + const parsed = await PostalMime.parse(bytes); + assert.strictEqual(lines(parsed.text), text); + assert.strictEqual(lines(parsed.html), `

${text}

`); + } +}); + +test('Encoding: Cyrillic in Base64 is far shorter than it was in quoted-printable', async () => { + const text = 'Серце має свої причини, яких розум не знає. '.repeat(40); + const { wire } = await compose({ text }); + const body = wire.slice(wire.indexOf('\r\n\r\n') + 4); + const utf8Length = Buffer.byteLength(text); + // Base64 is 4/3 of the bytes plus line breaks; quoted-printable took about 3. + assert.ok(body.length < utf8Length * 1.4, `${body.length} for ${utf8Length} bytes`); +}); + +test('Encoding: a bare LF or CR becomes CRLF before encoding', async () => { + const { bytes, wire } = await compose({ text: 'one\ntwo\rthree\r\nfour' }); + assert.deepStrictEqual(encodingsOf(wire), ['7bit']); + assert.ok(wire.endsWith('\r\n\r\none\r\ntwo\r\nthree\r\nfour\r\n')); + assert.ok(!wire.includes('=0A')); + assert.strictEqual(lines((await PostalMime.parse(bytes)).text), 'one\ntwo\nthree\nfour'); + const latin = await compose({ text: 'café\nnext' }); + assert.ok(!latin.wire.includes('=0A')); + assert.ok(latin.wire.includes('caf=C3=A9\r\nnext')); +}); + +test('Encoding: an empty body goes 7bit', async () => { + assert.deepStrictEqual(encodingsOf((await compose({})).wire), ['7bit']); + assert.deepStrictEqual(encodingsOf((await compose({ text: '' })).wire), ['7bit']); +}); + +test('Encoding: DKIM verifies over each encoding and both canonicalizations', async () => { + for (const canonicalization of ['simple/simple', 'relaxed/relaxed']) { + for (const text of ['plain ascii', 'trailing space \r\nx', 'Серце має свої причини']) { + const { wire } = await compose( + { text }, + { + dkim: { + signatures: [ + { + signingDomain: 'example.com', + selector: 'test', + privateKey: TEST_DKIM_PRIVATE_KEY, + canonicalization, + }, + ], + }, + }, + ); + verifyDkim(wire, TEST_DKIM_PUBLIC_KEY); + } + } +}); diff --git a/tests/unit/smtp-message-converter.integration.test.js b/tests/unit/smtp-message-converter.integration.test.js index 0c43d9a..7d8ad34 100644 --- a/tests/unit/smtp-message-converter.integration.test.js +++ b/tests/unit/smtp-message-converter.integration.test.js @@ -8,6 +8,22 @@ const { describe, test } = require('node:test'); // a message), is a test helper now. const { convertMessage } = require('../helpers/smtp/convert-message.js'); +// Changed in the port: the transfer encoding of a text body is chosen by its +// content, as nodemailer chooses it, so the tests of a body read it decoded. +const encodingOf = (raw) => /Content-Transfer-Encoding: (\S+)/.exec(raw)[1]; +const bodyOf = (raw) => { + const body = raw.slice(raw.indexOf('\r\n\r\n') + 4); + const encoding = encodingOf(raw); + if (encoding === 'base64') { + return Buffer.from(body.replace(/\r\n/g, ''), 'base64').toString('utf8'); + } + if (encoding !== 'quoted-printable') return body; + const bytes = body + .replace(/=\r\n/g, '') + .replace(/=([0-9A-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))); + return Buffer.from(bytes, 'latin1').toString('utf8'); +}; + describe('Message Converter Integration Tests', () => { const createTestMessage = (overrides = {}) => ({ sender: { name: 'John Doe', address: 'john@example.com' }, @@ -117,7 +133,8 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); assert.ok(result.raw.includes('Content-Type: text/html; charset=utf-8')); - assert.ok(result.raw.includes('Content-Transfer-Encoding: quoted-printable')); + // Changed in the port: short ASCII goes 7bit. + assert.ok(result.raw.includes('Content-Transfer-Encoding: 7bit')); assert.ok(result.raw.includes('

Hello

World!

')); assert.ok(!result.raw.includes('multipart')); }); @@ -542,13 +559,16 @@ describe('Message Converter Integration Tests', () => { }); test('should handle special characters in quoted-printable', async () => { + // Changed in the port: 'End=' alone goes 7bit; a Latin text beyond + // ASCII goes quoted-printable. const message = createTestMessage({ - content: { text: 'End=' }, + content: { text: 'Café End=' }, }); const result = await convertMessage(message); // Should escape equals at end of line + assert.equal(encodingOf(result.raw), 'quoted-printable'); assert.ok(result.raw.includes('=3D')); // Escaped equals }); @@ -559,11 +579,9 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); - assert.ok(result.raw.includes('Content-Transfer-Encoding: quoted-printable')); - - // Should encode Korean characters as UTF-8 quoted-printable - // 안녕하세요! = EC=95=88=EB=85=95=ED=95=98=EC=84=B8=EC=9A=94! - assert.ok(result.raw.includes('=EC=95=88=EB=85=95=ED=95=98=EC=84=B8=EC=9A=94!')); + // Changed in the port: text with more Hangul than Latin goes Base64. + assert.ok(result.raw.includes('Content-Transfer-Encoding: base64')); + assert.equal(bodyOf(result.raw), '안녕하세요! 이것은 한국어 테스트입니다.'); }); test('should handle mixed ASCII and CJK characters', async () => { @@ -591,11 +609,11 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); - // Should contain soft line breaks for long lines - assert.ok(result.raw.includes('=\r\n')); - - // Should properly encode Korean characters - assert.ok(result.raw.includes('=EC=9D=B4=EA=B2=83=EC=9D=80')); // 이것은 + // Changed in the port: Base64, wrapped at 76 columns. + assert.equal(encodingOf(result.raw), 'base64'); + const lines = result.raw.slice(result.raw.indexOf('\r\n\r\n') + 4).split('\r\n'); + assert.ok(lines.every((line) => line.length <= 76)); + assert.equal(bodyOf(result.raw), longKoreanText); }); test('should handle Japanese and Chinese characters', async () => { @@ -605,15 +623,9 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); - // Should encode Japanese hiragana (こんにちは) - assert.ok(result.raw.includes('=E3=81=93=E3=82=93=E3=81=AB=E3=81=A1=E3=81=AF')); - - // Should encode Chinese characters (你好) - assert.ok(result.raw.includes('=E4=BD=A0=E5=A5=BD')); - - // Should encode Korean characters (안녕하세요) - may be split across lines due to line length - const rawWithoutLineBreaks = result.raw.replace(/=\r\n/g, ''); - assert.ok(rawWithoutLineBreaks.includes('=EC=95=88=EB=85=95=ED=95=98=EC=84=B8=EC=9A=94')); + // Changed in the port: Base64 rather than quoted-printable. + assert.equal(encodingOf(result.raw), 'base64'); + assert.equal(bodyOf(result.raw), 'こんにちは 你好 안녕하세요'); }); test('should handle emoji and special Unicode characters', async () => { @@ -623,10 +635,9 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); - // Should encode emoji properly - assert.ok(result.raw.includes('=F0=9F=91=8B')); // 👋 - assert.ok(result.raw.includes('=F0=9F=8C=8D')); // 🌍 - assert.ok(result.raw.includes('=F0=9F=92=AC')); // 💬 + // Changed in the port: Base64 rather than quoted-printable. + assert.equal(encodingOf(result.raw), 'base64'); + assert.equal(bodyOf(result.raw), 'Hello 👋 世界 🌍 한국어 💬'); }); test('should handle line breaks with CJK characters', async () => { @@ -636,14 +647,10 @@ describe('Message Converter Integration Tests', () => { const result = await convertMessage(message); - // Should preserve line breaks - assert.ok(result.raw.includes('\n')); - assert.ok(result.raw.includes('\r\n')); - - // Should encode Korean characters properly - assert.ok(result.raw.includes('=EC=B2=AB')); // 첫 - assert.ok(result.raw.includes('=EB=91=90')); // 두 - assert.ok(result.raw.includes('=EC=84=B8')); // 세 + // Changed in the port: the bare LF becomes CRLF, the canonical line + // break of a text body, and the text goes Base64. + assert.equal(encodingOf(result.raw), 'base64'); + assert.equal(bodyOf(result.raw), '첫 번째 줄\r\n두 번째 줄\r\n세 번째 줄'); }); }); @@ -798,9 +805,8 @@ describe('Message Converter Integration Tests', () => { assert.deepStrictEqual(occurrences(result.raw, 'Content-Type'), [ 'text/plain; charset=utf-8', ]); - assert.deepStrictEqual(occurrences(result.raw, 'Content-Transfer-Encoding'), [ - 'quoted-printable', - ]); + // Changed in the port: short ASCII goes 7bit. + assert.deepStrictEqual(occurrences(result.raw, 'Content-Transfer-Encoding'), ['7bit']); }); test('should keep the real Content-Type first in multipart messages', async () => { diff --git a/tests/unit/smtp-mime-regression.test.js b/tests/unit/smtp-mime-regression.test.js index c6d2069..ef9e6df 100644 --- a/tests/unit/smtp-mime-regression.test.js +++ b/tests/unit/smtp-mime-regression.test.js @@ -8,8 +8,10 @@ const { createMessage } = require('../../src/core/message.js'); // a message), is a test helper now. const { convertMessage } = require('../helpers/smtp/convert-message.js'); +// Changed in the port: the transfer encoding is chosen by content, so short +// ASCII lines go 7bit, and a bare LF is written as CRLF. for (const text of ['first\r\nsecond', 'a'.repeat(150), ('x'.repeat(60) + '\n').repeat(20)]) { - test(`quoted-printable physical lines: ${JSON.stringify(text.slice(0, 20))}`, async () => { + test(`text body physical lines: ${JSON.stringify(text.slice(0, 20))}`, async () => { const { raw } = await convertMessage( createMessage({ from: 'from@example.com', @@ -21,10 +23,12 @@ for (const text of ['first\r\nsecond', 'a'.repeat(150), ('x'.repeat(60) + '\n'). const body = raw.split('\r\n\r\n')[1]; assert.ok(!/(? line.length <= 76)); - const decoded = body - .replace(/=\r\n/g, '') - .replace(/=([0-9A-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))); - assert.equal(decoded, text); + const decoded = raw.includes('Content-Transfer-Encoding: quoted-printable') + ? body + .replace(/=\r\n/g, '') + .replace(/=([0-9A-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) + : body; + assert.equal(decoded, text.replace(/\r?\n/g, '\r\n')); }); } From 05681087f469c496010e569d8c9c4ed47f87d41b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:20:11 +0300 Subject: [PATCH 006/143] feat(core): scheduledAt, tracking and metadata on the message From unemail's message model: scheduledAt asks the provider to deliver later, tracking turns open and click tracking on or off for one message, and metadata holds values the provider returns in its events. createMessage checks them: a valid Date, booleans of `opens` and `clicks`, and metadata keys of 1 to 128 of [A-Za-z0-9_.-] with values free of control characters, since a key may become a header or form field name (unemail wrote both into X-Metadata-* fields unchecked, U3). A transport that cannot schedule a message refuses it before anything is sent, with a non-retryable validation receipt, rather than delivering it at once: SMTP, JMAP and, until their scheduling APIs are mapped, every HTTP provider. Resend sends a chunk holding one individually, and JMAP sends the runs between refused messages, so each receipt keeps its place. The budgets of the root, Mailtrap, Maileroo and Lettermint go up. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 + docs/messages/compose.md | 30 ++++ index.d.ts | 45 +++++ scripts/size-budgets.json | 8 +- src/core/message.js | 62 ++++++- src/internal/schedule.js | 31 ++++ src/jmap/jmap-transport.js | 34 +++- src/lettermint/message-converter.js | 2 + src/maileroo/message-converter.js | 2 + src/mailgun/message-converter.js | 2 + src/mailtrap/message-converter.js | 2 + src/plunk/message-converter.js | 2 + src/resend/message-converter.js | 2 + src/resend/resend-transport.js | 7 +- src/sendgrid/message-converter.js | 2 + src/ses/message-converter.js | 2 + src/smtp/message-converter.js | 2 + tests/types/index.test-d.ts | 24 +++ tests/unit/message-fields.test.js | 260 ++++++++++++++++++++++++++++ 19 files changed, 515 insertions(+), 7 deletions(-) create mode 100644 src/internal/schedule.js create mode 100644 tests/unit/message-fields.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 2836073..c924aa7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -474,6 +474,9 @@ fails on upstream's behavior): writes what `URL` serializes. - U2. unemail: one-click went with any `unsubscribe.url`, `http:` included, and was on by default. RFC 8058 §3.1 asks for an https URI; `oneClick` is opt-in and needs one. +- U3. unemail: the keys and values of `metadata` became `X-Metadata-` header fields in eight + drivers with no check, so a key or value with a line break added header fields. `createMessage` + holds a key to 1 to 128 of `[A-Za-z0-9_.-]` and a value to a string without control characters. ## Repository layout diff --git a/docs/messages/compose.md b/docs/messages/compose.md index b439868..61d975b 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -241,6 +241,36 @@ described in the next section. They differ from the fields above in that a custom header is still honored when the dedicated field is left unset, so the header form keeps working. +## Scheduling, tracking and metadata + +Three fields ask the provider for what only some providers do: + +```js +const message = createMessage({ + from: 'shop@example.com', + to: 'customer@example.com', + subject: 'Your order has shipped', + content: { html: '

…

' }, + scheduledAt: new Date('2026-10-18T09:00:00Z'), + tracking: { opens: false, clicks: true }, + metadata: { orderId: '42', tenant: 'acme' }, +}); +``` + +- `scheduledAt` is when the provider should deliver the message. A transport + that cannot schedule it, SMTP and JMAP among them, refuses the message with a + non-retryable `validation` receipt rather than sending it at once, so `./pool` + does not fail over with it and `./retry` does not retry it. +- `tracking` turns open and click tracking on or off for this message, over the + configuration of the transport, where the provider takes it per message. +- `metadata` holds values the provider keeps with the message and returns in its + events. A key is 1 to 128 letters, digits, `_`, `-` or `.`, since it may become + the name of a header or form field; a value is a string without control + characters. + +SMTP and JMAP ignore `tracking` and `metadata`, which have no meaning in the +message itself. Each provider's page says which of the three it takes. + ## Mailing lists and one-click unsubscribe The `list` field writes the List-* header fields of [RFC 2369] and [RFC 2919], diff --git a/index.d.ts b/index.d.ts index 02b8e4e..907830e 100644 --- a/index.d.ts +++ b/index.d.ts @@ -487,6 +487,31 @@ export interface Message { * necessarily the scheduling semantics; see the transport's documentation. */ readonly calendar?: CalendarContent; + /** + * When the provider should deliver the message. A transport that cannot + * schedule it refuses the message with a non-retryable `validation` + * receipt rather than sending it at once. + */ + readonly scheduledAt?: Date; + /** + * Open and click tracking for this message, over what the transport is + * configured with, where its provider takes it per message. + */ + readonly tracking?: MessageTracking; + /** + * Values the provider keeps with the message and returns in its events + * (custom arguments, variables), where its provider takes them. + */ + readonly metadata?: Readonly>; +} +/** + * Open and click tracking of a message. + */ +export interface MessageTracking { + /** Whether the provider tracks opens, with a pixel. */ + readonly opens?: boolean; + /** Whether the provider tracks clicks, by rewriting links. */ + readonly clicks?: boolean; } /** * Represents the content of an email message, which can be either HTML @@ -652,6 +677,26 @@ export interface MessageConstructor { * than overriding it. Line endings are normalized to CRLF. */ readonly calendar?: CalendarConstructor; + /** + * When the provider should deliver the message. A transport that cannot + * schedule it (SMTP, JMAP, and every provider without a scheduling API) + * refuses the message with a non-retryable `validation` receipt rather + * than sending it at once. + */ + readonly scheduledAt?: Date; + /** + * Open and click tracking for this message, over what the transport is + * configured with, where its provider takes it per message. SMTP and JMAP + * ignore it. + */ + readonly tracking?: MessageTracking; + /** + * Values the provider keeps with the message and returns in its events. + * A key is 1 to 128 letters, digits, `_`, `-` or `.`, as it may become the + * name of a header or form field; a value is a string without control + * characters. SMTP and JMAP ignore it. + */ + readonly metadata?: Readonly>; } /** * A URL of a List-* field: an `http:`, `https:` or `mailto:` URL, or a bare diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index daabffa..256c90a 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -1,5 +1,5 @@ { - ".": 9.5, + ".": 10, "./mime": 7.5, "./smtp": 20.5, "./jmap": 12.5, @@ -7,9 +7,9 @@ "./resend": 5.5, "./sendgrid": 5.5, "./mailgun": 5.5, - "./mailtrap": 6, - "./maileroo": 5.5, - "./lettermint": 5.5, + "./mailtrap": 6.5, + "./maileroo": 6, + "./lettermint": 6, "./plunk": 5.5, "./pool": 3, "./retry": 3, diff --git a/src/core/message.js b/src/core/message.js index e2d7bfd..5f276c8 100644 --- a/src/core/message.js +++ b/src/core/message.js @@ -1,5 +1,8 @@ // Ported from upyo v0.7.0 (packages/core/src/message.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Also ported from unemail v0.7.0 (src/core/message.ts): `scheduledAt`, +// `tracking` and `metadata`, checked here rather than by each provider. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind const { generateId } = require('./id.js'); const { parseAddress } = require('./address.js'); @@ -38,7 +41,8 @@ const { addListHeaders } = require('./list.js'); * when the calendar content is not a single well-formed * iCalendar object declaring one supported method, when a * field of `list` is invalid or also given in `headers`, - * or when `generateId` returns no dot-atom-text. + * when `scheduledAt`, `tracking` or `metadata` is + * invalid, or when `generateId` returns no dot-atom-text. */ const createMessage = (constructor, options) => { const sender = checkAddress('sender', constructor.from); @@ -62,6 +66,9 @@ const createMessage = (constructor, options) => { references: checkMessageIds('references ', constructor.references), calendar: constructor.calendar == null ? undefined : resolveCalendarContent(constructor.calendar), + scheduledAt: checkScheduledAt(constructor.scheduledAt), + tracking: checkTracking(constructor.tracking), + metadata: checkMetadata(constructor.metadata), }; }; @@ -217,6 +224,59 @@ const checkDate = (date) => { return new Date(time); }; +const checkScheduledAt = (date) => { + if (date == null) return undefined; + const time = date instanceof Date ? date.getTime() : Number.NaN; + if (Number.isNaN(time)) throwTypeError(`Invalid scheduledAt: ${JSON.stringify(date)}`); + return new Date(time); +}; + +const checkTracking = (tracking) => { + if (tracking == null) return undefined; + if (typeof tracking !== 'object' || Array.isArray(tracking)) { + throwTypeError('Invalid tracking: it must be an object of `opens` and `clicks`.'); + } + const checked = {}; + for (const key in tracking) { + if (!Object.hasOwn(tracking, key) || tracking[key] === undefined) continue; + if (key !== 'opens' && key !== 'clicks') throwTypeError(`Invalid tracking field: ${key}`); + if (typeof tracking[key] !== 'boolean') throwTypeError(`Invalid tracking.${key}: a boolean.`); + checked[key] = tracking[key]; + } + return Object.freeze(checked); +}; + +// A key may become the name of a header field (`X-Metadata-`) or of a +// form field (`v:`), so it holds no more than letters, digits, `_`, `-` +// and `.`; a value no control character. +const METADATA_KEY = /^[A-Za-z0-9_.-]{1,128}$/; + +const hasControl = (text) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code < 0x20 || code === 0x7f) return true; + } + return false; +}; + +const checkMetadata = (metadata) => { + if (metadata == null) return undefined; + if (typeof metadata !== 'object' || Array.isArray(metadata)) { + throwTypeError('Invalid metadata: it must be an object of strings.'); + } + const checked = Object.create(null); + for (const key in metadata) { + if (!Object.hasOwn(metadata, key)) continue; + if (!METADATA_KEY.test(key)) throwTypeError(`Invalid metadata key: ${JSON.stringify(key)}`); + const value = metadata[key]; + if (typeof value !== 'string' || hasControl(value)) { + throwTypeError(`Invalid metadata value of ${key}: a string without control characters.`); + } + checked[key] = value; + } + return Object.freeze(checked); +}; + const ensureArray = (value) => { if (Array.isArray(value)) return value; return value == null ? [] : [value]; diff --git a/src/internal/schedule.js b/src/internal/schedule.js new file mode 100644 index 0000000..24009ae --- /dev/null +++ b/src/internal/schedule.js @@ -0,0 +1,31 @@ +// Not ported: the refusal of a message a transport would have to send at once +// although it asks for later. unemail refuses an option its driver does not +// take rather than dropping it (src/core/define.ts); a message scheduled for +// tomorrow and sent now is the worse of the two failures. + +/** + * The reason a transport gives for a message it cannot schedule. "Invalid" + * makes the receipt a non-retryable validation failure, so `./pool` does not + * fail over with it and `./retry` does not retry it. + * + * @param provider The transport that refuses it. + * @returns The error message. + * @internal + */ +const schedulingRefusal = (provider) => + `Invalid scheduledAt: ${provider} cannot schedule a message, so it is refused rather than sent at once.`; + +/** + * Refuses a message that asks for a later delivery its transport cannot + * schedule, before anything is sent. + * + * @param message The message. + * @param provider The transport that would send it. + * @throws {TypeError} If the message carries `scheduledAt`. + * @internal + */ +const refuseScheduledAt = (message, provider) => { + if (message.scheduledAt != null) throw new TypeError(schedulingRefusal(provider)); +}; + +module.exports = { refuseScheduledAt, schedulingRefusal }; diff --git a/src/jmap/jmap-transport.js b/src/jmap/jmap-transport.js index 68efcfe..98ec824 100644 --- a/src/jmap/jmap-transport.js +++ b/src/jmap/jmap-transport.js @@ -18,6 +18,7 @@ const { convertMessage } = require('./message-converter.js'); const { findMailAccount } = require('./session.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { chunks, readItems } = require('../internal/batch.js'); +const { schedulingRefusal } = require('../internal/schedule.js'); /** * JMAP capabilities required for email sending. @@ -171,6 +172,7 @@ class JmapTransport { async send(message, options) { const signal = options?.signal; signal?.throwIfAborted(); + if (message.scheduledAt != null) return refuseScheduled(); try { // Get or refresh session @@ -372,10 +374,31 @@ class JmapTransport { // a batch over it would be refused as a whole. const limit = getMaxObjectsInSet(discovery.discovered.session); for await (const batch of chunks(prepend(first.value, input), limit)) { - yield* this.sendBatch(discovery.discovered, batch, signal); + yield* this.sendRuns(discovery.discovered, batch, signal); } } + /** + * Sends the messages of a batch that can be sent, in runs between those + * it refuses, so that every receipt keeps the place of its message. + * + * @param discovered What the batches share. + * @param batch The messages, at most the limit of one Email/set. + * @param signal Cancellation signal. + * @returns Async iterable of receipts, one per message, in order. + */ + async *sendRuns(discovered, batch, signal) { + let start = 0; + for (let i = 0; i < batch.length; i++) { + if (batch[i].scheduledAt == null) continue; + if (i > start) yield* this.sendBatch(discovered, batch.slice(start, i), signal); + yield refuseScheduled(); + start = i + 1; + } + if (start === 0) yield* this.sendBatch(discovered, batch, signal); + else if (start < batch.length) yield* this.sendBatch(discovered, batch.slice(start), signal); + } + /** * Discovers what every batch of a sendMany() shares: the session, the * account, its drafts mailbox and its identities. @@ -959,6 +982,15 @@ const executeSubmission = async (client, apiUrl, request, signal) => { } }; +// JMAP holds a submission only with the FUTURERELEASE extension, which +// this transport does not ask for. +const refuseScheduled = () => + createJmapFailure(schedulingRefusal('jmap'), undefined, { + category: 'validation', + code: 'jmap.scheduling_unsupported', + retryable: false, + }); + const createJmapFailure = (message, error, override = {}) => { const attempts = getAttemptCount(error); diff --git a/src/lettermint/message-converter.js b/src/lettermint/message-converter.js index 23ae256..66ca64a 100644 --- a/src/lettermint/message-converter.js +++ b/src/lettermint/message-converter.js @@ -5,6 +5,7 @@ const { generateId } = require('../core/id.js'); const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -39,6 +40,7 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * @throws {RangeError} If the message has more than one tag. */ const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'lettermint'); if (message.tags.length > 1) { throw new RangeError('Lettermint supports at most one tag per message.'); } diff --git a/src/maileroo/message-converter.js b/src/maileroo/message-converter.js index bdf1516..61c8d25 100644 --- a/src/maileroo/message-converter.js +++ b/src/maileroo/message-converter.js @@ -5,6 +5,7 @@ const { generateId } = require('../core/id.js'); const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -39,6 +40,7 @@ const STANDARD_HEADERS = new Set([ */ const convertMessage = async (message, config, signal) => { signal?.throwIfAborted(); + refuseScheduledAt(message, 'maileroo'); const attachments = withCalendarAttachment(message); diff --git a/src/mailgun/message-converter.js b/src/mailgun/message-converter.js index 425adb1..60794a0 100644 --- a/src/mailgun/message-converter.js +++ b/src/mailgun/message-converter.js @@ -4,6 +4,7 @@ const { formatQuotedMailbox: formatAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { visitCustomHeaders, withCalendarAttachment } = require('../internal/provider.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const PRIORITIES = { high: '1', normal: '3', low: '5' }; @@ -25,6 +26,7 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * ``` */ const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'mailgun'); const formData = new FormData(); const attachments = withCalendarAttachment(message); diff --git a/src/mailtrap/message-converter.js b/src/mailtrap/message-converter.js index cda6bd3..28a5f85 100644 --- a/src/mailtrap/message-converter.js +++ b/src/mailtrap/message-converter.js @@ -4,6 +4,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -41,6 +42,7 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; */ const convertMessage = async (message, config, signal) => { signal?.throwIfAborted(); + refuseScheduledAt(message, 'mailtrap'); const attachments = withCalendarAttachment(message); diff --git a/src/plunk/message-converter.js b/src/plunk/message-converter.js index 2783b2d..00b728f 100644 --- a/src/plunk/message-converter.js +++ b/src/plunk/message-converter.js @@ -5,6 +5,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { collectCustomHeaders, withCalendarAttachment } = require('../internal/provider.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); /** * Converts a Upyo message to Plunk API format. @@ -33,6 +34,7 @@ const addressesOf = (message) => { }; const convertMessage = async (message, _config, signal) => { + refuseScheduledAt(message, 'plunk'); // Convert recipients - Plunk accepts either string or array const to = message.recipients.length === 1 diff --git a/src/resend/message-converter.js b/src/resend/message-converter.js index 8675459..9c29a55 100644 --- a/src/resend/message-converter.js +++ b/src/resend/message-converter.js @@ -5,6 +5,7 @@ const { generateId } = require('../core/id.js'); const { formatMailbox: formatAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -37,6 +38,7 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * ``` */ const convertMessage = async (message, _config, options = {}) => { + refuseScheduledAt(message, 'resend'); const attachments = withCalendarAttachment(message); const emailData = { diff --git a/src/resend/resend-transport.js b/src/resend/resend-transport.js index 07fcc13..90035cc 100644 --- a/src/resend/resend-transport.js +++ b/src/resend/resend-transport.js @@ -305,7 +305,12 @@ class ResendTransport { // API forbids, and `sendBatch()` fails a whole chunk on one conversion // error. Sending these individually keeps the failure — or the success // — confined to the message it belongs to. - if (message.attachments.length > 0 || message.tags.length > 0 || message.calendar != null) { + if ( + message.attachments.length > 0 || + message.tags.length > 0 || + message.calendar != null || + message.scheduledAt != null + ) { return false; } } diff --git a/src/sendgrid/message-converter.js b/src/sendgrid/message-converter.js index c30be7e..9308f71 100644 --- a/src/sendgrid/message-converter.js +++ b/src/sendgrid/message-converter.js @@ -4,6 +4,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { collectCustomHeaders, convertAttachments, @@ -29,6 +30,7 @@ const { * ``` */ const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'sendgrid'); const attachments = withCalendarAttachment(message); const personalization = { to: formatAddresses(message.recipients, formatAddress), diff --git a/src/ses/message-converter.js b/src/ses/message-converter.js index 271757e..1547d0c 100644 --- a/src/ses/message-converter.js +++ b/src/ses/message-converter.js @@ -5,6 +5,7 @@ const { Buffer } = require('node:buffer'); const { formatQuotedMailbox: formatAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -32,6 +33,7 @@ const MAX_SES_HEADERS = 15; const MAX_SES_HEADER_LENGTH = 996; const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'ses'); const destination = {}; if (message.recipients.length > 0) { diff --git a/src/smtp/message-converter.js b/src/smtp/message-converter.js index f50b289..98341d1 100644 --- a/src/smtp/message-converter.js +++ b/src/smtp/message-converter.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { prepareMimeMessage } = require('../mime/message.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); const { hasNonAsciiAddress, resolveSmtpEnvelope } = require('./envelope.js'); /** Snapshots MIME metadata and preserves the effective SMTP envelope. */ @@ -12,6 +13,7 @@ const prepareMessage = ( resolvedEnvelope = resolveSmtpEnvelope(message), options = {}, ) => { + refuseScheduledAt(message, 'smtp'); const plan = prepareMimeMessage(message, dkimConfig, options); const envelope = { ...resolvedEnvelope, dsn }; return { diff --git a/tests/types/index.test-d.ts b/tests/types/index.test-d.ts index cbf2017..be12971 100644 --- a/tests/types/index.test-d.ts +++ b/tests/types/index.test-d.ts @@ -11,6 +11,7 @@ import { type Address, type IdGenerator, type IdKind, + type MessageTracking, type Message, type RawTransport, type Receipt, @@ -34,6 +35,29 @@ expectError(createMessage({ from: 'alex@example.com', subject: 'Hello' })); expectError(createMessage({ from: 'alex', to: 'bob@example.com', subject: 'x', content: {} })); expectType
(parseAddress('Alex ')); +// Scheduling, tracking and metadata. +const fields = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, + scheduledAt: new Date(), + tracking: { opens: true }, + metadata: { orderId: '42' }, +}); +expectType(fields.scheduledAt); +expectType(fields.tracking); +expectType> | undefined>(fields.metadata); +expectError( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, + metadata: { orderId: 42 }, + }), +); + // The List-* fields of a newsletter. createMessage({ from: 'news@example.com', diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js new file mode 100644 index 0000000..d277967 --- /dev/null +++ b/tests/unit/message-fields.test.js @@ -0,0 +1,260 @@ +// Not ported: `scheduledAt`, `tracking` and `metadata` of a message (from +// unemail), checked by createMessage, and the refusal of a scheduled message by +// every transport that cannot schedule it, before anything is sent. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { SmtpTransport } = require('../../src/smtp/index.js'); +const { SesTransport } = require('../../src/ses/index.js'); +const { SendGridTransport } = require('../../src/sendgrid/index.js'); +const { MailgunTransport } = require('../../src/mailgun/index.js'); +const { ResendTransport } = require('../../src/resend/index.js'); +const { MailtrapTransport } = require('../../src/mailtrap/index.js'); +const { MailerooTransport } = require('../../src/maileroo/index.js'); +const { LettermintTransport } = require('../../src/lettermint/index.js'); +const { PlunkTransport } = require('../../src/plunk/index.js'); +const { JmapTransport } = require('../../src/jmap/index.js'); +const { PoolTransport } = require('../../src/pool/index.js'); +const { RetryTransport } = require('../../src/retry/index.js'); +const { MockTransport } = require('../../src/mock/index.js'); +const { MockSmtpServer } = require('../helpers/smtp/mock-smtp-server.js'); + +const base = { + from: 'alex@example.com', + to: 'bob@example.com', + subject: 'Hello', + content: { text: 'Hi' }, +}; +const tomorrow = new Date('2026-10-18T09:00:00Z'); +const scheduled = () => createMessage({ ...base, scheduledAt: tomorrow }); + +test('Fields: createMessage keeps scheduledAt, tracking and metadata, checked', () => { + const message = createMessage({ + ...base, + scheduledAt: tomorrow, + tracking: { opens: false, clicks: true }, + metadata: { orderId: '42', 'tenant.id': 'acme', step_2: '' }, + }); + assert.deepStrictEqual(message.scheduledAt, tomorrow); + assert.notStrictEqual(message.scheduledAt, tomorrow); + assert.deepStrictEqual(message.tracking, { opens: false, clicks: true }); + assert.ok(Object.isFrozen(message.tracking)); + assert.deepStrictEqual( + { ...message.metadata }, + { orderId: '42', 'tenant.id': 'acme', step_2: '' }, + ); + assert.strictEqual(Object.getPrototypeOf(message.metadata), null); + assert.ok(Object.isFrozen(message.metadata)); + const plain = createMessage(base); + assert.strictEqual(plain.scheduledAt, undefined); + assert.strictEqual(plain.tracking, undefined); + assert.strictEqual(plain.metadata, undefined); + assert.deepStrictEqual(createMessage({ ...base, tracking: { opens: undefined } }).tracking, {}); +}); + +test('Fields: an invalid scheduledAt, tracking or metadata is refused', () => { + const refused = [ + [{ scheduledAt: new Date('nope') }, /Invalid scheduledAt/], + [{ scheduledAt: '2026-10-18T09:00:00Z' }, /Invalid scheduledAt/], + [{ tracking: true }, /Invalid tracking: it must be an object/], + [{ tracking: ['opens'] }, /Invalid tracking: it must be an object/], + [{ tracking: { opened: true } }, /Invalid tracking field: opened/], + [{ tracking: { opens: 'yes' } }, /Invalid tracking.opens/], + [{ metadata: 'a=b' }, /Invalid metadata: it must be an object/], + [{ metadata: { 'bad key': 'x' } }, /Invalid metadata key/], + [{ metadata: { 'Bcc:\r\nx': 'x' } }, /Invalid metadata key/], + [{ metadata: { ключ: 'x' } }, /Invalid metadata key/], + [{ metadata: { ['k'.repeat(129)]: 'x' } }, /Invalid metadata key/], + [{ metadata: { key: 'line\r\nbreak' } }, /Invalid metadata value of key/], + [{ metadata: { key: 42 } }, /Invalid metadata value of key/], + ]; + for (const [fields, error] of refused) { + assert.throws(() => createMessage({ ...base, ...fields }), error, JSON.stringify(fields)); + } +}); + +const assertRefused = (receipt) => { + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(receipt.errors[0].category, 'validation'); + assert.match(receipt.errorMessages[0], /Invalid scheduledAt: \w+ cannot schedule a message/); +}; + +test('Fields: SMTP refuses a scheduled message before it connects', async () => { + const server = new MockSmtpServer(); + const port = await server.start(); + try { + const transport = new SmtpTransport({ host: 'localhost', port, pool: false }); + assertRefused(await transport.send(scheduled())); + const receipts = []; + for await (const receipt of transport.sendMany([scheduled(), createMessage(base)])) { + receipts.push(receipt); + } + assertRefused(receipts[0]); + assert.strictEqual(receipts[1].successful, true); + assert.strictEqual(server.getReceivedMessages().length, 1); + } finally { + await server.stop(); + } +}); + +const providers = { + ses: () => + new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, + retries: 0, + }), + sendgrid: () => new SendGridTransport({ apiKey: 'SG.key', retries: 0 }), + mailgun: () => new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com', retries: 0 }), + resend: () => new ResendTransport({ apiKey: 'key', retries: 0 }), + mailtrap: () => new MailtrapTransport({ apiToken: 'token', retries: 0 }), + maileroo: () => new MailerooTransport({ apiKey: 'key', retries: 0 }), + lettermint: () => new LettermintTransport({ apiToken: 'token', retries: 0 }), + plunk: () => new PlunkTransport({ apiKey: 'key', retries: 0 }), +}; + +for (const [name, make] of Object.entries(providers)) { + test(`Fields: ${name} refuses a scheduled message without a request`, async (t) => { + const fetch = t.mock.method(globalThis, 'fetch', async () => { + throw new Error('no request expected'); + }); + const transport = make(); + assertRefused(await transport.send(scheduled())); + const receipts = []; + for await (const receipt of transport.sendMany([scheduled(), scheduled()])) { + receipts.push(receipt); + } + assert.strictEqual(receipts.length, 2); + for (const receipt of receipts) assertRefused(receipt); + assert.strictEqual(fetch.mock.callCount(), 0); + }); +} + +test('Fields: Resend sends the others of a chunk in one batch request', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + bodies.push({ url: String(input), body: JSON.parse(init.body) }); + if (String(input).endsWith('/emails/batch')) { + return Response.json({ data: [{ id: 'r1' }, { id: 'r2' }] }); + } + return Response.json({ id: 'single' }); + }); + const transport = providers.resend(); + const receipts = []; + for await (const receipt of transport.sendMany([ + createMessage(base), + scheduled(), + createMessage(base), + ])) { + receipts.push(receipt); + } + // A scheduled message keeps the chunk off the batch API, so each goes alone. + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assertRefused(receipts[1]); + assert.strictEqual(bodies.length, 2); +}); + +const CORE = 'urn:ietf:params:jmap:core'; +const MAIL = 'urn:ietf:params:jmap:mail'; +const SUBMISSION = 'urn:ietf:params:jmap:submission'; + +test('Fields: JMAP refuses a scheduled message, and keeps the order of a batch', async (t) => { + const calls = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + const url = String(input); + if (url.endsWith('/.well-known/jmap')) { + return Response.json({ + capabilities: { [CORE]: {}, [MAIL]: {}, [SUBMISSION]: {} }, + accounts: { + a1: { + name: 'A', + isPersonal: true, + isReadOnly: false, + accountCapabilities: { [MAIL]: {}, [SUBMISSION]: {} }, + }, + }, + primaryAccounts: { [MAIL]: 'a1' }, + username: 'alex@example.com', + apiUrl: 'https://jmap.example.com/api', + downloadUrl: 'https://jmap.example.com/download/{accountId}/{blobId}/{name}', + uploadUrl: 'https://jmap.example.com/upload/{accountId}/', + eventSourceUrl: 'https://jmap.example.com/events', + state: '1', + }); + } + const methodResponses = []; + for (const [name, args, id] of JSON.parse(init.body).methodCalls) { + calls.push(name); + const created = {}; + for (const key in args.create ?? {}) created[key] = { id: key, blobId: 'b', threadId: 't' }; + const results = { + 'Mailbox/get': { list: [{ id: 'drafts', role: 'drafts', name: 'Drafts' }] }, + 'Identity/get': { list: [{ id: 'i1', email: 'alex@example.com', name: 'Alex' }] }, + 'Email/set': { created }, + 'EmailSubmission/set': { created }, + }; + methodResponses.push([name, results[name], id]); + } + return Response.json({ methodResponses }); + }); + const transport = new JmapTransport({ + sessionUrl: 'https://jmap.example.com/.well-known/jmap', + bearerToken: 'token', + retries: 0, + }); + assertRefused(await transport.send(scheduled())); + assert.deepStrictEqual(calls, []); + const receipts = []; + const input = [createMessage(base), scheduled(), createMessage(base), createMessage(base)]; + for await (const receipt of transport.sendMany(input)) receipts.push(receipt); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true, true], + ); + assertRefused(receipts[1]); + // The runs on either side of the refused message go as two Email/set calls. + assert.strictEqual(calls.filter((call) => call === 'Email/set').length, 2); + const allRefused = []; + for await (const receipt of transport.sendMany([scheduled(), scheduled()])) { + allRefused.push(receipt); + } + assert.strictEqual(allRefused.length, 2); + for (const receipt of allRefused) assertRefused(receipt); +}); + +test('Fields: the pool does not fail over with a refusal, and retry does not retry it', async (t) => { + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('no request expected'); + }); + const mock = new MockTransport(); + const pool = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: providers.ses(), priority: 2 }, + { transport: mock, priority: 1 }, + ], + }); + assertRefused(await pool.send(scheduled())); + assert.strictEqual(mock.getSentMessagesCount(), 0); + let attempts = 0; + const counted = { + send: async (message, options) => { + attempts++; + return providers.ses().send(message, options); + }, + async *sendMany() {}, + }; + const retry = new RetryTransport(counted, { maxAttempts: 3, initialDelay: 1 }); + assertRefused(await retry.send(scheduled())); + assert.strictEqual(attempts, 1); +}); + +test('Fields: the mock transport records a scheduled message as it is', async () => { + const mock = new MockTransport(); + const receipt = await mock.send(scheduled()); + assert.strictEqual(receipt.successful, true); + assert.deepStrictEqual(mock.getSentMessages()[0].scheduledAt, tomorrow); +}); From 4ba78ca4b4400a546fa3afb6b6a1a21c97cbed15 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:27:12 +0300 Subject: [PATCH 007/143] feat(attachments): attachments from a path, a URL, a data URI or a stream A new subpath makes an attachment of what nodemailer read through `path`, `href` and data URIs and sently through `resolveAttachments`: fromPath reads a regular file again each time the message is read, fromUrl fetches an http or https URL once through the injected fetch, fromDataUri decodes RFC 2397, fromStream reads a one-shot stream into memory, and inlineDataUrls turns the data-URI images of HTML into inline attachments (nodemailer's attachDataUrls). File names and types come from the path, the server or the URI, with a table of about sixty extensions. Fixed in the port: nodemailer read any path or URL an attachment named, its client following five redirects to any address with no bound on the body (N2), and turned a data URL over 50 MB into an empty attachment or read one that did not parse as a path (N3); sently's basePath compared paths as written, so a symbolic link led out of it (S1). Here nothing is read but through these helpers; fromUrl checks each redirect with allowUrl, by default refusing localhost and loopback, private and link-local addresses, sends headers to the first origin only and bounds time and bytes; fromPath compares realpaths. The budgets of Resend and SendGrid go to 6 KB with readBytes in the shared body reader. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 13 + docs/api/exports.md | 1 + docs/messages/attachments.md | 60 +++++ package.json | 4 + scripts/size-budgets.json | 5 +- src/attachments/attachment.js | 81 ++++++ src/attachments/data-uri.js | 131 ++++++++++ src/attachments/index.d.ts | 159 ++++++++++++ src/attachments/index.js | 11 + src/attachments/mime-types.js | 109 ++++++++ src/attachments/path.js | 67 +++++ src/attachments/stream.js | 45 ++++ src/attachments/url.js | 172 +++++++++++++ src/internal/body.js | 47 +++- tests/types/attachments.test-d.ts | 42 +++ tests/unit/attachments.test.js | 410 ++++++++++++++++++++++++++++++ 16 files changed, 1354 insertions(+), 3 deletions(-) create mode 100644 src/attachments/attachment.js create mode 100644 src/attachments/data-uri.js create mode 100644 src/attachments/index.d.ts create mode 100644 src/attachments/index.js create mode 100644 src/attachments/mime-types.js create mode 100644 src/attachments/path.js create mode 100644 src/attachments/stream.js create mode 100644 src/attachments/url.js create mode 100644 tests/types/attachments.test-d.ts create mode 100644 tests/unit/attachments.test.js diff --git a/CLAUDE.md b/CLAUDE.md index c924aa7..81a2e74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -468,6 +468,19 @@ fails on upstream's behavior): `List-Unsubscribepost`), and the default DKIM `h=` of neither nodemailer nor unemail covered it or `List-Unsubscribe`, as RFC 8058 §4 requires. `oneClick` of `list` writes it, and the default `h=` of `./mime` ends with `list-unsubscribe:list-unsubscribe-post`. +- N2. nodemailer: any `path` or `href` of an attachment (and of `html` and `text`) was read, a + file from disk or a URL through its own HTTP client, unless `disableFileAccess` and + `disableUrlAccess` were set; the client followed five redirects to any address, cloud metadata + included, with no bound on the body. In `./attachments` nothing is read but through `fromPath` + and `fromUrl`, which the application calls; `fromUrl` goes through the injected `fetch` with + `redirect: 'manual'`, checks each URL with `allowUrl` (by default `isPublicUrl`: no + `localhost`, loopback, private or link-local address), sends `headers` to the first origin + only, and bounds time and bytes. +- N3. nodemailer: a data URL over 50 MB became an empty attachment, and one that did not parse + was read as a file path. `fromDataUri` refuses both and checks base64, and percent-encoded + bytes stay bytes rather than going through `decodeURIComponent`. +- S1. sently (`resolveAttachments`): `basePath` compared the paths as written, so a symbolic link + inside it led out of it. `fromPath` compares the `realpath` of the file with that of `root`. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/api/exports.md b/docs/api/exports.md index 0686cd5..1c7acd4 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -42,6 +42,7 @@ The message model and the transport contract, ported from `@upyo/core`. | Subpath | Exports | Page | | --- | --- | --- | | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | +| `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails` | [SMTP](/transports/smtp) | | `@alexify/mailer/ses` | `SesTransport` | [Amazon SES](/transports/ses) | | `@alexify/mailer/resend` | `ResendTransport` | [Resend](/transports/resend) | diff --git a/docs/messages/attachments.md b/docs/messages/attachments.md index b833c88..4320b95 100644 --- a/docs/messages/attachments.md +++ b/docs/messages/attachments.md @@ -36,6 +36,66 @@ regular (non-inline) attachment. The `readFile()` call in this example allocates the complete file before constructing the message; use a content factory below when the file should be read incrementally. +## Files, URLs, data URIs and streams + +*@alexify/mailer/attachments* makes an attachment of a path, a URL, a data URI or a +stream, with its file name and type found for you. Nothing reads a file or a URL a +message merely names: these helpers are the only way in, and you call them. + +```js +const { createMessage } = require('@alexify/mailer'); +const { fromPath, fromUrl, fromDataUri, fromStream } = require('@alexify/mailer/attachments'); + +const message = createMessage({ + from: 'finance@example.com', + to: 'manager@example.com', + subject: 'Monthly report', + content: { text: 'Attached.' }, + attachments: [ + await fromPath('./reports/october.pdf'), + await fromUrl('https://cdn.example.com/terms.pdf'), + fromDataUri('data:text/csv,month%2Ctotal%0Aoctober%2C42'), + await fromStream(response.body, { filename: 'export.zip' }), + ], +}); +``` + +- **`fromPath(path, options)`** checks that the path is a regular file and reads it + again each time the message is read (a DKIM signature, a retry, another transport + of a pool), never holding it. When the path comes from a user, pass `root`: the + file must be inside it once both are resolved with `realpath`, so neither `..` nor + a symbolic link leads out. +- **`fromUrl(url, options)`** fetches http and https URLs once, through the `fetch` + you pass or the global one, and holds the bytes. It follows at most `maxRedirects` + (5), checks every URL with `allowUrl`, and sends `headers` to the origin of the + first URL only. It gives up after `timeout` (30 s) or `maxBytes` (25 MiB). By + default `allowUrl` is `isPublicUrl`, which refuses `localhost` and loopback, + private and link-local addresses (the cloud metadata endpoint among them). A host + name is fetched whatever it resolves to, so a sender that fetches URLs its users + give passes an `allowUrl` of its own or a `fetch` whose dispatcher checks the + address it connects to. +- **`fromDataUri(uri, options)`** decodes a base64 or percent-encoded `data:` URI + (RFC 2397) up to `maxBytes`, and refuses one that does not parse. +- **`fromStream(source, options)`** reads a Node.js `Readable`, a web + `ReadableStream` or another async iterable of bytes once, up to `maxBytes`, since + a stream cannot be read again. + +Each takes the fields of the attachment as options, `filename`, `contentType`, +`contentId` and `inline`; an inline one needs the `contentId` its HTML refers to. + +### Images of HTML as data URIs + +Many clients block images given as `data:` URIs. `inlineDataUrls(html)` turns each +`` into an inline attachment and the `src` into its `cid:`, as +nodemailer's `attachDataUrls` does: + +```js +const { inlineDataUrls } = require('@alexify/mailer/attachments'); + +const { html, attachments } = inlineDataUrls(renderedHtml, { domain: 'example.com' }); +const message = createMessage({ ...fields, content: { html }, attachments }); +``` + ## Multiple attachments You can attach multiple files to a single message by providing an array of diff --git a/package.json b/package.json index f94795f..dbd1795 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,10 @@ "types": "./src/mime/index.d.ts", "default": "./src/mime/index.js" }, + "./attachments": { + "types": "./src/attachments/index.d.ts", + "default": "./src/attachments/index.js" + }, "./smtp": { "types": "./src/smtp/index.d.ts", "default": "./src/smtp/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 256c90a..24f96c4 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -1,11 +1,12 @@ { ".": 10, "./mime": 7.5, + "./attachments": 4, "./smtp": 20.5, "./jmap": 12.5, "./ses": 6.5, - "./resend": 5.5, - "./sendgrid": 5.5, + "./resend": 6, + "./sendgrid": 6, "./mailgun": 5.5, "./mailtrap": 6.5, "./maileroo": 6, diff --git a/src/attachments/attachment.js b/src/attachments/attachment.js new file mode 100644 index 0000000..6c73620 --- /dev/null +++ b/src/attachments/attachment.js @@ -0,0 +1,81 @@ +// Not ported: the attachment object the helpers of ./attachments return, its +// fields checked when it is made rather than when the message is sent. + +const { checkAttachment } = require('../core/message.js'); + +// RFC 6838 §4.2: a type and a subtype of restricted-name characters. +const MEDIA_TYPE = /^[A-Za-z0-9][\w!#$&^.+-]{0,126}\/[A-Za-z0-9][\w!#$&^.+-]{0,126}$/; + +const DEFAULT_MAX_BYTES = 25 * 1024 * 1024; + +const hasControl = (text) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code < 0x20 || code === 0x7f) return true; + } + return false; +}; + +/** + * The media type of a value, without its parameters, if it is one. + * + * @param value A `Content-Type` value or a media type. + * @returns The type in lower case, or `undefined`. + * @internal + */ +const mediaTypeOf = (value) => { + if (typeof value !== 'string') return undefined; + const end = value.indexOf(';'); + const type = (end === -1 ? value : value.slice(0, end)).trim().toLowerCase(); + return MEDIA_TYPE.test(type) ? type : undefined; +}; + +/** + * Checks a positive whole number of bytes. + * + * @param value The option. + * @returns The bound, `DEFAULT_MAX_BYTES` when it is not given. + * @throws {TypeError} If it is not a positive whole number. + * @internal + */ +const checkMaxBytes = (value) => { + if (value === undefined) return DEFAULT_MAX_BYTES; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`maxBytes must be a positive whole number: ${value}`); + } + return value; +}; + +/** + * Makes an attachment of content and the options a helper was given. + * + * @param options `filename`, `contentType`, `contentId` and `inline`. + * @param content The content. + * @param defaults The `filename` and `contentType` the helper found. + * @returns The attachment. + * @throws {TypeError} If a field is invalid, or an inline attachment has no + * content ID for the HTML to refer to. + * @internal + */ +const createAttachment = (options, content, defaults) => { + const filename = options.filename ?? defaults.filename; + if (typeof filename !== 'string' || filename === '' || hasControl(filename)) { + throw new TypeError(`Invalid attachment filename: ${JSON.stringify(filename)}`); + } + const contentType = options.contentType ?? defaults.contentType; + if (typeof contentType !== 'string' || !MEDIA_TYPE.test(contentType)) { + throw new TypeError(`Invalid attachment content type: ${JSON.stringify(contentType)}`); + } + const inline = options.inline ?? false; + if (typeof inline !== 'boolean') throw new TypeError('inline must be a boolean.'); + const contentId = options.contentId ?? ''; + if (typeof contentId !== 'string' || hasControl(contentId) || /[\s<>]/.test(contentId)) { + throw new TypeError(`Invalid attachment content ID: ${JSON.stringify(contentId)}`); + } + if (inline && contentId === '') { + throw new TypeError('An inline attachment needs the contentId its HTML refers to as cid:.'); + } + return checkAttachment({ inline, filename, content, contentType, contentId }); +}; + +module.exports = { DEFAULT_MAX_BYTES, checkMaxBytes, createAttachment, mediaTypeOf }; diff --git a/src/attachments/data-uri.js b/src/attachments/data-uri.js new file mode 100644 index 0000000..5481a3b --- /dev/null +++ b/src/attachments/data-uri.js @@ -0,0 +1,131 @@ +// Ported from nodemailer v10.0.16 (src/shared/index.ts): `parseDataURI`, an +// attachment of a data URI (RFC 2397). +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Also ported from nodemailer v10.0.16 (src/mailer/index.ts): `_convertDataImages`, +// the data URIs of the images of HTML as inline attachments (`attachDataUrls`). +// Changed in the port: a data URI over `maxBytes`, or one that does not parse, +// is refused, where nodemailer gave an empty attachment or read the URI as a +// file path; base64 is checked rather than decoded leniently; percent-encoded +// bytes are bytes, not UTF-8 text; and a URI without a type is text/plain, as +// RFC 2397 says. + +const { Buffer } = require('node:buffer'); +const { generateId } = require('../core/id.js'); +const { isDotAtomText } = require('../core/text.js'); +const { checkMaxBytes, createAttachment, mediaTypeOf } = require('./attachment.js'); +const { extensionOfType } = require('./mime-types.js'); + +const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/; +const ASCII_WHITESPACE = /[\t\n\f\r ]+/g; +const HEX = /^[0-9A-Fa-f]{2}$/; + +const invalid = (uri) => + new TypeError(`Invalid data URI: ${JSON.stringify(String(uri).slice(0, 64))}`); + +const decodeBase64 = (uri, data, maxBytes) => { + const text = data.replace(ASCII_WHITESPACE, ''); + if (!BASE64.test(text) || text.length % 4 === 1) throw invalid(uri); + if (Math.floor((text.length * 3) / 4) > maxBytes) { + throw new RangeError(`The data URI holds more than ${maxBytes} bytes.`); + } + return new Uint8Array(Buffer.from(text, 'base64')); +}; + +// RFC 2397 data is a URL-encoded string of octets: `%XX` is a byte, and any +// other character stands for its UTF-8 bytes. +const decodePercent = (uri, data, maxBytes) => { + const bytes = []; + let text = ''; + const flush = () => { + if (text === '') return; + const encoded = Buffer.from(text, 'utf8'); + for (let i = 0; i < encoded.length; i++) bytes.push(encoded[i]); + text = ''; + }; + for (let i = 0; i < data.length; i++) { + if (data.charCodeAt(i) !== 0x25) { + text += data[i]; + continue; + } + const hex = data.slice(i + 1, i + 3); + if (!HEX.test(hex)) throw invalid(uri); + flush(); + bytes.push(Number.parseInt(hex, 16)); + i += 2; + } + flush(); + if (bytes.length > maxBytes) { + throw new RangeError(`The data URI holds more than ${maxBytes} bytes.`); + } + return Uint8Array.from(bytes); +}; + +/** + * Makes an attachment of a data URI. + * + * @param uri A `data:` URI, base64 or percent-encoded. + * @param options `maxBytes` (25 MiB), and the fields of the attachment: by + * default the type of the URI and `attachment.`. + * @returns The attachment, its content in memory. + * @throws {TypeError} If the URI does not parse, or a field is invalid. + * @throws {RangeError} If it holds more than `maxBytes`. + */ +const fromDataUri = (uri, options = {}) => { + const maxBytes = checkMaxBytes(options.maxBytes); + if (typeof uri !== 'string' || !/^data:/i.test(uri)) throw invalid(uri); + const comma = uri.indexOf(','); + if (comma === -1) throw invalid(uri); + const meta = uri.slice(5, comma); + const isBase64 = /;base64$/i.test(meta); + const typeEnd = meta.indexOf(';'); + const declared = (typeEnd === -1 ? meta : meta.slice(0, typeEnd)).trim(); + const type = declared === '' ? 'text/plain' : mediaTypeOf(declared); + if (type === undefined) throw invalid(uri); + const data = uri.slice(comma + 1); + const content = isBase64 ? decodeBase64(uri, data, maxBytes) : decodePercent(uri, data, maxBytes); + return createAttachment(options, content, { + filename: `attachment.${extensionOfType(type)}`, + contentType: type, + }); +}; + +// nodemailer's pattern, bounded so that no input makes it backtrack long: an +// whose src is a data URI with a parameter (`;base64`). +const DATA_IMAGE = /(]{0,1024} src\s{0,20}=[\s"']{0,20})(data:([^;]+);[^"'>\s]+)/gi; + +/** + * Turns the images of HTML given as data URIs into inline attachments, each + * referred to by its Content-ID, which most clients show where they block + * data URIs. + * + * @param html The HTML. + * @param options `maxBytes` of one image (25 MiB), `generateId` for the left + * half of each Content-ID and `domain` for its right half (`localhost`). + * @returns The HTML with `cid:` URLs, and the attachments to send with it. + * @throws {TypeError} If an image's data URI does not parse. + * @throws {RangeError} If an image holds more than `maxBytes`. + */ +const inlineDataUrls = (html, options = {}) => { + if (typeof html !== 'string') throw new TypeError('inlineDataUrls() needs HTML.'); + const domain = options.domain ?? 'localhost'; + if (typeof domain !== 'string' || !isDotAtomText(domain)) { + throw new TypeError(`Invalid Content-ID domain: ${JSON.stringify(domain)}`); + } + const attachments = []; + const replaced = html.replace(DATA_IMAGE, (match, prefix, uri, declared) => { + const contentId = `${generateId('content-id', options.generateId)}@${domain}`; + const type = mediaTypeOf(declared) ?? 'application/octet-stream'; + attachments.push( + fromDataUri(uri, { + maxBytes: options.maxBytes, + inline: true, + contentId, + filename: `image-${attachments.length + 1}.${extensionOfType(type)}`, + }), + ); + return `${prefix}cid:${contentId}`; + }); + return { html: replaced, attachments }; +}; + +module.exports = { fromDataUri, inlineDataUrls }; diff --git a/src/attachments/index.d.ts b/src/attachments/index.d.ts new file mode 100644 index 0000000..2d70a4c --- /dev/null +++ b/src/attachments/index.d.ts @@ -0,0 +1,159 @@ +// Not ported: the types of ./attachments, attachments read from a path, a URL, +// a data URI or a stream (after nodemailer and sently). + +import type { Attachment, IdGenerator } from '../../index.js'; + +/** + * The fields of the attachment a helper makes, each with a default the helper + * finds: the file name of the path or URL, and the type of its extension or + * the one the server or data URI gives. + */ +export interface AttachmentOptions { + /** The file name the recipient sees. */ + readonly filename?: string; + /** The media type, such as `application/pdf`. */ + readonly contentType?: `${string}/${string}`; + /** + * The Content-ID the HTML refers to as `cid:`, needed for an inline + * attachment. + */ + readonly contentId?: string; + /** + * Whether the attachment is shown in the HTML rather than listed. + * @default false + */ + readonly inline?: boolean; +} + +/** Options of {@link fromPath}. */ +export interface FromPathOptions extends AttachmentOptions { + /** + * A directory the file must be in, once both are resolved with `realpath`, + * so that neither `..` nor a symbolic link leads out of it. Pass it when + * the path comes from a user. + */ + readonly root?: string; +} + +/** Options of {@link fromUrl}. */ +export interface FromUrlOptions extends AttachmentOptions { + /** The `fetch` to use, such as one with an undici dispatcher. */ + readonly fetch?: typeof globalThis.fetch; + /** Request header fields, sent to the origin of the URL only. */ + readonly headers?: Readonly>; + /** Cancels the request. */ + readonly signal?: AbortSignal; + /** + * The time the request and its body may take, in milliseconds; 0 or + * `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * The most bytes the body may have. + * @default 26214400 + */ + readonly maxBytes?: number; + /** + * The most redirects to follow, each checked by `allowUrl`. + * @default 5 + */ + readonly maxRedirects?: number; + /** + * Whether to fetch a URL, the first and each one a redirect leads to. By + * default ({@link isPublicUrl}) `localhost` and loopback, private and + * link-local addresses are refused; a name is fetched whatever it resolves + * to, so a sender that fetches URLs from users checks them here, or passes + * a `fetch` whose dispatcher checks the address it connects to. + */ + readonly allowUrl?: (url: URL) => boolean | Promise; +} + +/** Options of {@link fromDataUri}. */ +export interface FromDataUriOptions extends AttachmentOptions { + /** + * The most bytes the data URI may hold. + * @default 26214400 + */ + readonly maxBytes?: number; +} + +/** Options of {@link fromStream}. */ +export interface FromStreamOptions extends AttachmentOptions { + /** + * The most bytes the stream may give. + * @default 26214400 + */ + readonly maxBytes?: number; + /** Stops reading. */ + readonly signal?: AbortSignal; +} + +/** Options of {@link inlineDataUrls}. */ +export interface InlineDataUrlsOptions { + /** + * The most bytes one image may hold. + * @default 26214400 + */ + readonly maxBytes?: number; + /** Makes the left half of each Content-ID instead of a UUID. */ + readonly generateId?: IdGenerator; + /** + * The right half of each Content-ID. + * @default "localhost" + */ + readonly domain?: string; +} + +/** + * Makes an attachment of a file, read again each time the message is read. + * + * @example + * ```js + * const { fromPath } = require('@alexify/mailer/attachments'); + * const report = await fromPath('./reports/october.pdf'); + * ``` + */ +export declare function fromPath(path: string, options?: FromPathOptions): Promise; + +/** + * Makes an attachment of what an http or https URL answers, fetched now and + * held in memory. + */ +export declare function fromUrl(url: string | URL, options?: FromUrlOptions): Promise; + +/** + * Makes an attachment of a `data:` URI (RFC 2397), base64 or + * percent-encoded. + */ +export declare function fromDataUri(uri: string, options?: FromDataUriOptions): Attachment; + +/** + * Makes an attachment of what a stream gives, read now and held in memory, + * since a message may be read more than once and a stream cannot be. + */ +export declare function fromStream( + source: AsyncIterable | ReadableStream, + options?: FromStreamOptions, +): Promise; + +/** + * Turns the images of HTML given as data URIs into inline attachments, each + * referred to by its Content-ID. + * + * @example + * ```js + * const { html, attachments } = inlineDataUrls(''); + * createMessage({ ...fields, content: { html }, attachments }); + * ``` + */ +export declare function inlineDataUrls( + html: string, + options?: InlineDataUrlsOptions, +): { readonly html: string; readonly attachments: readonly Attachment[] }; + +/** + * The check of a URL {@link fromUrl} makes when no `allowUrl` is given: no + * `localhost`, and no loopback, private or link-local address. + */ +export declare function isPublicUrl(url: URL): boolean; diff --git a/src/attachments/index.js b/src/attachments/index.js new file mode 100644 index 0000000..8044f4b --- /dev/null +++ b/src/attachments/index.js @@ -0,0 +1,11 @@ +// Not ported: attachments read from a path, a URL, a data URI or a stream, +// after nodemailer's `path`, `href` and data URIs and sently's +// `resolveAttachments`, each a helper the application calls, so that nothing +// reads a file or a URL a message merely names. + +const { fromDataUri, inlineDataUrls } = require('./data-uri.js'); +const { fromPath } = require('./path.js'); +const { fromStream } = require('./stream.js'); +const { fromUrl, isPublicUrl } = require('./url.js'); + +module.exports = { fromDataUri, fromPath, fromStream, fromUrl, inlineDataUrls, isPublicUrl }; diff --git a/src/attachments/mime-types.js b/src/attachments/mime-types.js new file mode 100644 index 0000000..09b300f --- /dev/null +++ b/src/attachments/mime-types.js @@ -0,0 +1,109 @@ +// Ported from nodemailer v10.0.16 (src/mime-funcs/mime-types.ts): the types of +// the file extensions an email attachment commonly has, `detectMimeType` and +// `detectExtension`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: about sixty extensions in place of some 2,000 lines, as +// the table is shipped to every bundle that reads an attachment from a path. + +const DEFAULT_TYPE = 'application/octet-stream'; + +// The extension, its type; the first extension of a type is the one written +// for it. +const TYPES = [ + ['txt', 'text/plain'], + ['text', 'text/plain'], + ['log', 'text/plain'], + ['htm', 'text/html'], + ['html', 'text/html'], + ['css', 'text/css'], + ['csv', 'text/csv'], + ['ics', 'text/calendar'], + ['md', 'text/markdown'], + ['vcf', 'text/vcard'], + ['xml', 'application/xml'], + ['json', 'application/json'], + ['js', 'text/javascript'], + ['png', 'image/png'], + ['jpg', 'image/jpeg'], + ['jpeg', 'image/jpeg'], + ['jpe', 'image/jpeg'], + ['gif', 'image/gif'], + ['webp', 'image/webp'], + ['avif', 'image/avif'], + ['heic', 'image/heic'], + ['svg', 'image/svg+xml'], + ['bmp', 'image/bmp'], + ['tif', 'image/tiff'], + ['tiff', 'image/tiff'], + ['ico', 'image/vnd.microsoft.icon'], + ['pdf', 'application/pdf'], + ['rtf', 'application/rtf'], + ['doc', 'application/msword'], + ['docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'], + ['xls', 'application/vnd.ms-excel'], + ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], + ['ppt', 'application/vnd.ms-powerpoint'], + ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'], + ['odt', 'application/vnd.oasis.opendocument.text'], + ['ods', 'application/vnd.oasis.opendocument.spreadsheet'], + ['odp', 'application/vnd.oasis.opendocument.presentation'], + ['epub', 'application/epub+zip'], + ['zip', 'application/zip'], + ['gz', 'application/gzip'], + ['tgz', 'application/gzip'], + ['tar', 'application/x-tar'], + ['7z', 'application/x-7z-compressed'], + ['rar', 'application/vnd.rar'], + ['eml', 'message/rfc822'], + ['mp3', 'audio/mpeg'], + ['m4a', 'audio/mp4'], + ['wav', 'audio/wav'], + ['ogg', 'audio/ogg'], + ['mp4', 'video/mp4'], + ['m4v', 'video/mp4'], + ['mov', 'video/quicktime'], + ['webm', 'video/webm'], + ['avi', 'video/x-msvideo'], + ['woff', 'font/woff'], + ['woff2', 'font/woff2'], + ['ttf', 'font/ttf'], + ['otf', 'font/otf'], + ['p7s', 'application/pkcs7-signature'], + ['asc', 'application/pgp-signature'], + ['pem', 'application/x-pem-file'], + ['bin', DEFAULT_TYPE], +]; + +const TYPE_OF_EXTENSION = new Map(); +const EXTENSION_OF_TYPE = new Map(); +for (let i = 0; i < TYPES.length; i++) { + const entry = TYPES[i]; + TYPE_OF_EXTENSION.set(entry[0], entry[1]); + if (!EXTENSION_OF_TYPE.has(entry[1])) EXTENSION_OF_TYPE.set(entry[1], entry[0]); +} + +/** + * The media type of a file name by its extension. + * + * @param filename A file name or a path. + * @returns The type, or `application/octet-stream` for an extension it does + * not know. + * @internal + */ +const typeOfFilename = (filename) => { + const dot = filename.lastIndexOf('.'); + const slash = Math.max(filename.lastIndexOf('/'), filename.lastIndexOf('\\')); + if (dot <= slash + 1) return DEFAULT_TYPE; + return TYPE_OF_EXTENSION.get(filename.slice(dot + 1).toLowerCase()) ?? DEFAULT_TYPE; +}; + +/** + * The extension written for a media type. + * + * @param type A media type, without parameters. + * @returns The extension, or `bin`. + * @internal + */ +const extensionOfType = (type) => EXTENSION_OF_TYPE.get(type.toLowerCase()) ?? 'bin'; + +module.exports = { DEFAULT_TYPE, extensionOfType, typeOfFilename }; diff --git a/src/attachments/path.js b/src/attachments/path.js new file mode 100644 index 0000000..27f8caa --- /dev/null +++ b/src/attachments/path.js @@ -0,0 +1,67 @@ +// Ported from sently v1.2.2 (src/transports/resolve-attachments.ts): an +// attachment read from a file, kept within a directory when one is given. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Also ported from nodemailer v10.0.16 (src/mail-composer/index.ts): the file +// name and the type of an attachment by its path. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: the path and the root are resolved with realpath, so a +// symbolic link cannot lead out of the root, which sently compared as written; +// the file is read again for each reading rather than held in memory; and no +// `path` of an attachment is read but through this helper, where nodemailer +// read any it was given. + +const fs = require('node:fs'); +const fsp = require('node:fs/promises'); +const path = require('node:path'); +const { createAttachment } = require('./attachment.js'); +const { typeOfFilename } = require('./mime-types.js'); + +// Whether a resolved path is the directory or under it; a name such as +// `..data` under it is, `../data` is not. +const isWithin = (file, directory) => { + const relative = path.relative(directory, file); + if (path.isAbsolute(relative)) return false; + return relative !== '..' && !relative.startsWith(`..${path.sep}`); +}; + +// Opens the file for each reading, so the content can be read again (for +// DKIM, a retry, another transport of a pool) without holding it. +const readFile = (file) => + async function* (signal) { + yield* fs.createReadStream(file, { signal }); + }; + +/** + * Makes an attachment of a file, read when the message is sent. + * + * @param filePath The path of the file. + * @param options The `root` the file must be in, and the fields of the + * attachment: by default its file name and the type of its extension. + * @returns The attachment. + * @throws {Error} If the file does not exist or cannot be resolved. + * @throws {RangeError} If it is outside `root`. + * @throws {TypeError} If it is not a regular file, or a field is invalid. + */ +const fromPath = async (filePath, options = {}) => { + if (typeof filePath !== 'string' || filePath === '') { + throw new TypeError('fromPath() needs the path of a file.'); + } + const file = await fsp.realpath(filePath); + if (options.root != null) { + const root = await fsp.realpath(options.root); + if (!isWithin(file, root)) { + throw new RangeError( + `${JSON.stringify(filePath)} is outside ${JSON.stringify(options.root)}.`, + ); + } + } + const stats = await fsp.stat(file); + if (!stats.isFile()) throw new TypeError(`${JSON.stringify(filePath)} is not a file.`); + const filename = path.basename(filePath); + return createAttachment(options, readFile(file), { + filename, + contentType: typeOfFilename(options.filename ?? filename), + }); +}; + +module.exports = { fromPath }; diff --git a/src/attachments/stream.js b/src/attachments/stream.js new file mode 100644 index 0000000..4baab68 --- /dev/null +++ b/src/attachments/stream.js @@ -0,0 +1,45 @@ +// Not ported: an attachment of a stream, read once and held, since a message +// may be read more than once (a DKIM signature, a retry, another transport of a +// pool) and a stream cannot be. + +const { joinChunks } = require('../internal/body.js'); +const { checkMaxBytes, createAttachment } = require('./attachment.js'); +const { DEFAULT_TYPE, extensionOfType, typeOfFilename } = require('./mime-types.js'); + +/** + * Makes an attachment of what a stream gives, read now. + * + * @param source A Node.js `Readable`, a web `ReadableStream` or another async + * iterable of bytes. + * @param options `maxBytes` (25 MiB) and `signal`, and the fields of the + * attachment: by default `attachment.bin` and the type of its extension. + * @returns The attachment, its content in memory. + * @throws {TypeError} If the source is not an async iterable of bytes, or a + * field is invalid. + * @throws {RangeError} If it gives more than `maxBytes`. + */ +const fromStream = async (source, options = {}) => { + const maxBytes = checkMaxBytes(options.maxBytes); + if (source == null || typeof source[Symbol.asyncIterator] !== 'function') { + throw new TypeError('fromStream() needs an async iterable of bytes.'); + } + options.signal?.throwIfAborted(); + const chunks = []; + let sizeBytes = 0; + for await (const chunk of source) { + options.signal?.throwIfAborted(); + if (!(chunk instanceof Uint8Array)) { + throw new TypeError('fromStream() reads bytes; a stream of text needs no encoding set.'); + } + sizeBytes += chunk.byteLength; + if (sizeBytes > maxBytes) throw new RangeError(`The stream gives more than ${maxBytes} bytes.`); + chunks.push(chunk); + } + const filename = `attachment.${extensionOfType(DEFAULT_TYPE)}`; + return createAttachment(options, joinChunks(chunks, sizeBytes), { + filename, + contentType: typeOfFilename(options.filename ?? filename), + }); +}; + +module.exports = { fromStream }; diff --git a/src/attachments/url.js b/src/attachments/url.js new file mode 100644 index 0000000..1fc5075 --- /dev/null +++ b/src/attachments/url.js @@ -0,0 +1,172 @@ +// Ported from nodemailer v10.0.16 (src/mail-composer/index.ts): an attachment +// fetched from a URL, its file name taken from the path of the URL. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: the attachment is fetched once, when it is made, +// through the `fetch` passed in, over http and https only; every redirect is +// checked again and sends no header to another origin; time and bytes are +// bounded; and a loopback, private or link-local address is refused unless +// `allowUrl` says otherwise. nodemailer's own client followed five redirects +// with none of these checks, whatever the address. + +const net = require('node:net'); +const { readBytes } = require('../internal/body.js'); +const { checkMaxBytes, createAttachment, mediaTypeOf } = require('./attachment.js'); +const { DEFAULT_TYPE, extensionOfType, typeOfFilename } = require('./mime-types.js'); + +// The addresses no attachment of a sender's message should come from: this +// network, the private ranges, CGNAT, loopback, link-local (where cloud +// metadata answers) and benchmarking; IPv6 unspecified, loopback, unique +// local and link-local. BlockList matches IPv4-mapped IPv6 forms too. +const PRIVATE = new net.BlockList(); +PRIVATE.addSubnet('0.0.0.0', 8, 'ipv4'); +PRIVATE.addSubnet('10.0.0.0', 8, 'ipv4'); +PRIVATE.addSubnet('100.64.0.0', 10, 'ipv4'); +PRIVATE.addSubnet('127.0.0.0', 8, 'ipv4'); +PRIVATE.addSubnet('169.254.0.0', 16, 'ipv4'); +PRIVATE.addSubnet('172.16.0.0', 12, 'ipv4'); +PRIVATE.addSubnet('192.168.0.0', 16, 'ipv4'); +PRIVATE.addSubnet('198.18.0.0', 15, 'ipv4'); +PRIVATE.addAddress('::', 'ipv6'); +PRIVATE.addAddress('::1', 'ipv6'); +PRIVATE.addSubnet('fc00::', 7, 'ipv6'); +PRIVATE.addSubnet('fe80::', 10, 'ipv6'); + +const REDIRECTS = new Set([301, 302, 303, 307, 308]); +const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_MAX_REDIRECTS = 5; + +/** + * The check of a URL when `allowUrl` is not given: no `localhost`, and no + * loopback, private or link-local address. The URL parser has already + * written an address such as `0x7f.1` or `2130706433` as `127.0.0.1`. What a + * name resolves to is not known here: a sender that fetches URLs from users + * passes an `allowUrl` of its own, or a `fetch` whose dispatcher checks the + * address it connects to. + * + * @param url The URL to fetch. + * @returns Whether to fetch it. + */ +const isPublicUrl = (url) => { + let host = url.hostname.toLowerCase(); + if (host.startsWith('[')) host = host.slice(1, -1); + if (host.endsWith('.')) host = host.slice(0, -1); + if (host === 'localhost' || host.endsWith('.localhost')) return false; + const family = net.isIP(host); + if (family === 0) return true; + return !PRIVATE.check(host, family === 4 ? 'ipv4' : 'ipv6'); +}; + +const parseHttpUrl = (value) => { + let url; + try { + url = value instanceof URL ? new URL(value.href) : new URL(value); + } catch { + throw new TypeError(`Invalid attachment URL: ${JSON.stringify(String(value))}`); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new TypeError(`An attachment URL must be http or https: ${url.protocol}`); + } + return url; +}; + +// 0 and Infinity mean no timeout, as for the transports. +const checkTimeout = (value) => { + if (value === undefined) return DEFAULT_TIMEOUT_MS; + if (typeof value !== 'number' || Number.isNaN(value) || value < 0) { + throw new TypeError(`timeout must be a non-negative number: ${value}`); + } + return value; +}; + +const signalOf = (timeout, signal) => { + const timer = + timeout === 0 || timeout === Number.POSITIVE_INFINITY + ? undefined + : AbortSignal.timeout(timeout); + if (timer === undefined) return signal; + return signal == null ? timer : AbortSignal.any([timer, signal]); +}; + +// A decoded segment without the controls and separators a file name of a +// header or of the receiver's disk must not hold. +const withoutUnsafe = (name) => { + let safe = ''; + for (let i = 0; i < name.length; i++) { + const code = name.charCodeAt(i); + if (code >= 0x20 && code !== 0x7f && code !== 0x2f && code !== 0x5c) safe += name[i]; + } + return safe; +}; + +// The last segment of the path, as nodemailer names a file it fetched, with +// the extension of its type when it has none. +const filenameOf = (url, contentType) => { + const path = url.pathname; + let name = path.slice(path.lastIndexOf('/') + 1); + try { + name = decodeURIComponent(name); + } catch { + // Kept percent-encoded. + } + name = withoutUnsafe(name); + if (name === '') name = 'attachment'; + return name.includes('.') ? name : `${name}.${extensionOfType(contentType)}`; +}; + +/** + * Makes an attachment of what a URL answers, fetched now. + * + * @param input The http or https URL. + * @param options `fetch`, `headers`, `signal`, `timeout` (30 s), `maxBytes` + * (25 MiB), `maxRedirects` (5) and `allowUrl`, and the fields of the + * attachment: by default the last segment of the path and the type the + * server gives. + * @returns The attachment, its content in memory. + * @throws {TypeError} If the URL is not http or https, or a field is invalid. + * @throws {RangeError} If a URL is not allowed, or the body is too long. + * @throws {Error} If the server answers with an error or too many redirects. + */ +const fromUrl = async (input, options = {}) => { + const fetch = options.fetch ?? globalThis.fetch; + const maxBytes = checkMaxBytes(options.maxBytes); + const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS; + if (!Number.isSafeInteger(maxRedirects) || maxRedirects < 0) { + throw new TypeError(`maxRedirects must be a whole number: ${maxRedirects}`); + } + const allowUrl = options.allowUrl ?? isPublicUrl; + const signal = signalOf(checkTimeout(options.timeout), options.signal); + const origin = parseHttpUrl(input).origin; + let url = parseHttpUrl(input); + for (let redirects = 0; ; redirects++) { + if (!(await allowUrl(url))) { + throw new RangeError(`The attachment URL is not allowed: ${url.origin}`); + } + // Headers such as Authorization go to the origin they were given for. + const headers = url.origin === origin ? options.headers : undefined; + const response = await fetch(url, { headers, redirect: 'manual', signal }); + if (REDIRECTS.has(response.status)) { + await response.body?.cancel(); + const location = response.headers.get('location'); + if (location === null) throw new Error(`HTTP ${response.status} without a Location.`); + if (redirects >= maxRedirects) throw new Error(`More than ${maxRedirects} redirects.`); + url = parseHttpUrl(new URL(location, url)); + continue; + } + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`Fetching the attachment failed with HTTP ${response.status}.`); + } + const bytes = await readBytes(response, maxBytes); + // A server that names no type, or only application/octet-stream, says + // less than the extension of the file. + const served = mediaTypeOf(response.headers.get('content-type')) ?? DEFAULT_TYPE; + const filename = filenameOf(url, served); + const named = typeOfFilename(options.filename ?? filename); + return createAttachment(options, bytes, { + filename, + contentType: served === DEFAULT_TYPE ? named : served, + }); + } +}; + +module.exports = { fromUrl, isPublicUrl }; diff --git a/src/internal/body.js b/src/internal/body.js index 0f6cc8c..3e9e3d5 100644 --- a/src/internal/body.js +++ b/src/internal/body.js @@ -50,4 +50,49 @@ const readText = async (response, maxBytes = MAX_RESPONSE_BYTES) => { */ const readJson = async (response) => JSON.parse(await readText(response)); -module.exports = { readJson, readText }; +/** + * Reads the bytes of a body, refusing one over `maxBytes` by its + * `Content-Length` or as it comes. + * + * @param response The response. + * @param maxBytes The most bytes to read. + * @returns The bytes, in one buffer. + * @throws {RangeError} If the body is longer. + * @internal + */ +const readBytes = async (response, maxBytes) => { + const tooLong = () => new RangeError(`The response body exceeds ${maxBytes} bytes.`); + if (Number(response.headers.get('content-length')) > maxBytes) { + await response.body?.cancel(); + throw tooLong(); + } + if (response.body == null) return new Uint8Array(0); + const reader = response.body.getReader(); + const chunks = []; + let sizeBytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + sizeBytes += value.byteLength; + if (sizeBytes > maxBytes) { + await reader.cancel(); + throw tooLong(); + } + chunks.push(value); + } + return joinChunks(chunks, sizeBytes); +}; + +// One buffer of the chunks, without a copy when there is one. +const joinChunks = (chunks, sizeBytes) => { + if (chunks.length === 1) return chunks[0]; + const bytes = new Uint8Array(sizeBytes); + let offset = 0; + for (let i = 0; i < chunks.length; i++) { + bytes.set(chunks[i], offset); + offset += chunks[i].byteLength; + } + return bytes; +}; + +module.exports = { joinChunks, readBytes, readJson, readText }; diff --git a/tests/types/attachments.test-d.ts b/tests/types/attachments.test-d.ts new file mode 100644 index 0000000..580d64b --- /dev/null +++ b/tests/types/attachments.test-d.ts @@ -0,0 +1,42 @@ +import { expectError, expectType } from 'tsd'; +import { createMessage, type Attachment } from '../../index.js'; +import { + fromDataUri, + fromPath, + fromStream, + fromUrl, + inlineDataUrls, + isPublicUrl, +} from '../../src/attachments/index.js'; + +expectType>(fromPath('./report.pdf', { root: '/srv/files' })); +expectType>( + fromUrl(new URL('https://example.com/a.png'), { + fetch: globalThis.fetch, + headers: { authorization: 'Bearer x' }, + timeout: 1000, + maxBytes: 1024, + maxRedirects: 0, + allowUrl: async (url) => url.hostname === 'example.com', + inline: true, + contentId: 'logo@example.com', + }), +); +expectType(fromDataUri('data:text/plain,hi', { filename: 'hi.txt' })); +expectType>(fromStream(new ReadableStream())); +const inlined = inlineDataUrls('', { domain: 'example.com' }); +expectType(inlined.html); +expectType(inlined.attachments); +expectType(isPublicUrl(new URL('https://example.com'))); + +createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { html: inlined.html }, + attachments: [fromDataUri('data:,x'), ...inlined.attachments], +}); + +expectError(fromPath(42)); +expectError(fromUrl('https://example.com', { contentType: 'pdf' })); +expectError(fromDataUri('data:,x', { inline: 'yes' })); diff --git a/tests/unit/attachments.test.js b/tests/unit/attachments.test.js new file mode 100644 index 0000000..f58a8c6 --- /dev/null +++ b/tests/unit/attachments.test.js @@ -0,0 +1,410 @@ +// Not ported: the helpers of ./attachments, after nodemailer's `path`, `href` +// and data URIs and sently's `resolveAttachments`. Files are made in a +// temporary directory, URLs answered by a local server. +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const { Readable } = require('node:stream'); +const PostalMime = require('postal-mime'); +const { createMessage, readAttachmentContent } = require('../../index.js'); +const { + fromDataUri, + fromPath, + fromStream, + fromUrl, + inlineDataUrls, + isPublicUrl, +} = require('../../src/attachments/index.js'); +const { composeMessage } = require('../../src/mime/index.js'); + +const bytesOf = (text) => new TextEncoder().encode(text); +const textOf = async (attachment) => + new TextDecoder().decode(await readAttachmentContent(attachment.content)); + +const tempDir = (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-attachments-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +}; + +test('Attachments: fromPath reads the file at each reading, named by its path', async (t) => { + const dir = tempDir(t); + const file = path.join(dir, 'Report October.pdf'); + fs.writeFileSync(file, 'first'); + const attachment = await fromPath(file); + assert.strictEqual(attachment.filename, 'Report October.pdf'); + assert.strictEqual(attachment.contentType, 'application/pdf'); + assert.strictEqual(attachment.inline, false); + assert.strictEqual(attachment.contentId, ''); + assert.strictEqual(typeof attachment.content, 'function'); + assert.strictEqual(await textOf(attachment), 'first'); + // Read again for a retry or a DKIM pass: what is on disk now. + fs.writeFileSync(file, 'second'); + assert.strictEqual(await textOf(attachment), 'second'); + const named = await fromPath(file, { + filename: 'r.txt', + contentId: 'r@example.com', + inline: true, + }); + assert.strictEqual(named.contentType, 'text/plain'); + assert.strictEqual(named.inline, true); + const unknown = path.join(dir, 'data.unknown'); + fs.writeFileSync(unknown, 'x'); + assert.strictEqual((await fromPath(unknown)).contentType, 'application/octet-stream'); +}); + +test('Attachments: fromPath keeps a file within root, a symbolic link too', async (t) => { + const dir = tempDir(t); + const root = path.join(dir, 'files'); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, 'ok.txt'), 'ok'); + fs.writeFileSync(path.join(root, '..notes.txt'), 'ok'); + fs.writeFileSync(path.join(dir, 'secret.txt'), 'secret'); + assert.strictEqual(await textOf(await fromPath(path.join(root, 'ok.txt'), { root })), 'ok'); + assert.strictEqual( + (await fromPath(path.join(root, '..notes.txt'), { root })).filename, + '..notes.txt', + ); + await assert.rejects(fromPath(path.join(root, '../secret.txt'), { root }), RangeError); + // sently compared the paths as written, so a link inside the root that + // points out of it passed. + if (process.platform !== 'win32') { + fs.symlinkSync(path.join(dir, 'secret.txt'), path.join(root, 'link.txt')); + await assert.rejects(fromPath(path.join(root, 'link.txt'), { root }), /is outside/); + } + await assert.rejects(fromPath(root), /is not a file/); + await assert.rejects(fromPath(path.join(root, 'missing.txt')), { code: 'ENOENT' }); + await assert.rejects(fromPath(''), TypeError); + await assert.rejects( + fromPath(path.join(root, 'ok.txt'), { inline: true }), + /needs the contentId/, + ); +}); + +test('Attachments: fromPath stops reading on abort', async (t) => { + const dir = tempDir(t); + const file = path.join(dir, 'big.bin'); + fs.writeFileSync(file, Buffer.alloc(1024 * 1024)); + const attachment = await fromPath(file); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(readAttachmentContent(attachment.content, controller.signal)); +}); + +// A local server: each path answers as its handler says. +const serve = async (t, handlers) => { + const requests = []; + const server = http.createServer((request, response) => { + requests.push({ url: request.url, headers: request.headers }); + const handler = handlers[request.url.split('?')[0]]; + if (handler === undefined) { + response.writeHead(404).end(); + return; + } + handler(request, response); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + return { base: `http://127.0.0.1:${server.address().port}`, requests }; +}; +const anyUrl = () => true; + +test('Attachments: fromUrl fetches once, named by the path and typed by the server', async (t) => { + const { base, requests } = await serve(t, { + '/files/logo.png': (request, response) => { + response.writeHead(200, { 'content-type': 'image/png; charset=binary' }); + response.end('PNG'); + }, + '/download': (request, response) => { + response.writeHead(200, { 'content-type': 'application/pdf' }); + response.end('PDF'); + }, + '/files/r%C3%A9sum%C3%A9.txt': (request, response) => { + response.writeHead(200, { 'content-type': 'application/octet-stream' }); + response.end('cv'); + }, + }); + const logo = await fromUrl(`${base}/files/logo.png?v=2`, { allowUrl: anyUrl }); + assert.strictEqual(logo.filename, 'logo.png'); + assert.strictEqual(logo.contentType, 'image/png'); + assert.ok(logo.content instanceof Uint8Array); + assert.strictEqual(await textOf(logo), 'PNG'); + await textOf(logo); + assert.strictEqual(requests.length, 1); + const pdf = await fromUrl(new URL(`${base}/download`), { allowUrl: anyUrl }); + assert.strictEqual(pdf.filename, 'download.pdf'); + // application/octet-stream says less than the extension. + const cv = await fromUrl(`${base}/files/r%C3%A9sum%C3%A9.txt`, { allowUrl: anyUrl }); + assert.strictEqual(cv.filename, 'résumé.txt'); + assert.strictEqual(cv.contentType, 'text/plain'); +}); + +test('Attachments: fromUrl checks each redirect and sends headers to the origin only', async (t) => { + const other = await serve(t, { + '/a.txt': (request, response) => response.writeHead(200).end('moved'), + }); + const { base, requests } = await serve(t, { + '/start': (request, response) => response.writeHead(302, { location: '/next' }).end(), + '/next': (request, response) => + response.writeHead(301, { location: `${other.base}/a.txt` }).end(), + '/loop': (request, response) => response.writeHead(307, { location: '/loop' }).end(), + '/nowhere': (request, response) => response.writeHead(302).end(), + '/ftp': (request, response) => + response.writeHead(302, { location: 'ftp://example.com/a' }).end(), + }); + const seen = []; + const attachment = await fromUrl(`${base}/start`, { + headers: { authorization: 'Bearer secret' }, + allowUrl: (url) => { + seen.push(url.href); + return true; + }, + }); + assert.strictEqual(await textOf(attachment), 'moved'); + assert.deepStrictEqual(seen, [`${base}/start`, `${base}/next`, `${other.base}/a.txt`]); + assert.strictEqual(requests[0].headers.authorization, 'Bearer secret'); + assert.strictEqual(other.requests[0].headers.authorization, undefined); + await assert.rejects( + fromUrl(`${base}/start`, { allowUrl: (url) => url.origin === base }), + /not allowed/, + ); + await assert.rejects(fromUrl(`${base}/loop`, { allowUrl: anyUrl }), /More than 5 redirects/); + await assert.rejects( + fromUrl(`${base}/start`, { allowUrl: anyUrl, maxRedirects: 1 }), + /More than 1 redirects/, + ); + await assert.rejects(fromUrl(`${base}/nowhere`, { allowUrl: anyUrl }), /without a Location/); + await assert.rejects(fromUrl(`${base}/ftp`, { allowUrl: anyUrl }), /must be http or https/); +}); + +test('Attachments: fromUrl bounds the bytes and the time, and fails on an error status', async (t) => { + const { base } = await serve(t, { + '/big': (request, response) => { + response.writeHead(200, { 'content-length': '2048' }); + response.end(Buffer.alloc(2048)); + }, + '/chunked': (request, response) => { + response.writeHead(200); + response.write(Buffer.alloc(600)); + response.end(Buffer.alloc(600)); + }, + '/slow': (request, response) => { + response.writeHead(200); + response.write('a'); + }, + '/gone': (request, response) => response.writeHead(410).end(), + }); + const options = { allowUrl: anyUrl, maxBytes: 1024 }; + await assert.rejects(fromUrl(`${base}/big`, options), /exceeds 1024 bytes/); + await assert.rejects(fromUrl(`${base}/chunked`, options), /exceeds 1024 bytes/); + await assert.rejects(fromUrl(`${base}/slow`, { allowUrl: anyUrl, timeout: 50 }), { + name: 'TimeoutError', + }); + await assert.rejects(fromUrl(`${base}/gone`, { allowUrl: anyUrl }), /HTTP 410/); + const controller = new AbortController(); + controller.abort(new Error('stopped')); + await assert.rejects( + fromUrl(`${base}/big`, { allowUrl: anyUrl, signal: controller.signal }), + /stopped/, + ); +}); + +test('Attachments: fromUrl refuses local addresses by default, and takes an injected fetch', async () => { + const refused = [ + 'http://localhost/a', + 'http://app.localhost/a', + 'http://127.0.0.1/a', + 'http://2130706433/a', + 'http://0x7f.1/a', + 'http://10.0.0.1/a', + 'http://172.16.5.4/a', + 'http://192.168.1.1/a', + 'http://169.254.169.254/latest/meta-data/', + 'http://100.64.0.1/a', + 'http://0.0.0.0/a', + 'http://[::1]/a', + 'http://[fd00::1]/a', + 'http://[fe80::1]/a', + 'http://[::ffff:127.0.0.1]/a', + ]; + for (const url of refused) { + assert.strictEqual(isPublicUrl(new URL(url)), false, url); + await assert.rejects(fromUrl(url), /not allowed/, url); + } + for (const url of ['https://example.com/a', 'http://93.184.216.34/a', 'http://[2606:4700::1]/']) { + assert.strictEqual(isPublicUrl(new URL(url)), true, url); + } + await assert.rejects(fromUrl('file:///etc/passwd'), /must be http or https/); + await assert.rejects(fromUrl('not a url'), /Invalid attachment URL/); + const calls = []; + const attachment = await fromUrl('https://cdn.example.com/a.css', { + fetch: async (url, init) => { + calls.push({ url: url.href, redirect: init.redirect }); + return new Response('body{}', { headers: { 'content-type': 'text/css' } }); + }, + }); + assert.deepStrictEqual(calls, [{ url: 'https://cdn.example.com/a.css', redirect: 'manual' }]); + assert.strictEqual(attachment.contentType, 'text/css'); + // A segment that does not decode keeps its percent-encoding. + const raw = await fromUrl('https://cdn.example.com/a%E0.txt', { + fetch: async () => new Response('x'), + }); + assert.strictEqual(raw.filename, 'a%E0.txt'); + await assert.rejects(fromUrl('https://example.com', { maxBytes: 0 }), /maxBytes/); + await assert.rejects(fromUrl('https://example.com', { timeout: -1 }), /timeout/); + await assert.rejects(fromUrl('https://example.com', { maxRedirects: 1.5 }), /maxRedirects/); +}); + +test('Attachments: fromDataUri decodes base64 and percent-encoded bytes', async () => { + const png = fromDataUri('data:image/png;base64,iVBO\nRw0K'); + assert.strictEqual(png.contentType, 'image/png'); + assert.strictEqual(png.filename, 'attachment.png'); + assert.deepStrictEqual([...png.content], [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + const text = fromDataUri('data:,Hello%2C%20World%FF'); + assert.strictEqual(text.contentType, 'text/plain'); + assert.deepStrictEqual( + Buffer.from(text.content), + Buffer.concat([Buffer.from('Hello, World'), Buffer.from([0xff])]), + ); + assert.strictEqual( + new TextDecoder().decode(fromDataUri('data:text/plain;charset=utf-8,Привіт').content), + 'Привіт', + ); + assert.strictEqual(fromDataUri('DATA:Text/HTML;base64,PGI+').contentType, 'text/html'); +}); + +test('Attachments: fromDataUri refuses what does not parse or is too long', () => { + // nodemailer read a URI it could not parse as a file path, and turned one + // over 50 MB into an empty attachment. + const invalid = [ + 'image/png;base64,AAAA', + 'data:image/png;base64', + 'data:image/png;base64,AA!A', + 'data:image/png;base64,AAAAA', + 'data:,100%', + 'data:,%G0', + 'data:not a type;base64,AAAA', + 42, + ]; + for (const uri of invalid) assert.throws(() => fromDataUri(uri), /Invalid data URI/, String(uri)); + assert.throws( + () => fromDataUri(`data:;base64,${'A'.repeat(4000)}`, { maxBytes: 1000 }), + RangeError, + ); + assert.throws(() => fromDataUri(`data:,${'a'.repeat(1001)}`, { maxBytes: 1000 }), RangeError); + assert.throws(() => fromDataUri('data:,x', { maxBytes: -1 }), /maxBytes/); + assert.throws( + () => fromDataUri('data:,x', { filename: 'a\r\nb' }), + /Invalid attachment filename/, + ); + assert.throws( + () => fromDataUri('data:,x', { contentType: 'pdf' }), + /Invalid attachment content type/, + ); + assert.throws( + () => fromDataUri('data:,x', { contentId: 'a b' }), + /Invalid attachment content ID/, + ); + assert.throws(() => fromDataUri('data:,x', { inline: 'yes' }), /inline must be a boolean/); +}); + +test('Attachments: fromStream reads a Node stream, a web stream or an async iterable once', async () => { + const node = await fromStream(Readable.from([Buffer.from('ab'), Buffer.from('cd')]), { + filename: 'n.csv', + }); + assert.strictEqual(await textOf(node), 'abcd'); + assert.strictEqual(node.contentType, 'text/csv'); + const web = await fromStream(new Blob(['web']).stream()); + assert.strictEqual(await textOf(web), 'web'); + assert.strictEqual(web.filename, 'attachment.bin'); + assert.strictEqual(web.contentType, 'application/octet-stream'); + const iterable = await fromStream( + (async function* () { + yield bytesOf('it'); + })(), + ); + assert.strictEqual(await textOf(iterable), 'it'); + await assert.rejects(fromStream(Readable.from(['text'])), /reads bytes/); + await assert.rejects(fromStream('text'), /async iterable of bytes/); + await assert.rejects( + fromStream(Readable.from([Buffer.alloc(2000)]), { maxBytes: 1000 }), + RangeError, + ); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects( + fromStream(Readable.from([Buffer.from('a')]), { signal: controller.signal }), + /stop/, + ); +}); + +test('Attachments: inlineDataUrls makes inline attachments of the images of HTML', async () => { + const pixel = 'iVBORw0KGgo='; + const html = + `

a

` + + `` + + 'link'; + let next = 0; + const result = inlineDataUrls(html, { generateId: () => `id${++next}`, domain: 'example.com' }); + assert.strictEqual( + result.html, + '

a

' + + 'link', + ); + assert.deepStrictEqual( + result.attachments.map((attachment) => [ + attachment.filename, + attachment.contentType, + attachment.contentId, + attachment.inline, + ]), + [ + ['image-1.png', 'image/png', 'id1@example.com', true], + ['image-2.gif', 'image/gif', 'id2@example.com', true], + ], + ); + assert.deepStrictEqual(inlineDataUrls('

none

'), { html: '

none

', attachments: [] }); + assert.match( + inlineDataUrls(``).html, + /cid:[\w.-]+@localhost/, + ); + assert.throws(() => inlineDataUrls(''), /Invalid data URI/); + assert.throws(() => inlineDataUrls(42), TypeError); + assert.throws(() => inlineDataUrls('', { domain: 'a b' }), /Content-ID domain/); +}); + +test('Attachments: a composed message carries them, read back by postal-mime', async (t) => { + const dir = tempDir(t); + const file = path.join(dir, 'notes.txt'); + fs.writeFileSync(file, 'from a file'); + const { html, attachments } = inlineDataUrls(''); + const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Files', + content: { html }, + attachments: [ + await fromPath(file), + fromDataUri('data:text/csv,a%2Cb'), + await fromStream(Readable.from([Buffer.from('streamed')]), { filename: 's.txt' }), + ...attachments, + ], + }); + const composed = await composeMessage(message); + const parsed = await PostalMime.parse(await readAttachmentContent(composed.content)); + const found = {}; + for (const attachment of parsed.attachments) { + found[attachment.filename] = [ + attachment.mimeType, + new TextDecoder().decode(attachment.content), + attachment.related === true, + ]; + } + assert.deepStrictEqual(found['notes.txt'], ['text/plain', 'from a file', false]); + assert.deepStrictEqual(found['attachment.csv'], ['text/csv', 'a,b', false]); + assert.deepStrictEqual(found['s.txt'], ['text/plain', 'streamed', false]); + assert.strictEqual(found['image-1.png'][2], true); +}); From 800e8b3784fe0fe9df5eb6ba659977f996ffb336 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:28:14 +0300 Subject: [PATCH 008/143] test(mailpit): the message features of 1.1 against Mailpit An IDN domain arrives as its A-label, a one-click List-Unsubscribe passes Mailpit's own check of the field and of List-Unsubscribe-Post under a DKIM signature an independent verifier accepts, and bodies in base64 and 7bit and an attachment of a data URI arrive as written. Co-Authored-By: Claude Opus 5.5 --- tests/mailpit/features.mailpit.test.js | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/mailpit/features.mailpit.test.js diff --git a/tests/mailpit/features.mailpit.test.js b/tests/mailpit/features.mailpit.test.js new file mode 100644 index 0000000..7c1621c --- /dev/null +++ b/tests/mailpit/features.mailpit.test.js @@ -0,0 +1,102 @@ +// Not ported: the message features of 1.1 against a real SMTP server, Mailpit, +// which also checks List-Unsubscribe and List-Unsubscribe-Post itself. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { createMessage } = require('../../index.js'); +const { SmtpTransport } = require('../../src/smtp/index.js'); +const { fromDataUri } = require('../../src/attachments/index.js'); +const { MailpitClient } = require('../helpers/smtp/mailpit-client.js'); +const { waitForMailpitDelivery } = require('../helpers/smtp/mailpit-delivery-utils.js'); +const { getTestConfig, isMailpitTestingEnabled } = require('../helpers/smtp/test-config.js'); +const { + TEST_DKIM_PRIVATE_KEY, + TEST_DKIM_PUBLIC_KEY, +} = require('../helpers/mime/dkim-test-keys.js'); +const { verifyDkim } = require('../helpers/mime/verify-dkim.js'); + +const skip = !isMailpitTestingEnabled(); + +const deliver = async (message, smtp = {}) => { + const config = getTestConfig(); + const client = new MailpitClient(config.mailpit); + await client.deleteAllMessages(); + const transport = new SmtpTransport({ ...config.smtp, ...smtp }); + try { + const receipt = await transport.send(message); + assert.ok(receipt.successful, JSON.stringify(receipt)); + } finally { + await transport.closeAllConnections(); + } + const delivered = await waitForMailpitDelivery(client, { subject: message.subject }); + return { delivered, source: await client.getMessageSource(delivered.ID) }; +}; + +test('Mailpit: a Unicode domain arrives as its A-label', { skip }, async () => { + const { delivered, source } = await deliver( + createMessage({ + from: { name: 'Jörg', address: 'jorg@bücher.de' }, + to: 'leser@bücher.de', + subject: 'IDN A-label', + content: { text: 'Hallo' }, + }), + ); + assert.equal(delivered.From.Address, 'jorg@xn--bcher-kva.de'); + assert.equal(delivered.From.Name, 'Jörg'); + assert.equal(delivered.To[0].Address, 'leser@xn--bcher-kva.de'); + assert.match(source, /^To: leser@xn--bcher-kva\.de\r$/m); +}); + +test('Mailpit: one-click List-Unsubscribe passes its check, under DKIM', { skip }, async () => { + const { delivered, source } = await deliver( + createMessage({ + from: 'news@example.com', + to: 'reader@example.com', + subject: 'List-Unsubscribe one-click', + content: { html: '

Weekly

', text: 'Weekly' }, + list: { + id: { id: 'news.example.com', name: 'Weekly News' }, + unsubscribe: ['https://example.com/unsubscribe?u=42', 'unsubscribe@example.com'], + oneClick: true, + }, + }), + { + dkim: { + signatures: [ + { signingDomain: 'example.com', selector: 'test', privateKey: TEST_DKIM_PRIVATE_KEY }, + ], + }, + }, + ); + const check = delivered.ListUnsubscribe; + assert.equal(check.Errors, '', JSON.stringify(check)); + assert.equal(check.HeaderPost, 'List-Unsubscribe=One-Click'); + assert.deepEqual(check.Links, [ + 'https://example.com/unsubscribe?u=42', + 'mailto:unsubscribe@example.com', + ]); + verifyDkim(source, TEST_DKIM_PUBLIC_KEY); + assert.match(source, /h=[^;]*list-unsubscribe:list-unsubscribe-post;/); +}); + +test( + 'Mailpit: each text body arrives as it was written, whatever its encoding', + { skip }, + async () => { + const text = 'Серце має свої причини, яких розум не знає.\r\nLe cœur a ses raisons.'; + const html = '

Short ASCII

'; + const { delivered, source } = await deliver( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Encodings', + content: { text, html }, + attachments: fromDataUri('data:text/csv,a%2Cb', { filename: 'data.csv' }), + }), + ); + assert.match(source, /Content-Transfer-Encoding: base64/); + assert.match(source, /Content-Transfer-Encoding: 7bit/); + assert.equal(delivered.Text.replace(/\r\n/g, '\n').trim(), text.replace(/\r\n/g, '\n')); + assert.equal(delivered.HTML.trim(), html); + assert.equal(delivered.Attachments[0].FileName, 'data.csv'); + }, +); From dbd944d4a36a3d75ee5c4a579a96c89ce06f6737 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:54:04 +0300 Subject: [PATCH 009/143] feat(smtp): an injected connect and an HTTP CONNECT proxy `connect` opens each connection in place of a TCP connection to the host, as nodemailer's getSocket does: a proxy, a tunnel or a socket of another module, given the host, port, localAddress and a signal that the connection timeout and the caller's cancellation abort. Implicit TLS and STARTTLS go over the socket it resolves to, checked against the host. createHttpProxyConnector, ported from nodemailer's http-proxy-client, makes one for an HTTP or HTTPS proxy; `localAddress` picks the address to connect from. Fixed in the port (N4): an IPv6 destination is bracketed in the CONNECT line and Host field, an HTTPS proxy takes `ca` and is sent its name for SNI, the credentials of the proxy URL are decoded as UTF-8, and what the server sent with the reply waits in the paused socket. A connection through `connect` counts as local for plain-text AUTH by its host only, as the peer of a tunnel is the proxy. The ./smtp budget goes to 22.5 KB. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 + docs/transports/smtp.md | 54 ++++ scripts/size-budgets.json | 2 +- src/smtp/config.js | 21 +- src/smtp/http-proxy.js | 206 +++++++++++++++ src/smtp/index.d.ts | 78 ++++++ src/smtp/index.js | 2 + src/smtp/smtp-connection.js | 152 ++++++++++-- src/smtp/smtp-transport.js | 8 +- tests/types/smtp.test-d.ts | 25 ++ tests/unit/smtp-connect.test.js | 427 ++++++++++++++++++++++++++++++++ 11 files changed, 960 insertions(+), 22 deletions(-) create mode 100644 src/smtp/http-proxy.js create mode 100644 tests/unit/smtp-connect.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 81a2e74..8d7d4af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -481,6 +481,13 @@ fails on upstream's behavior): bytes stay bytes rather than going through `decodeURIComponent`. - S1. sently (`resolveAttachments`): `basePath` compared the paths as written, so a symbolic link inside it led out of it. `fromPath` compares the `realpath` of the file with that of `root`. +- N4. nodemailer (`http-proxy-client.ts`): an IPv6 destination went bare into the CONNECT + request line and Host field (`CONNECT ::1:25`), which a proxy reads as another port; an HTTPS + proxy took `rejectUnauthorized` alone, no `ca` and no SNI; the credentials of the proxy URL + went through without percent-decoding. `createHttpProxyConnector` (`src/smtp/http-proxy.js`) + brackets it, passes the `tls` options and the proxy's name, decodes the credentials as UTF-8, + and puts back what came with the reply in the paused socket. A connection opened by `connect` + counts as local for plain-text AUTH only by its `host`, never by the proxy it reaches. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index 0df1662..16f31b3 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -179,6 +179,60 @@ without one, and the message ID of a receipt the server named none for (see is used for the token refresh of OAuth 2.0, in place of the global one, such as one bound to an undici `Agent` or a proxy. +### Proxies and connections of your own + +`connect` opens each connection in place of a TCP connection to `host` and +`port`, as nodemailer's `getSocket` does: through a proxy, a tunnel, or a socket +from another module. `createHttpProxyConnector()` makes one for an HTTP or HTTPS +proxy that accepts `CONNECT`: + +```js +const { SmtpTransport, createHttpProxyConnector } = require('@alexify/mailer/smtp'); + +const transport = new SmtpTransport({ + host: 'smtp.example.com', + port: 465, + auth: { user: 'user@example.com', pass: 'secret' }, + connect: createHttpProxyConnector({ + url: 'http://proxy-user:proxy-pass@proxy.internal:3128', + // For an https:// proxy: tls: { ca: [proxyCa] } + }), +}); +``` + +The credentials of the proxy URL go as Basic authentication, as UTF-8; `headers` +adds fields to each CONNECT request, and `timeout` bounds its answer (30 s). A +SOCKS proxy is a module of your own in `connect`, which receives `{ host, port, +localAddress, signal }` and resolves to a connected `net.Socket`, paused or with +nothing read from it yet: + +```js +const { SocksClient } = require('socks'); + +const transport = new SmtpTransport({ + host: 'smtp.example.com', + port: 587, + connect: async ({ host, port }) => { + const { socket } = await SocksClient.createConnection({ + proxy: { host: '127.0.0.1', port: 1080, type: 5 }, + command: 'connect', + destination: { host, port }, + }); + return socket; + }, +}); +``` + +`secure: true` and STARTTLS put TLS over the socket the connector gives, checked +against `host` as over a TCP connection, and the connection timeout and the +cancellation of the send abort its `signal`. Authentication in plain text, which +the transport allows only to this machine, is allowed through a connector only +when `host` itself is a loopback host: the peer of a tunnel is the proxy, often +on this machine, not the server. + +`localAddress` names the address of this machine to connect from, on one with +several. + ### Diagnostics A `logger`, pino-like, hears what the receipts do not tell: `debug` for a diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 24f96c4..44337c9 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -2,7 +2,7 @@ ".": 10, "./mime": 7.5, "./attachments": 4, - "./smtp": 20.5, + "./smtp": 22.5, "./jmap": 12.5, "./ses": 6.5, "./resend": 6, diff --git a/src/smtp/config.js b/src/smtp/config.js index be4e4b8..b6c00fb 100644 --- a/src/smtp/config.js +++ b/src/smtp/config.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/smtp/src/config.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { isIP } = require('node:net'); const { checkDkimConfig } = require('../mime/dkim/types.js'); /** @@ -29,6 +30,8 @@ const createSmtpConfig = (config) => { connectionTimeout: config.connectionTimeout ?? 60000, socketTimeout: config.socketTimeout ?? 60000, localName: config.localName ?? 'localhost', + localAddress: checkLocalAddress(config.localAddress), + connect: checkConnect(config.connect), pool: config.pool ?? true, poolSize: config.poolSize ?? 5, dkim: config.dkim, @@ -38,6 +41,22 @@ const createSmtpConfig = (config) => { }; }; +// `localAddress` is an address of this machine to connect from. +const checkLocalAddress = (address) => { + if (address === undefined) return undefined; + if (typeof address !== 'string' || isIP(address) === 0) { + throw new TypeError(`localAddress must be an IP address: ${JSON.stringify(address)}`); + } + return address; +}; + +const checkConnect = (connect) => { + if (connect !== undefined && typeof connect !== 'function') { + throw new TypeError('connect must be a function that resolves to a connected socket.'); + } + return connect; +}; + // The mechanisms of each kind of credentials; an unknown one is refused when // the transport is made rather than at the first send. const PASSWORD_METHODS = new Set(['plain', 'login']); @@ -52,4 +71,4 @@ const checkAuthMethod = (auth) => { } }; -module.exports = { checkAuthMethod, createSmtpConfig }; +module.exports = { checkAuthMethod, checkConnect, checkLocalAddress, createSmtpConfig }; diff --git a/src/smtp/http-proxy.js b/src/smtp/http-proxy.js new file mode 100644 index 0000000..d7cb605 --- /dev/null +++ b/src/smtp/http-proxy.js @@ -0,0 +1,206 @@ +// Ported from nodemailer v10.0.16 (src/smtp-connection/http-proxy-client.ts): +// an SMTP connection tunneled through an HTTP or HTTPS proxy with CONNECT. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: a connector for the `connect` option, with a promise and +// a signal in place of a callback; an IPv6 destination is bracketed in the +// request line and Host field, which nodemailer wrote bare; an HTTPS proxy +// takes `ca` and the other TLS options and is sent its name for SNI, where +// nodemailer passed `rejectUnauthorized` alone; the credentials of the proxy URL +// are percent-decoded and encoded as UTF-8; and the socket is paused with what +// came after the reply put back, so a greeting sent at once is not lost. + +const { Buffer } = require('node:buffer'); +const net = require('node:net'); +const tls = require('node:tls'); + +// The most bytes of the reply to CONNECT before its blank line, so a proxy +// that never ends its header fields cannot fill the memory of the sender. +const MAX_REPLY_BYTES = 64 * 1024; +const DEFAULT_TIMEOUT_MS = 30_000; +// RFC 9110 §5.6.2: a field name is a token; a value holds no line break. +const TOKEN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; + +const throwTypeError = (message) => { + throw new TypeError(message); +}; + +const decodeCredential = (text) => { + try { + return decodeURIComponent(text); + } catch { + return text; + } +}; + +// The header fields of the CONNECT request after its Host, one per line. +const headerLines = (url, headers) => { + let lines = ''; + if (url.username !== '' || url.password !== '') { + const credentials = `${decodeCredential(url.username)}:${decodeCredential(url.password)}`; + lines += `Proxy-Authorization: Basic ${Buffer.from(credentials, 'utf8').toString('base64')}\r\n`; + } + if (headers == null) return lines; + for (const name in headers) { + if (!Object.hasOwn(headers, name)) continue; + const value = headers[name]; + if (!TOKEN.test(name) || typeof value !== 'string' || /[\r\n\0]/.test(value)) { + throwTypeError(`Invalid proxy header field: ${JSON.stringify(name)}`); + } + lines += `${name}: ${value}\r\n`; + } + return lines; +}; + +const parseProxyUrl = (value) => { + let url; + try { + url = new URL(value); + } catch { + throwTypeError('The proxy URL is not a URL.'); + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throwTypeError(`The proxy URL must be http or https: ${url.protocol}`); + } + return url; +}; + +const checkDestination = (host, port) => { + if (typeof host !== 'string' || host === '' || /[\s\0]/.test(host)) { + throwTypeError(`Invalid proxy destination host: ${JSON.stringify(host)}`); + } + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throwTypeError(`Invalid proxy destination port: ${port}`); + } +}; + +// The reply's status: `HTTP/1.x NNN`. +const statusOf = (head) => { + const match = /^HTTP\/1\.[01] (\d{3})(?: |\r|$)/.exec(head); + return match === null ? undefined : Number(match[1]); +}; + +const proxyError = (status, target) => { + if (status === 407) { + return new Error(`The proxy asks for authentication to reach ${target} (HTTP 407).`); + } + if (status === undefined) { + return new Error(`The proxy did not answer CONNECT ${target} with an HTTP status line.`); + } + return new Error(`The proxy did not connect to ${target}: HTTP ${status}.`); +}; + +/** + * Makes a connector for the `connect` option of `SmtpTransport` that opens + * each connection through an HTTP or HTTPS proxy with `CONNECT`. + * + * @param options `url` of the proxy, with credentials for Basic authentication + * if it needs them; `headers` to add to each CONNECT; `tls` options for an + * HTTPS proxy; `timeout` of the CONNECT exchange (30 s, 0 for none). + * @returns The connector. + * @throws {TypeError} If the URL is not http or https, or a header is invalid. + */ +const createHttpProxyConnector = (options) => { + if (options == null || typeof options !== 'object') { + throwTypeError('createHttpProxyConnector() needs { url }.'); + } + const url = parseProxyUrl(options.url); + const fields = headerLines(url, options.headers); + const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; + if (typeof timeout !== 'number' || Number.isNaN(timeout) || timeout < 0) { + throwTypeError(`Invalid proxy timeout: ${timeout}`); + } + const isHttps = url.protocol === 'https:'; + const defaultPort = isHttps ? 443 : 80; + const proxy = { + isHttps, + host: url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname, + port: url.port === '' ? defaultPort : Number(url.port), + tls: options.tls, + }; + + return async ({ host, port, localAddress, signal }) => { + checkDestination(host, port); + signal?.throwIfAborted(); + const target = net.isIPv6(host) ? `[${host}]:${port}` : `${host}:${port}`; + const request = `CONNECT ${target} HTTP/1.1\r\nHost: ${target}\r\n${fields}\r\n`; + return tunnel(openSocket(proxy, localAddress), proxy.isHttps, request, target, timeout, signal); + }; +}; + +const openSocket = (proxy, localAddress) => + proxy.isHttps + ? tls.connect({ + ...proxy.tls, + host: proxy.host, + port: proxy.port, + servername: net.isIP(proxy.host) === 0 ? proxy.host.replace(/\.$/, '') : undefined, + localAddress, + }) + : net.connect({ host: proxy.host, port: proxy.port, localAddress }); + +// Sends the CONNECT request once the proxy is reached and reads its reply: +// the socket for a 2xx, paused, with what came after the reply put back. +const tunnel = (socket, isHttps, request, target, timeout, signal) => + new Promise((resolve, reject) => { + const connected = isHttps ? 'secureConnect' : 'connect'; + const chunks = []; + let received = 0; + let tail = ''; + + const cleanup = () => { + socket.off(connected, onConnect); + socket.off('data', onData); + socket.off('error', fail); + socket.off('close', onClose); + socket.off('timeout', onTimeout); + signal?.removeEventListener('abort', onAbort); + socket.setTimeout(0); + }; + const fail = (error) => { + cleanup(); + socket.destroy(); + reject(error); + }; + const onAbort = () => fail(signal.reason); + const onClose = () => fail(new Error(`The proxy closed the connection to ${target}.`)); + const onTimeout = () => fail(new Error(`The proxy timed out on CONNECT ${target}.`)); + const onConnect = () => socket.write(request); + // Only the bytes that just came, and the three before them, are searched + // for the end of the header fields. + const onData = (chunk) => { + const window = tail + chunk.toString('latin1'); + const end = window.indexOf('\r\n\r\n'); + chunks.push(chunk); + received += chunk.length; + tail = window.slice(-3); + if (end === -1) { + if (received > MAX_REPLY_BYTES) fail(proxyError(undefined, target)); + return; + } + const reply = Buffer.concat(chunks, received); + const headEnd = received - window.length + end; + const status = statusOf(reply.toString('latin1', 0, headEnd)); + if (status === undefined || status < 200 || status > 299) { + fail(proxyError(status, target)); + return; + } + cleanup(); + // What the server sent with the reply, its greeting perhaps, waits in + // the paused socket for the first reader. + socket.pause(); + if (headEnd + 4 < received) socket.unshift(reply.subarray(headEnd + 4)); + resolve(socket); + }; + + socket.once(connected, onConnect); + socket.on('data', onData); + socket.once('error', fail); + socket.once('close', onClose); + if (timeout > 0 && timeout !== Number.POSITIVE_INFINITY) { + socket.setTimeout(timeout); + socket.once('timeout', onTimeout); + } + signal?.addEventListener('abort', onAbort, { once: true }); + }); + +module.exports = { createHttpProxyConnector }; diff --git a/src/smtp/index.d.ts b/src/smtp/index.d.ts index 6021d3d..7b0a8a9 100644 --- a/src/smtp/index.d.ts +++ b/src/smtp/index.d.ts @@ -91,6 +91,22 @@ export interface SmtpConfig extends TransportDependencies { * The name to use for the local hostname in the HELO/EHLO command. */ readonly localName?: string; + /** + * The local address to connect from, on a machine with several. + */ + readonly localAddress?: string; + /** + * Opens the connection instead of a TCP connection to `host` and `port`: + * through an HTTP proxy ({@link createHttpProxyConnector}), a SOCKS proxy + * module of your own, or any tunnel. It resolves to a connected socket to + * the server, paused or with nothing read from it yet; `secure: true` and + * STARTTLS put TLS over it as over a TCP connection. It is given the + * connection timeout and the caller's cancellation as `signal`. + * + * Plain-text authentication is allowed only when `host` itself is a + * loopback host, whatever address the tunnel connects to. + */ + readonly connect?: (options: SmtpConnectOptions) => SmtpSocket | Promise; /** * Pool connections to reuse them. * @default true @@ -615,6 +631,68 @@ export type SmtpReceipt = * } * ``` */ +/** + * What the `connect` of {@link SmtpConfig} is given. + */ +export interface SmtpConnectOptions { + /** The SMTP server to reach. */ + readonly host: string; + /** Its port. */ + readonly port: number; + /** The `localAddress` of the configuration, if any. */ + readonly localAddress?: string; + /** Aborts on the connection timeout and on the caller's cancellation. */ + readonly signal: AbortSignal; +} +/** + * A connected socket: a `net.Socket` of Node.js, or a duplex stream with its + * methods. + */ +export interface SmtpSocket { + write(chunk: any, ...rest: any[]): boolean; + setTimeout(timeout: number, callback?: () => void): unknown; + destroy(error?: Error): unknown; + on(event: string, listener: (...args: any[]) => void): unknown; +} +/** + * Options of {@link createHttpProxyConnector}. + */ +export interface HttpProxyConnectorOptions { + /** + * The proxy, `http://` or `https://`, with the user name and password of + * its Basic authentication if it asks for one: + * `http://user:pass@proxy.example.com:3128`. + */ + readonly url: string; + /** Header fields to add to each CONNECT request. */ + readonly headers?: Readonly>; + /** TLS options of an `https://` proxy, such as its `ca`. */ + readonly tls?: SmtpTlsOptions; + /** + * The time the proxy may take to answer CONNECT, in milliseconds; 0 or + * `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; +} +/** + * Makes a `connect` for {@link SmtpConfig} that opens each connection through + * an HTTP or HTTPS proxy with CONNECT. + * + * @example + * ```js + * const { SmtpTransport, createHttpProxyConnector } = require('@alexify/mailer/smtp'); + * const transport = new SmtpTransport({ + * host: 'smtp.example.com', + * port: 465, + * auth: { user, pass }, + * connect: createHttpProxyConnector({ url: 'http://proxy.internal:3128' }), + * }); + * ``` + */ +export declare function createHttpProxyConnector( + options: HttpProxyConnectorOptions, +): (options: SmtpConnectOptions) => Promise; export declare class SmtpTransport implements RawTransport<'smtp'>, VerifiableTransport<'smtp'>, AsyncDisposable { diff --git a/src/smtp/index.js b/src/smtp/index.js index 9d59c11..4c22b48 100644 --- a/src/smtp/index.js +++ b/src/smtp/index.js @@ -12,6 +12,7 @@ const { isSmtpResponseProviderDetails } = require('./smtp-receipt.js'); const { SmtpAttachmentReplayError } = require('./message-stream.js'); const { Smtp8BitMimeUnsupportedError } = require('./raw-message.js'); +const { createHttpProxyConnector } = require('./http-proxy.js'); module.exports = { SmtpTransport, @@ -24,4 +25,5 @@ module.exports = { isSmtpResponseProviderDetails, SmtpAttachmentReplayError, Smtp8BitMimeUnsupportedError, + createHttpProxyConnector, }; diff --git a/src/smtp/smtp-connection.js b/src/smtp/smtp-connection.js index fb57f22..10851d5 100644 --- a/src/smtp/smtp-connection.js +++ b/src/smtp/smtp-connection.js @@ -1,5 +1,9 @@ // Ported from upyo v0.7.0 (packages/smtp/src/smtp-connection.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Also ported from nodemailer v10.0.16 (src/smtp-connection/index.ts): a +// connection opened by the `connect` of the caller (nodemailer's `getSocket`), +// such as a proxy tunnel, and from `localAddress`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const { prepareRawSmtpStream, Smtp8BitMimeUnsupportedError } = require('./raw-message.js'); const { Buffer } = require('node:buffer'); @@ -104,6 +108,9 @@ const createReplyCollector = () => { */ const listenToSocket = (socket, signal, onData, onError, onClose, onAbort) => { socket.on('data', onData); + // A connector may hand over its socket paused, with what came early put + // back; a listener alone does not start a stream that was paused. + if (socket.readableFlowing === false) socket.resume(); socket.on('error', onError); socket.on('close', onClose); signal?.addEventListener('abort', onAbort, { once: true }); @@ -373,6 +380,36 @@ const replyCodes = (responses) => { * @param host The configured SMTP host. * @returns `true` if the connected peer or fallback host is loopback. */ +// The TLS handshake of a socket, ended by `secureConnect`, an error, a close +// or the signal. +const secureConnectOf = (socket, signal) => + new Promise((resolve, reject) => { + const cleanup = () => { + socket.off('secureConnect', onSecure); + socket.off('error', onError); + socket.off('close', onClose); + signal.removeEventListener('abort', onAbort); + }; + const onSecure = () => { + cleanup(); + resolve(); + }; + const onError = (error) => { + cleanup(); + reject(error); + }; + // Node's TLSSocket reports a peer that leaves during the handshake as an + // error before the close, so this guard does not run on Node. + /* c8 ignore next */ + const onClose = () => onError(new TypeError('SMTP connection closed before establishment.')); + const onAbort = () => onError(signal.reason); + socket.once('secureConnect', onSecure); + socket.once('error', onError); + socket.once('close', onClose); + signal.addEventListener('abort', onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); + const isLoopbackConnection = (socket, host) => { const remoteAddress = socket?.remoteAddress; return remoteAddress == null ? isLoopbackHost(host) : isLoopbackAddress(remoteAddress); @@ -391,6 +428,11 @@ class SmtpConnection { authenticating = false; /** Whether the connection sits in a pool and must be reset before reuse. */ needsReset = false; + /** + * Whether the socket came from the `connect` of the caller, whose peer, a + * proxy perhaps, is not the SMTP server. + */ + tunneled = false; get usable() { return this.socket != null && !this.socket.destroyed && this.socket.writable; @@ -414,6 +456,9 @@ class SmtpConnection { * this machine, where plain text is allowed for local development. */ isLocal() { + // The peer of a tunnel is the proxy, often on this machine, while the + // server is wherever the host says. + if (this.tunneled) return isLoopbackHost(this.config.host); return isLoopbackConnection(this.socket, this.config.host); } @@ -423,6 +468,7 @@ class SmtpConnection { } signal?.throwIfAborted(); + if (this.config.connect != null) return this.connectThrough(signal); // Implicit TLS is established once its handshake completes, not when the // TCP connection opens, so the connection timeout covers the handshake. @@ -472,23 +518,7 @@ class SmtpConnection { }; try { - if (this.config.secure) { - this.socket = tlsConnect({ - host: this.config.host, - servername: serverNameOf(this.config.host), - port: this.config.port, - rejectUnauthorized: this.config.tls?.rejectUnauthorized ?? true, - ca: this.config.tls?.ca, - key: this.config.tls?.key, - cert: this.config.tls?.cert, - minVersion: this.config.tls?.minVersion, - maxVersion: this.config.tls?.maxVersion, - }); - } else { - this.socket = new Socket(); - this.socket.connect(this.config.port, this.config.host); - } - + this.socket = this.openSocket(); this.socket.setTimeout(this.config.socketTimeout); this.observeSocket(this.socket); this.socket.once(establishedEvent, onConnect); @@ -505,6 +535,94 @@ class SmtpConnection { }); } + // A TCP socket to the server, or the TLS of `secure: true` on one. + openSocket() { + if (this.config.secure) { + return tlsConnect({ + host: this.config.host, + servername: serverNameOf(this.config.host), + port: this.config.port, + localAddress: this.config.localAddress, + rejectUnauthorized: this.config.tls?.rejectUnauthorized ?? true, + ca: this.config.tls?.ca, + key: this.config.tls?.key, + cert: this.config.tls?.cert, + minVersion: this.config.tls?.minVersion, + maxVersion: this.config.tls?.maxVersion, + }); + } + const socket = new Socket(); + socket.connect({ + port: this.config.port, + host: this.config.host, + localAddress: this.config.localAddress, + }); + return socket; + } + + /** + * Opens the connection through the `connect` of the caller, within the + * connection timeout, and over it the TLS of `secure: true`. + * + * @param signal Cancels the connection. + * @throws The reason of `signal`, a timeout, or what the connector throws. + */ + async connectThrough(signal) { + const timeout = new AbortController(); + const timer = setTimeout( + () => timeout.abort(new TypeError('Connection timeout.')), + this.config.connectionTimeout, + ); + const combined = signal == null ? timeout.signal : AbortSignal.any([timeout.signal, signal]); + let tunnel; + try { + tunnel = await this.config.connect({ + host: this.config.host, + port: this.config.port, + localAddress: this.config.localAddress, + signal: combined, + }); + if ( + tunnel == null || + typeof tunnel.write !== 'function' || + typeof tunnel.setTimeout !== 'function' + ) { + throw new TypeError('connect must resolve to a connected net.Socket.'); + } + combined.throwIfAborted(); + this.tunneled = true; + this.socket = this.config.secure ? this.wrapInTls(tunnel) : tunnel; + this.socket.setTimeout(this.config.socketTimeout); + this.observeSocket(this.socket); + if (this.config.secure) await secureConnectOf(this.socket, combined); + this.log.write( + 'debug', + () => ({ port: this.config.port, secure: this.config.secure, tunneled: true }), + 'SMTP connected', + ); + } catch (error) { + tunnel?.destroy?.(); + this.socket?.destroy(); + throw error; + } finally { + clearTimeout(timer); + } + } + + wrapInTls(socket) { + return tlsConnect({ + socket, + host: this.config.host, + servername: serverNameOf(this.config.host), + rejectUnauthorized: this.config.tls?.rejectUnauthorized ?? true, + ca: this.config.tls?.ca, + key: this.config.tls?.key, + cert: this.config.tls?.cert, + minVersion: this.config.tls?.minVersion, + maxVersion: this.config.tls?.maxVersion, + }); + } + sendCommand(command, signal) { return this.sendCommands([command], signal).then((responses) => responses[0]); } diff --git a/src/smtp/smtp-transport.js b/src/smtp/smtp-transport.js index 072ee02..1bf8e8e 100644 --- a/src/smtp/smtp-transport.js +++ b/src/smtp/smtp-transport.js @@ -23,7 +23,7 @@ const { SmtpAttachmentReplayError } = require('./message-stream.js'); const { SmtpDeliveryUnknownError } = require('./data-stream.js'); const { checkDkimConfig } = require('../mime/dkim/types.js'); const { checkDkimKeys } = require('../mime/dkim/sign.js'); -const { checkAuthMethod } = require('./config.js'); +const { checkAuthMethod, checkConnect, checkLocalAddress } = require('./config.js'); const { parseEnhancedSmtpStatusCode } = require('./smtp-status-code.js'); const { TLSSocket } = require('node:tls'); const { warnDegradation } = require('../internal/log.js'); @@ -128,6 +128,8 @@ class SmtpTransport { checkDkimConfig(config.dkim); checkDkimKeys(config.dkim); checkAuthMethod(config.auth); + checkLocalAddress(config.localAddress); + checkConnect(config.connect); this.config = config; const poolSize = config.poolSize ?? 5; if ( @@ -512,7 +514,7 @@ class SmtpTransport { let pooled = this.connectionPool.pop(); while (pooled != null) { if (pooled.usable) { - pooled.socket.ref(); + pooled.socket.ref?.(); if (!pooled.needsReset) return pooled; try { await pooled.reset(signal); @@ -674,7 +676,7 @@ class SmtpTransport { // wait for it, and it finds a connection the server dropped while idle. connection.needsReset = true; // An idle connection does not keep the process alive. - connection.socket.unref(); + connection.socket.unref?.(); // The connection keeps its slot while it sits idle, so no separate check // against `poolSize` is needed here: at most `poolSize` connections can be diff --git a/tests/types/smtp.test-d.ts b/tests/types/smtp.test-d.ts index e8bf2b1..2af327a 100644 --- a/tests/types/smtp.test-d.ts +++ b/tests/types/smtp.test-d.ts @@ -7,6 +7,8 @@ import { } from '../../index.js'; import { SmtpTransport, + createHttpProxyConnector, + type SmtpSocket, SmtpResponseError, isSmtpResponseProviderDetails, type DkimConfig, @@ -58,3 +60,26 @@ expectAssignable({ fetch: globalThis.fetch, }); expectError(new SmtpTransport({ host: 'smtp.example.com', generateId: 'ulid' })); + +// A connection opened through an HTTP proxy, or a connector of one's own. +new SmtpTransport({ + host: 'smtp.example.com', + localAddress: '192.0.2.10', + connect: createHttpProxyConnector({ + url: 'https://user:pass@proxy.example.com:3128', + headers: { 'X-Team': 'mail' }, + tls: { ca: ['-----BEGIN CERTIFICATE-----'] }, + timeout: 5000, + }), +}); +new SmtpTransport({ + host: 'smtp.example.com', + connect: async ({ host, port, signal }) => { + expectType(host); + expectType(port); + expectType(signal); + return {} as SmtpSocket; + }, +}); +expectError(createHttpProxyConnector({})); +expectError(new SmtpTransport({ host: 'smtp.example.com', connect: 'proxy' })); diff --git a/tests/unit/smtp-connect.test.js b/tests/unit/smtp-connect.test.js new file mode 100644 index 0000000..422e7aa --- /dev/null +++ b/tests/unit/smtp-connect.test.js @@ -0,0 +1,427 @@ +// Not ported: the `connect` and `localAddress` of the SMTP transport, and the +// HTTP proxy connector ported from nodemailer, against local servers: an SMTP +// server of plain TCP, implicit TLS or STARTTLS, and an HTTP or HTTPS proxy. +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const http = require('node:http'); +const https = require('node:https'); +const net = require('node:net'); +const path = require('node:path'); +const tls = require('node:tls'); +const { createMessage } = require('../../index.js'); +const { SmtpTransport, createHttpProxyConnector } = require('../../src/smtp/index.js'); + +const credentials = { + key: fs.readFileSync(path.join(__dirname, '../fixtures/smtp/server-key.pem')), + cert: fs.readFileSync(path.join(__dirname, '../fixtures/smtp/server-cert.pem')), +}; +const ca = [credentials.cert.toString()]; + +const message = () => + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Hi', + content: { text: 'Hi' }, + }); + +const listen = (server, host = '127.0.0.1') => + new Promise((resolve) => server.listen(0, host, () => resolve(server.address().port))); +const close = (server) => + new Promise((resolve) => { + server.closeAllConnections?.(); + server.close(() => resolve()); + }); + +// An SMTP server answering a whole transaction; `mode` is 'plain' (with AUTH +// offered in clear), 'implicit' or 'starttls'. +const smtpServer = async (t, mode = 'plain') => { + const state = { commands: [], messages: 0, sockets: new Set() }; + const handle = (socket, upgraded) => { + state.sockets.add(socket); + socket.on('error', () => {}); + if (!upgraded) socket.write('220 ready\r\n'); + let buffer = ''; + let inData = false; + const onData = (bytes) => { + buffer += bytes.toString('latin1'); + let end; + while ((end = buffer.indexOf('\r\n')) >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + if (inData) { + if (line === '.') { + inData = false; + state.messages++; + socket.write('250 queued\r\n'); + } + continue; + } + state.commands.push(line); + const verb = line.split(' ')[0].toUpperCase(); + if (verb === 'EHLO') { + const extra = mode === 'starttls' && !upgraded ? 'STARTTLS' : 'AUTH PLAIN'; + socket.write(`250-hello\r\n250 ${extra}\r\n`); + } else if (verb === 'STARTTLS') { + socket.off('data', onData); + socket.write('220 go\r\n'); + const secure = new tls.TLSSocket(socket, { isServer: true, ...credentials }); + handle(secure, true); + return; + } else if (verb === 'DATA') { + inData = true; + socket.write('354 go\r\n'); + } else if (verb === 'QUIT') { + socket.end('221 bye\r\n'); + } else if (verb === 'AUTH') { + socket.write('235 ok\r\n'); + } else { + socket.write('250 ok\r\n'); + } + } + }; + socket.on('data', onData); + }; + const server = + mode === 'implicit' + ? tls.createServer(credentials, (socket) => handle(socket, false)) + : net.createServer((socket) => handle(socket, false)); + const port = await listen(server); + t.after(() => { + for (const socket of state.sockets) socket.destroy(); + return close(server); + }); + return { port, state }; +}; + +// An HTTP proxy that tunnels CONNECT to 127.0.0.1, or answers as `reply` says. +const proxyServer = async (t, { secure = false, reply } = {}) => { + const requests = []; + // A socket handed over to 'connect' is no longer the server's to close, + // and an HTTP server keeps it half open after the client leaves. + const tunnels = new Set(); + const server = secure ? https.createServer(credentials) : http.createServer(); + server.on('connect', (request, client, head) => { + requests.push({ url: request.url, headers: request.headers }); + tunnels.add(client); + client.on('error', () => {}); + if (reply !== undefined) { + reply(request, client); + return; + } + const port = Number(request.url.slice(request.url.lastIndexOf(':') + 1)); + const upstream = net.connect(port, '127.0.0.1', () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head.length > 0) upstream.write(head); + upstream.pipe(client); + client.pipe(upstream); + }); + upstream.on('error', () => client.destroy()); + }); + const port = await listen(server); + t.after(() => { + for (const tunnel of tunnels) tunnel.destroy(); + return close(server); + }); + return { port, requests, url: `${secure ? 'https' : 'http'}://127.0.0.1:${port}` }; +}; + +const send = async (config) => { + const transport = new SmtpTransport({ pool: false, ...config }); + try { + return await transport.send(message()); + } finally { + await transport.closeAllConnections(); + } +}; + +test('SMTP connect: the connector opens the connection, given the host, port and a signal', async (t) => { + const { port, state } = await smtpServer(t); + const calls = []; + const receipt = await send({ + host: 'localhost', + port, + localAddress: '127.0.0.1', + connect: (options) => { + calls.push(options); + return net.connect(port, '127.0.0.1'); + }, + }); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(state.messages, 1); + assert.strictEqual(calls.length, 1); + assert.strictEqual(calls[0].host, 'localhost'); + assert.strictEqual(calls[0].port, port); + assert.strictEqual(calls[0].localAddress, '127.0.0.1'); + assert.ok(calls[0].signal instanceof AbortSignal); +}); + +test('SMTP connect: implicit TLS and STARTTLS go over the tunnel', async (t) => { + for (const mode of ['implicit', 'starttls']) { + const { port, state } = await smtpServer(t, mode); + const receipt = await send({ + host: 'localhost', + port, + secure: mode === 'implicit', + requireTls: true, + tls: { ca }, + auth: { user: 'u', pass: 'p' }, + connect: ({ port: target }) => net.connect(target, '127.0.0.1'), + }); + assert.strictEqual(receipt.successful, true, `${mode}: ${receipt.errorMessages?.join()}`); + assert.ok( + state.commands.some((command) => command.startsWith('AUTH PLAIN')), + mode, + ); + } +}); + +test('SMTP connect: TLS over the tunnel fails on an untrusted certificate or a silent server', async (t) => { + const { port } = await smtpServer(t, 'implicit'); + const untrusted = await send({ + host: 'localhost', + port, + secure: true, + connect: () => net.connect(port, '127.0.0.1'), + }); + assert.strictEqual(untrusted.successful, false); + assert.strictEqual(untrusted.retryable, false); + const silent = net.createServer((socket) => socket.on('error', () => {})); + const silentPort = await listen(silent); + t.after(() => { + silent.close(); + }); + const sockets = []; + const timedOut = await send({ + host: 'localhost', + port: silentPort, + secure: true, + connectionTimeout: 50, + connect: () => { + const socket = net.connect(silentPort, '127.0.0.1'); + sockets.push(socket); + return socket; + }, + }); + assert.match(timedOut.errorMessages.join(), /Connection timeout/); + assert.ok(sockets[0].destroyed); +}); + +test('SMTP connect: a tunnel to a remote host may not authenticate in clear', async (t) => { + const { port, state } = await smtpServer(t); + // The peer of the tunnel is 127.0.0.1, but the server is smtp.example.com. + const remote = await send({ + host: 'smtp.example.com', + port, + auth: { user: 'u', pass: 'p' }, + connect: () => net.connect(port, '127.0.0.1'), + }); + assert.strictEqual(remote.successful, false); + assert.match(remote.errorMessages.join(), /requires a TLS-secured connection/); + assert.ok(!state.commands.some((command) => command.startsWith('AUTH'))); + const local = await send({ + host: 'localhost', + port, + auth: { user: 'u', pass: 'p' }, + connect: () => net.connect(port, '127.0.0.1'), + }); + assert.strictEqual(local.successful, true, local.errorMessages?.join()); +}); + +test('SMTP connect: the connection timeout and the caller abort the connector', async () => { + let seen; + const timedOut = await send({ + host: 'localhost', + connectionTimeout: 30, + connect: ({ signal }) => + new Promise((resolve, reject) => { + seen = signal; + signal.addEventListener('abort', () => reject(signal.reason)); + }), + }); + assert.strictEqual(timedOut.successful, false); + assert.match(timedOut.errorMessages.join(), /Connection timeout/); + assert.strictEqual(seen.aborted, true); + const transport = new SmtpTransport({ + host: 'localhost', + pool: false, + connect: ({ signal }) => + new Promise((resolve, reject) => + signal.addEventListener('abort', () => reject(signal.reason)), + ), + }); + const controller = new AbortController(); + const sending = transport.send(message(), { signal: controller.signal }); + controller.abort(new Error('stopped')); + await assert.rejects(sending, /stopped/); + const bad = await send({ host: 'localhost', connect: () => ({}) }); + assert.match(bad.errorMessages.join(), /connect must resolve to a connected net.Socket/); +}); + +test('SMTP connect: localAddress and connect are checked when the transport is made', () => { + assert.throws( + () => new SmtpTransport({ host: 'h', localAddress: 'eth0' }), + /localAddress must be an IP/, + ); + assert.throws( + () => new SmtpTransport({ host: 'h', connect: 'proxy' }), + /connect must be a function/, + ); +}); + +test('SMTP proxy: CONNECT through an HTTP proxy, with Basic credentials in UTF-8', async (t) => { + const { port, state } = await smtpServer(t); + const proxy = await proxyServer(t); + const url = new URL(proxy.url); + url.username = encodeURIComponent('юзер'); + url.password = encodeURIComponent('p@ss:word'); + const receipt = await send({ + host: '127.0.0.1', + port, + connect: createHttpProxyConnector({ url: url.href, headers: { 'X-Team': 'mail' } }), + }); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(state.messages, 1); + assert.strictEqual(proxy.requests[0].url, `127.0.0.1:${port}`); + assert.strictEqual(proxy.requests[0].headers.host, `127.0.0.1:${port}`); + assert.strictEqual(proxy.requests[0].headers['x-team'], 'mail'); + const basic = proxy.requests[0].headers['proxy-authorization'].slice('Basic '.length); + assert.strictEqual(Buffer.from(basic, 'base64').toString('utf8'), 'юзер:p@ss:word'); +}); + +test('SMTP proxy: an HTTPS proxy is checked against its CA', async (t) => { + const { port, state } = await smtpServer(t); + const proxy = await proxyServer(t, { secure: true }); + const url = proxy.url.replace('127.0.0.1', 'localhost'); + const receipt = await send({ + host: '127.0.0.1', + port, + connect: createHttpProxyConnector({ url, tls: { ca } }), + }); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(state.messages, 1); + const untrusted = await send({ + host: '127.0.0.1', + port, + connect: createHttpProxyConnector({ url }), + }); + assert.strictEqual(untrusted.successful, false); + assert.strictEqual(untrusted.retryable, false); +}); + +test('SMTP proxy: an IPv6 destination is bracketed, which nodemailer wrote bare', async (t) => { + const proxy = await proxyServer(t, { + reply: (request, client) => client.end('HTTP/1.1 403 Forbidden\r\n\r\n'), + }); + const receipt = await send({ + host: '::1', + port: 2525, + connect: createHttpProxyConnector({ url: proxy.url }), + }); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(proxy.requests[0].url, '[::1]:2525'); + assert.strictEqual(proxy.requests[0].headers.host, '[::1]:2525'); + assert.match(receipt.errorMessages.join(), /did not connect to \[::1\]:2525: HTTP 403/); + assert.strictEqual(receipt.retryable, true); +}); + +test('SMTP proxy: a greeting sent with the reply to CONNECT is not lost', async (t) => { + // A proxy that is the SMTP server too, answering CONNECT and greeting in + // one write. + let commands = 0; + const server = net.createServer((socket) => { + socket.on('error', () => {}); + let buffer = ''; + let connected = false; + let inData = false; + socket.on('data', (bytes) => { + buffer += bytes.toString('latin1'); + if (!connected) { + if (!buffer.includes('\r\n\r\n')) return; + connected = true; + buffer = buffer.slice(buffer.indexOf('\r\n\r\n') + 4); + socket.write('HTTP/1.1 200 OK\r\n\r\n220 ready\r\n'); + return; + } + let end; + while ((end = buffer.indexOf('\r\n')) >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + commands++; + if (inData) { + if (line === '.') { + inData = false; + socket.write('250 queued\r\n'); + } + } else if (line.startsWith('DATA')) { + inData = true; + socket.write('354 go\r\n'); + } else if (line.startsWith('QUIT')) { + socket.end('221 bye\r\n'); + } else { + socket.write('250 ok\r\n'); + } + } + }); + }); + const port = await listen(server); + t.after(() => close(server)); + const receipt = await send({ + host: 'localhost', + port: 25, + connect: createHttpProxyConnector({ url: `http://127.0.0.1:${port}` }), + }); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.ok(commands > 0); +}); + +test('SMTP proxy: 407, a broken reply, a timeout and an abort fail the connection', async (t) => { + const answers = { + auth: (request, client) => client.end('HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'), + garbage: (request, client) => client.end('SMTP/1.0 200 OK\r\n\r\n'), + silent: () => {}, + huge: (request, client) => client.write(`HTTP/1.1 200 OK\r\nX: ${'a'.repeat(70 * 1024)}`), + closed: (request, client) => client.destroy(), + }; + const failures = {}; + for (const [name, reply] of Object.entries(answers)) { + const proxy = await proxyServer(t, { reply }); + const receipt = await send({ + host: 'smtp.example.com', + connect: createHttpProxyConnector({ url: proxy.url, timeout: 100 }), + }); + assert.strictEqual(receipt.successful, false, name); + failures[name] = receipt; + } + assert.match(failures.auth.errorMessages.join(), /asks for authentication .* \(HTTP 407\)/); + assert.strictEqual(failures.auth.retryable, false); + assert.match(failures.garbage.errorMessages.join(), /with an HTTP status line/); + assert.match(failures.silent.errorMessages.join(), /timed out on CONNECT/); + assert.match(failures.huge.errorMessages.join(), /with an HTTP status line/); + assert.match(failures.closed.errorMessages.join(), /closed the connection/); + const proxy = await proxyServer(t, { reply: () => {} }); + const connector = createHttpProxyConnector({ url: proxy.url, timeout: 0 }); + const controller = new AbortController(); + const connecting = connector({ host: 'smtp.example.com', port: 25, signal: controller.signal }); + setTimeout(() => controller.abort(new Error('stopped')), 20); + await assert.rejects(connecting, /stopped/); +}); + +test('SMTP proxy: the options and the destination are checked', async () => { + assert.throws(() => createHttpProxyConnector(), /needs \{ url \}/); + assert.throws(() => createHttpProxyConnector({ url: 'socks5://proxy:1080' }), /http or https/); + assert.throws(() => createHttpProxyConnector({ url: 'not a url' }), /not a URL/); + assert.throws( + () => createHttpProxyConnector({ url: 'http://proxy', headers: { 'Bad Name': 'x' } }), + /Invalid proxy header field/, + ); + assert.throws( + () => createHttpProxyConnector({ url: 'http://proxy', headers: { 'X-A': 'a\r\nB: b' } }), + /Invalid proxy header field/, + ); + assert.throws(() => createHttpProxyConnector({ url: 'http://proxy', timeout: -1 }), /timeout/); + const connector = createHttpProxyConnector({ url: 'http://127.0.0.1:9' }); + await assert.rejects(connector({ host: 'a\r\nb', port: 25 }), /destination host/); + await assert.rejects(connector({ host: 'smtp.example.com', port: 0 }), /destination port/); +}); From 43946bdcc666cc27c1ff111c8cb818a621f18e06 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:55:43 +0300 Subject: [PATCH 010/143] feat(smtp): an FQDN for EHLO by default Upstream greeted every server with EHLO localhost, which a server that checks the name, such as Postfix with reject_non_fqdn_helo_hostname, refuses. As nodemailer's _getHostname does, the default is the host name of this machine when it is a fully qualified domain name, its address as an address literal, and [127.0.0.1] otherwise. An explicit localName, which goes into EHLO as it is, is refused when it holds whitespace or a control character that would end the command. Co-Authored-By: Claude Opus 5.5 --- docs/transports/smtp.md | 5 +++ src/smtp/config.js | 53 +++++++++++++++++++++++++++++-- src/smtp/index.d.ts | 5 ++- src/smtp/smtp-transport.js | 3 +- tests/unit/smtp-ehlo.test.js | 61 ++++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 tests/unit/smtp-ehlo.test.js diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index 16f31b3..a5dea77 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -141,6 +141,11 @@ while `SmtpConfig.secure` determines whether to use implicit TLS and `SmtpConfig.requireTls` makes a STARTTLS upgrade mandatory for plaintext connections. Connection and socket timeouts prevent hanging connections, and the `SmtpConfig.localName` identifies your server during the SMTP handshake. +Without one, EHLO gives the host name of this machine when it is a fully +qualified domain name, its address as an address literal, or `[127.0.0.1]`, as +nodemailer does; a server that checks the name, such as Postfix with +`reject_non_fqdn_helo_hostname`, refuses `localhost`. A name with whitespace or +a control character is refused when the transport is made. Connection pooling improves performance by reusing connections across multiple messages. diff --git a/src/smtp/config.js b/src/smtp/config.js index b6c00fb..523aebb 100644 --- a/src/smtp/config.js +++ b/src/smtp/config.js @@ -1,7 +1,11 @@ // Ported from upyo v0.7.0 (packages/smtp/src/config.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Also ported from nodemailer v10.0.16 (src/smtp-connection/index.ts): the name +// EHLO gives by default, `_getHostname`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const { isIP } = require('node:net'); +const os = require('node:os'); const { checkDkimConfig } = require('../mime/dkim/types.js'); /** @@ -29,7 +33,7 @@ const createSmtpConfig = (config) => { tls: config.tls, connectionTimeout: config.connectionTimeout ?? 60000, socketTimeout: config.socketTimeout ?? 60000, - localName: config.localName ?? 'localhost', + localName: config.localName == null ? defaultLocalName() : checkLocalName(config.localName), localAddress: checkLocalAddress(config.localAddress), connect: checkConnect(config.connect), pool: config.pool ?? true, @@ -41,6 +45,44 @@ const createSmtpConfig = (config) => { }; }; +// A host name of letters, digits and hyphens, in labels of up to 63, with a +// dot: one a server checking `reject_non_fqdn_helo_hostname` takes. +const FQDN = + /^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/; + +/** + * The name EHLO gives when `localName` is not set, as nodemailer finds it: the + * host name of this machine when it is a fully qualified domain name, its + * address as an address literal, and `[127.0.0.1]` otherwise. A server such + * as Postfix with `reject_non_fqdn_helo_hostname` refuses `localhost`, the + * name upstream gave. + * + * @returns The name. + * @internal + */ +const defaultLocalName = () => { + let name; + try { + name = os.hostname().replace(/\.$/, ''); + } catch { + return '[127.0.0.1]'; + } + const family = isIP(name); + if (family === 4) return `[${name}]`; + if (family === 6) return `[IPv6:${name}]`; + return FQDN.test(name) ? name : '[127.0.0.1]'; +}; + +// The name goes into EHLO as it is, so it holds no whitespace or control +// character, which would end the command. +const checkLocalName = (name) => { + // oxlint-disable-next-line no-control-regex -- the controls are what it refuses + if (typeof name !== 'string' || name === '' || /[\s\0-\x1f\x7f]/.test(name)) { + throw new TypeError(`Invalid localName: ${JSON.stringify(name)}`); + } + return name; +}; + // `localAddress` is an address of this machine to connect from. const checkLocalAddress = (address) => { if (address === undefined) return undefined; @@ -71,4 +113,11 @@ const checkAuthMethod = (auth) => { } }; -module.exports = { checkAuthMethod, checkConnect, checkLocalAddress, createSmtpConfig }; +module.exports = { + checkAuthMethod, + checkConnect, + checkLocalAddress, + checkLocalName, + createSmtpConfig, + defaultLocalName, +}; diff --git a/src/smtp/index.d.ts b/src/smtp/index.d.ts index 7b0a8a9..3b5928b 100644 --- a/src/smtp/index.d.ts +++ b/src/smtp/index.d.ts @@ -88,7 +88,10 @@ export interface SmtpConfig extends TransportDependencies { */ readonly socketTimeout?: number; /** - * The name to use for the local hostname in the HELO/EHLO command. + * The name to use for the local hostname in the HELO/EHLO command. By + * default the host name of this machine when it is a fully qualified + * domain name, its address as an address literal, or `[127.0.0.1]`, as + * nodemailer gives it; a server that checks the name refuses `localhost`. */ readonly localName?: string; /** diff --git a/src/smtp/smtp-transport.js b/src/smtp/smtp-transport.js index 1bf8e8e..177b1b4 100644 --- a/src/smtp/smtp-transport.js +++ b/src/smtp/smtp-transport.js @@ -23,7 +23,7 @@ const { SmtpAttachmentReplayError } = require('./message-stream.js'); const { SmtpDeliveryUnknownError } = require('./data-stream.js'); const { checkDkimConfig } = require('../mime/dkim/types.js'); const { checkDkimKeys } = require('../mime/dkim/sign.js'); -const { checkAuthMethod, checkConnect, checkLocalAddress } = require('./config.js'); +const { checkAuthMethod, checkConnect, checkLocalAddress, checkLocalName } = require('./config.js'); const { parseEnhancedSmtpStatusCode } = require('./smtp-status-code.js'); const { TLSSocket } = require('node:tls'); const { warnDegradation } = require('../internal/log.js'); @@ -129,6 +129,7 @@ class SmtpTransport { checkDkimKeys(config.dkim); checkAuthMethod(config.auth); checkLocalAddress(config.localAddress); + if (config.localName != null) checkLocalName(config.localName); checkConnect(config.connect); this.config = config; const poolSize = config.poolSize ?? 5; diff --git a/tests/unit/smtp-ehlo.test.js b/tests/unit/smtp-ehlo.test.js new file mode 100644 index 0000000..c4ea03f --- /dev/null +++ b/tests/unit/smtp-ehlo.test.js @@ -0,0 +1,61 @@ +// Not ported: the name EHLO gives by default, after nodemailer's +// `_getHostname`, where upyo gave `localhost`. +const { test } = require('node:test'); +const assert = require('node:assert'); +const os = require('node:os'); +const { createMessage } = require('../../index.js'); +const { SmtpTransport } = require('../../src/smtp/index.js'); +const { defaultLocalName } = require('../../src/smtp/config.js'); +const { MockSmtpServer } = require('../helpers/smtp/mock-smtp-server.js'); + +test('EHLO: the host name of this machine when it is fully qualified', (t) => { + const names = [ + ['mail.example.com', 'mail.example.com'], + ['mail.example.com.', 'mail.example.com'], + ['host-1.eu.example.org', 'host-1.eu.example.org'], + ['my-laptop', '[127.0.0.1]'], + ['bad_name.local', '[127.0.0.1]'], + ['-dash.example.com', '[127.0.0.1]'], + ['', '[127.0.0.1]'], + ['192.0.2.7', '[192.0.2.7]'], + ['2001:db8::7', '[IPv6:2001:db8::7]'], + ]; + const hostname = t.mock.method(os, 'hostname', () => ''); + for (const [name, expected] of names) { + hostname.mock.mockImplementation(() => name); + assert.strictEqual(defaultLocalName(), expected, name); + } + hostname.mock.mockImplementation(() => { + throw new Error('uv_os_gethostname'); + }); + assert.strictEqual(defaultLocalName(), '[127.0.0.1]'); +}); + +test('EHLO: the transport greets with that name, or with localName', async (t) => { + t.mock.method(os, 'hostname', () => 'relay.example.net'); + const server = new MockSmtpServer(); + const port = await server.start(); + const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Hi', + content: { text: 'Hi' }, + }); + try { + for (const localName of [undefined, 'sender.example.com']) { + const transport = new SmtpTransport({ host: 'localhost', port, pool: false, localName }); + const receipt = await transport.send(message); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + } + const greetings = server.getReceivedCommands().filter((command) => command.startsWith('EHLO')); + assert.deepStrictEqual(greetings, ['EHLO relay.example.net', 'EHLO sender.example.com']); + } finally { + await server.stop(); + } +}); + +test('EHLO: a localName that would end the command is refused', () => { + for (const localName of ['a\r\nQUIT', 'two words', '', 42]) { + assert.throws(() => new SmtpTransport({ host: 'localhost', localName }), /Invalid localName/); + } +}); From 8f6f7d1f9573494bfc9f8e7529d7eb1c90db1ca4 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 20:58:40 +0300 Subject: [PATCH 011/143] feat(smtp): maxMessages per pooled connection Upstream reused a connection for any number of messages, while servers limit the messages of one connection and answer the next with an error. As nodemailer's pool does, a connection that has started maxMessages transactions (100 by default, Infinity for none) is closed with QUIT instead of going back to the pool, and sendMany takes a new connection for the rest of its batch, keeping the order of the receipts. Co-Authored-By: Claude Opus 5.5 --- docs/transports/smtp.md | 5 ++ src/smtp/config.js | 13 ++++ src/smtp/index.d.ts | 9 +++ src/smtp/smtp-connection.js | 8 ++ src/smtp/smtp-transport.js | 31 ++++++-- tests/types/smtp.test-d.ts | 4 + tests/unit/smtp-max-messages.test.js | 111 +++++++++++++++++++++++++++ 7 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 tests/unit/smtp-max-messages.test.js diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index a5dea77..a56860d 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -169,6 +169,11 @@ the process alive, so a script that sends a message and returns can exit without `closeAllConnections()`, though closing still ends the session politely with QUIT. +A connection sends `maxMessages` messages (100 by default, as nodemailer's pool), +then it is closed with QUIT and the next message opens a new one, within a +`sendMany()` batch too: servers limit the messages of one connection and answer +the next with an error. Pass `Infinity` to keep a connection for good. + > [!NOTE] > Because a `SmtpTransport.sendMany()` call holds its connection until the > iteration ends, running more concurrent `SmtpTransport.sendMany()` calls than diff --git a/src/smtp/config.js b/src/smtp/config.js index 523aebb..ea1618d 100644 --- a/src/smtp/config.js +++ b/src/smtp/config.js @@ -38,6 +38,7 @@ const createSmtpConfig = (config) => { connect: checkConnect(config.connect), pool: config.pool ?? true, poolSize: config.poolSize ?? 5, + maxMessages: checkMaxMessages(config.maxMessages), dkim: config.dkim, fetch: config.fetch, generateId: config.generateId, @@ -83,6 +84,17 @@ const checkLocalName = (name) => { return name; }; +// As nodemailer's pool, a connection sends 100 messages, then a new one is +// opened: servers limit the messages of a connection, and answer the next +// with an error. +const checkMaxMessages = (value) => { + if (value === undefined) return 100; + if (value !== Number.POSITIVE_INFINITY && (!Number.isSafeInteger(value) || value < 1)) { + throw new TypeError(`maxMessages must be a positive whole number or Infinity: ${value}`); + } + return value; +}; + // `localAddress` is an address of this machine to connect from. const checkLocalAddress = (address) => { if (address === undefined) return undefined; @@ -118,6 +130,7 @@ module.exports = { checkConnect, checkLocalAddress, checkLocalName, + checkMaxMessages, createSmtpConfig, defaultLocalName, }; diff --git a/src/smtp/index.d.ts b/src/smtp/index.d.ts index 3b5928b..33e156f 100644 --- a/src/smtp/index.d.ts +++ b/src/smtp/index.d.ts @@ -132,6 +132,15 @@ export interface SmtpConfig extends TransportDependencies { * @default 5 */ readonly poolSize?: number; + /** + * The messages a connection sends before it is closed and a new one is + * opened, as servers limit the messages of one connection (nodemailer's + * `maxMessages`). `sendMany()` opens a new connection within a batch. + * `Infinity` for no limit. + * + * @default 100 + */ + readonly maxMessages?: number; /** * DKIM signing configuration. * When provided, all outgoing emails will be signed with DKIM. diff --git a/src/smtp/smtp-connection.js b/src/smtp/smtp-connection.js index 10851d5..4bbdc5e 100644 --- a/src/smtp/smtp-connection.js +++ b/src/smtp/smtp-connection.js @@ -428,6 +428,8 @@ class SmtpConnection { authenticating = false; /** Whether the connection sits in a pool and must be reset before reuse. */ needsReset = false; + /** The transactions this connection has started, for `maxMessages`. */ + messageCount = 0; /** * Whether the socket came from the `connect` of the caller, whose peer, a * proxy perhaps, is not the SMTP server. @@ -438,6 +440,11 @@ class SmtpConnection { return this.socket != null && !this.socket.destroyed && this.socket.writable; } + /** Whether it has started as many transactions as `maxMessages` allows. */ + get exhausted() { + return this.messageCount >= this.config.maxMessages; + } + observeSocket(socket) { socket.on('error', () => socket.destroy()); socket.on('timeout', () => { @@ -1133,6 +1140,7 @@ class SmtpConnection { async sendMessage(message, signal) { this.active = true; this.socket?.setTimeout(0); + this.messageCount++; try { return await this.sendPreparedMessage(message, signal); } finally { diff --git a/src/smtp/smtp-transport.js b/src/smtp/smtp-transport.js index 177b1b4..af3bb95 100644 --- a/src/smtp/smtp-transport.js +++ b/src/smtp/smtp-transport.js @@ -23,7 +23,13 @@ const { SmtpAttachmentReplayError } = require('./message-stream.js'); const { SmtpDeliveryUnknownError } = require('./data-stream.js'); const { checkDkimConfig } = require('../mime/dkim/types.js'); const { checkDkimKeys } = require('../mime/dkim/sign.js'); -const { checkAuthMethod, checkConnect, checkLocalAddress, checkLocalName } = require('./config.js'); +const { + checkAuthMethod, + checkConnect, + checkLocalAddress, + checkLocalName, + checkMaxMessages, +} = require('./config.js'); const { parseEnhancedSmtpStatusCode } = require('./smtp-status-code.js'); const { TLSSocket } = require('node:tls'); const { warnDegradation } = require('../internal/log.js'); @@ -130,6 +136,7 @@ class SmtpTransport { checkAuthMethod(config.auth); checkLocalAddress(config.localAddress); if (config.localName != null) checkLocalName(config.localName); + checkMaxMessages(config.maxMessages); checkConnect(config.connect); this.config = config; const poolSize = config.poolSize ?? 5; @@ -383,11 +390,10 @@ class SmtpTransport { } } + let receipt; try { options?.signal?.throwIfAborted(); - const result = await connection.sendMessage(smtpMessage, options?.signal); - - yield sentReceipt(result); + receipt = sentReceipt(await connection.sendMessage(smtpMessage, options?.signal)); } catch (error) { // Cancellation rejects rather than producing a receipt. options?.signal?.throwIfAborted(); @@ -396,8 +402,16 @@ class SmtpTransport { connectionValid = false; } - yield failureOf(error); + receipt = failureOf(error); } + // A connection that has started `maxMessages` transactions is closed, + // and the next message of the batch takes a new one. + if (connectionValid && connection.exhausted) { + const used = connection; + connection = undefined; + await this.returnConnection(used); + } + yield receipt; } } catch (error) { // Error during iteration setup or while reading the messages @@ -667,7 +681,12 @@ class SmtpTransport { * waiting caller. */ async returnConnection(connection) { - if (!connection.usable || !connection.config.pool || this.closing != null) { + if ( + !connection.usable || + !connection.config.pool || + connection.exhausted || + this.closing != null + ) { await this.discardConnection(connection); return; } diff --git a/tests/types/smtp.test-d.ts b/tests/types/smtp.test-d.ts index 2af327a..c1b73a4 100644 --- a/tests/types/smtp.test-d.ts +++ b/tests/types/smtp.test-d.ts @@ -83,3 +83,7 @@ new SmtpTransport({ }); expectError(createHttpProxyConnector({})); expectError(new SmtpTransport({ host: 'smtp.example.com', connect: 'proxy' })); + +// maxMessages +new SmtpTransport({ host: 'smtp.example.com', maxMessages: 50 }); +expectError(new SmtpTransport({ host: 'smtp.example.com', maxMessages: '50' })); diff --git a/tests/unit/smtp-max-messages.test.js b/tests/unit/smtp-max-messages.test.js new file mode 100644 index 0000000..01bc69b --- /dev/null +++ b/tests/unit/smtp-max-messages.test.js @@ -0,0 +1,111 @@ +// Not ported: `maxMessages`, the messages one SMTP connection sends before a +// new one is opened, after nodemailer's pool (upyo reused a connection +// without a limit). +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { SmtpTransport } = require('../../src/smtp/index.js'); +const { MockSmtpServer } = require('../helpers/smtp/mock-smtp-server.js'); + +const message = (index) => + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: `Message ${index}`, + content: { text: 'Hi' }, + }); +const messages = (count) => { + const list = []; + for (let i = 0; i < count; i++) list.push(message(i)); + return list; +}; + +const withServer = async (run) => { + const server = new MockSmtpServer(); + const port = await server.start(); + try { + await run(server, port); + } finally { + await server.stop(); + } +}; + +const quits = (server) => server.getReceivedCommands().filter((c) => c === 'QUIT').length; + +test('maxMessages: send() opens a new connection after so many messages', async () => { + await withServer(async (server, port) => { + const transport = new SmtpTransport({ host: 'localhost', port, poolSize: 1, maxMessages: 2 }); + for (const each of messages(5)) { + const receipt = await transport.send(each); + assert.strictEqual(receipt.successful, true); + } + await transport.closeAllConnections(); + assert.strictEqual(server.getConnectionCount(), 3); + // The two that reached the limit are closed with QUIT. + assert.ok(quits(server) >= 2); + }); +}); + +test('maxMessages: sendMany() changes connection within a batch, in order', async () => { + await withServer(async (server, port) => { + const transport = new SmtpTransport({ host: 'localhost', port, maxMessages: 2 }); + const receipts = []; + for await (const receipt of transport.sendMany(messages(5))) receipts.push(receipt); + await transport.closeAllConnections(); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, true, true, true, true], + ); + assert.strictEqual(server.getConnectionCount(), 3); + assert.strictEqual(server.getReceivedMessages().length, 5); + }); +}); + +test('maxMessages: a refused message counts, and 100 is the default', async () => { + await withServer(async (server, port) => { + // A message the connection refuses, keeping it usable: the server offers + // no SMTPUTF8 for this address. + const refused = createMessage({ + from: 'a@example.com', + to: 'josé@example.com', + subject: 'Refused', + content: { text: 'Hi' }, + }); + const transport = new SmtpTransport({ host: 'localhost', port, maxMessages: 2 }); + const receipts = []; + for await (const receipt of transport.sendMany([refused, message(1), message(2)])) { + receipts.push(receipt); + } + await transport.closeAllConnections(); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [false, true, true], + ); + assert.strictEqual(server.getConnectionCount(), 2); + }); + await withServer(async (server, port) => { + const transport = new SmtpTransport({ host: 'localhost', port }); + for await (const receipt of transport.sendMany(messages(101))) { + assert.strictEqual(receipt.successful, true); + } + await transport.closeAllConnections(); + assert.strictEqual(server.getConnectionCount(), 2); + }); +}); + +test('maxMessages: Infinity keeps the connection, an invalid value is refused', async () => { + await withServer(async (server, port) => { + const transport = new SmtpTransport({ host: 'localhost', port, maxMessages: Infinity }); + for await (const receipt of transport.sendMany(messages(3))) { + assert.strictEqual(receipt.successful, true); + } + await transport.closeAllConnections(); + assert.strictEqual(server.getConnectionCount(), 1); + }); + for (const maxMessages of [0, -1, 1.5, '10', Number.NaN]) { + assert.throws( + () => new SmtpTransport({ host: 'localhost', maxMessages }), + /maxMessages must be a positive whole number/, + ); + } +}); From 7a38dc9339742c809f44d3ba83573498a0888aa0 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:08:09 +0300 Subject: [PATCH 012/143] feat(sendmail): a transport over the sendmail command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new subpath, ported from nodemailer's sendmail transport, pipes each message to the sendmail command of the system (Postfix, Exim, msmtp, …), one process per message: ` -i -f -- `, or `args` in place of -f. The message is composed by ./mime, so it gets the DKIM signature, encodings and IDN addresses of SMTP; sendRaw pipes serialized MIME, and verify finds the command at its path or on PATH. Line endings are LF by default, and `spawn` is passed in. Fixed in the port (N5): a process killed by a signal was a success and the code of `close` went on as an error; only an exit with 0 succeeds, a signal after the whole message is deliveryUnknown, and the sysexits codes are classified (75 and 69 retryable, 127 configuration). stderr, never read there, is read bounded and redacted into the failure; `--` ends the options before the recipients; blind recipients go on the command line only, never in a Bcc field for the MTA to strip; an abort kills the process, and an attachment that cannot be read kills it with SIGINT so no part is delivered. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 3 +- docs/transports/sendmail.md | 87 +++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/sendmail/config.js | 57 +++++ src/sendmail/index.d.ts | 95 +++++++ src/sendmail/index.js | 7 + src/sendmail/sendmail-transport.js | 334 +++++++++++++++++++++++++ tests/types/sendmail.test-d.ts | 33 +++ tests/unit/graph.test.js | 4 + tests/unit/sendmail.test.js | 388 +++++++++++++++++++++++++++++ 13 files changed, 1019 insertions(+), 1 deletion(-) create mode 100644 docs/transports/sendmail.md create mode 100644 src/sendmail/config.js create mode 100644 src/sendmail/index.d.ts create mode 100644 src/sendmail/index.js create mode 100644 src/sendmail/sendmail-transport.js create mode 100644 tests/types/sendmail.test-d.ts create mode 100644 tests/unit/sendmail.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 8d7d4af..a490cf5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -488,6 +488,12 @@ fails on upstream's behavior): brackets it, passes the `tls` options and the proxy's name, decodes the credentials as UTF-8, and puts back what came with the reply in the paused socket. A connection opened by `connect` counts as local for plain-text AUTH only by its `host`, never by the proxy it reaches. +- N5. nodemailer (`sendmail-transport`): a process killed by a signal (`code === null`) was a + success, the code of `close` was handed on as an error value, stderr was never read, the + recipients followed the options with no `--`, and the message kept its Bcc field for the MTA to + strip. `./sendmail` succeeds on an exit with 0 alone, reports a signal after the whole message + as `deliveryUnknown`, classifies the sysexits codes, reads stderr bounded and redacted, ends the + options with `--`, and puts blind recipients on the command line only. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 2792539..b595ddf 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -75,6 +75,7 @@ const guide = [ text: 'Transports', items: [ { text: 'SMTP', link: '/transports/smtp' }, + { text: 'Sendmail', link: '/transports/sendmail' }, { text: 'Amazon SES', link: '/transports/ses' }, { text: 'Resend', link: '/transports/resend' }, { text: 'SendGrid', link: '/transports/sendgrid' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 1c7acd4..7fb86b8 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -43,7 +43,8 @@ The message model and the transport contract, ported from `@upyo/core`. | --- | --- | --- | | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | -| `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails` | [SMTP](/transports/smtp) | +| `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | +| `@alexify/mailer/sendmail` | `SendmailTransport` | [Sendmail](/transports/sendmail) | | `@alexify/mailer/ses` | `SesTransport` | [Amazon SES](/transports/ses) | | `@alexify/mailer/resend` | `ResendTransport` | [Resend](/transports/resend) | | `@alexify/mailer/sendgrid` | `SendGridTransport`, `SendGridApiError` | [SendGrid](/transports/sendgrid) | diff --git a/docs/transports/sendmail.md b/docs/transports/sendmail.md new file mode 100644 index 0000000..87831dd --- /dev/null +++ b/docs/transports/sendmail.md @@ -0,0 +1,87 @@ +# Sendmail transport + +The sendmail transport hands each message to the `sendmail` command of the +system, the one Postfix, Exim, OpenSMTPD, msmtp and sendmail itself install, which +queues it and delivers it with the configuration of the machine. It comes from +nodemailer's sendmail transport, with the fixes listed below. + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { SendmailTransport } = require('@alexify/mailer/sendmail'); + +const transport = new SendmailTransport({ path: '/usr/sbin/sendmail' }); + +const receipt = await transport.send( + createMessage({ + from: 'app@example.com', + to: 'user@example.com', + bcc: 'archive@example.com', + subject: 'Welcome', + content: { text: 'Hello!' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); +``` + +Each message runs one process, ` -i -f -- `, and goes +to it on stdin, composed by *@alexify/mailer/mime*: the same MIME, DKIM signature, +encodings and IDN addresses as the SMTP transport. `-i` keeps a line of a single +dot in the message, and `--` keeps an address from being read as an option; an +address that starts with `-` is refused anyway. Blind recipients go on the +command line only: the message has no Bcc field for sendmail to strip. + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `path` | `'sendmail'` | The command, a path or a name on PATH. | +| `args` | | Arguments in place of `-f `: ` -i -- `. | +| `newline` | `'lf'` | `'lf'`, which every sendmail takes, or `'crlf'`. | +| `spawn` | `child_process.spawn` | Starts the process: pass your own to run sendmail in a container or as another user. | +| `dkim` | | A DKIM configuration, as for [SMTP](/transports/smtp#dkim-signing). | +| `generateId` | | Makes the left half of a Message-ID, and the id of a raw receipt. | +| `logger` | | A pino-like logger: `debug` when a process starts and exits. | + +## Receipts + +A send succeeds when sendmail exits with code 0, and the receipt carries the +Message-ID of the message. Any other end is a failed receipt, with what sendmail +wrote to stderr (its first 4 KiB, addresses replaced by `[REDACTED]`): + +| Exit | Category | Retryable | +| --- | --- | --- | +| 75 `EX_TEMPFAIL`, 69 `EX_UNAVAILABLE` | `service-unavailable` | yes | +| 67 `EX_NOUSER`, 68 `EX_NOHOST` | `rejected` | no | +| 65 `EX_DATAERR` | `validation` | no | +| 64, 77, 78, 126, 127, command not found | `configuration` | no | +| any other code | `server-error` | no | +| killed by a signal | `server-error` | before it had the whole message; after, `deliveryUnknown` | + +An abort of the `signal` kills the process. A message whose content cannot be +read (an attachment that fails) kills it with SIGINT, so that no part of it is +delivered. `scheduledAt` is refused, as sendmail cannot schedule. + +## Raw messages and verification + +`sendRaw()` pipes serialized MIME as it is but for its line endings, to the +recipients of its envelope, `-f <>` for a null sender. `verify()` checks that the +command exists and is executable, at its path or on PATH; with `spawn` passed in, +which decides what runs, it checks nothing. + +## What changed from nodemailer + +- A process killed by a signal was a success there (`code === null`), and the code + of `close` was handed on as an error value. Here only an exit with 0 succeeds. +- stderr was never read; here it is, bounded and redacted, in the failure. +- The recipients followed the options with no `--`. +- The message kept its Bcc field for sendmail to strip, which depends on the MTA. diff --git a/package.json b/package.json index dbd1795..42fc702 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,10 @@ "types": "./src/smtp/index.d.ts", "default": "./src/smtp/index.js" }, + "./sendmail": { + "types": "./src/sendmail/index.d.ts", + "default": "./src/sendmail/index.js" + }, "./jmap": { "types": "./src/jmap/index.d.ts", "default": "./src/jmap/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 44337c9..d9cd1dc 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,6 +3,7 @@ "./mime": 7.5, "./attachments": 4, "./smtp": 22.5, + "./sendmail": 10.5, "./jmap": 12.5, "./ses": 6.5, "./resend": 6, diff --git a/src/sendmail/config.js b/src/sendmail/config.js new file mode 100644 index 0000000..7eafb85 --- /dev/null +++ b/src/sendmail/config.js @@ -0,0 +1,57 @@ +// Ported from nodemailer v10.0.16 (src/sendmail-transport/index.ts): the +// options of the sendmail transport, `path`, `args` and `newline`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: checked when the transport is made, with `spawn` passed +// in, LF line endings by default, and DKIM from ./mime. + +const { checkDkimConfig } = require('../mime/dkim/types.js'); + +const NEWLINES = new Set(['lf', 'crlf']); + +const throwTypeError = (message) => { + throw new TypeError(message); +}; + +// An argument goes to the process as it is; only a NUL cannot. +const isArgument = (value) => typeof value === 'string' && !value.includes('\0'); + +const checkArgs = (args) => { + if (args === undefined) return undefined; + if (!Array.isArray(args)) throwTypeError('args must be an array of strings.'); + const checked = []; + for (let i = 0; i < args.length; i++) { + if (!isArgument(args[i])) throwTypeError(`Invalid sendmail argument at index ${i}.`); + checked.push(args[i]); + } + return checked; +}; + +/** + * Resolves the options of the sendmail transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createSendmailConfig = (config = {}) => { + const path = config.path ?? 'sendmail'; + if (!isArgument(path) || path === '') throwTypeError('path must name the sendmail command.'); + const newline = config.newline ?? 'lf'; + if (!NEWLINES.has(newline)) throwTypeError(`newline must be 'lf' or 'crlf': ${newline}`); + if (config.spawn !== undefined && typeof config.spawn !== 'function') { + throwTypeError('spawn must be a function, such as child_process.spawn.'); + } + checkDkimConfig(config.dkim); + return { + path, + args: checkArgs(config.args), + newline, + spawn: config.spawn, + dkim: config.dkim, + generateId: config.generateId, + logger: config.logger, + }; +}; + +module.exports = { createSendmailConfig }; diff --git a/src/sendmail/index.d.ts b/src/sendmail/index.d.ts new file mode 100644 index 0000000..b2c2c7c --- /dev/null +++ b/src/sendmail/index.d.ts @@ -0,0 +1,95 @@ +// Types ported from nodemailer v10.0.16 (src/sendmail-transport) +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman + +import type { + IdGenerator, + Logger, + Message, + RawMessage, + RawTransport, + Receipt, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; +import type { DkimConfig } from '../mime/index.js'; + +/** + * A child process as `spawn` gives it: the pipes the transport writes the + * message to and reads stderr from, its events and `kill`. + */ +export interface SendmailProcess { + readonly stdin: { + write(chunk: Uint8Array): boolean; + end(chunk?: Uint8Array): unknown; + readonly destroyed: boolean; + on(event: string, listener: (...args: any[]) => void): unknown; + once(event: string, listener: (...args: any[]) => void): unknown; + } | null; + readonly stderr: { on(event: 'data', listener: (chunk: Uint8Array) => void): unknown } | null; + once(event: 'error', listener: (error: Error & { code?: string }) => void): unknown; + once(event: 'close', listener: (code: number | null, signal: string | null) => void): unknown; + kill(signal?: string): boolean; +} + +/** + * The options of {@link SendmailTransport}. + */ +export interface SendmailConfig { + /** + * The command: a path, or a name looked up on PATH. + * @default "sendmail" + */ + readonly path?: string; + /** + * Arguments in place of `-f `: the transport runs + * ` -i -- `. By default it passes the sender of + * the envelope with `-f` (`-f <>` for none). + */ + readonly args?: readonly string[]; + /** + * The line endings the message is written with: `lf`, which every sendmail + * takes, or `crlf`. + * @default "lf" + */ + readonly newline?: 'lf' | 'crlf'; + /** + * Starts the process, `child_process.spawn` by default: pass your own to run + * sendmail in a container or under another user. + */ + readonly spawn?: ( + command: string, + args: readonly string[], + options: { stdio: ['pipe', 'ignore', 'pipe']; windowsHide: boolean }, + ) => SendmailProcess; + /** Signs each message, as `./mime` and `./smtp` do. */ + readonly dkim?: DkimConfig; + /** Makes the left half of a Message-ID, and the id of a raw receipt. */ + readonly generateId?: IdGenerator; + /** A pino-like logger for the start and the exit of each process. */ + readonly logger?: Logger; +} + +/** + * A transport that pipes each message to the sendmail command of the system + * (Postfix, Exim, sendmail, msmtp, …), one process per message. + * + * @example + * ```js + * const { SendmailTransport } = require('@alexify/mailer/sendmail'); + * const transport = new SendmailTransport({ path: '/usr/sbin/sendmail' }); + * const receipt = await transport.send(message); + * ``` + */ +export declare class SendmailTransport + implements RawTransport<'sendmail'>, VerifiableTransport<'sendmail'> +{ + readonly id = 'sendmail'; + constructor(config?: SendmailConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + verify(options?: TransportOptions): Promise; +} diff --git a/src/sendmail/index.js b/src/sendmail/index.js new file mode 100644 index 0000000..9f0f981 --- /dev/null +++ b/src/sendmail/index.js @@ -0,0 +1,7 @@ +// Ported from nodemailer v10.0.16 (src/sendmail-transport/index.ts): the +// entry of the sendmail transport. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman + +const { SendmailTransport } = require('./sendmail-transport.js'); + +module.exports = { SendmailTransport }; diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js new file mode 100644 index 0000000..eb8473a --- /dev/null +++ b/src/sendmail/sendmail-transport.js @@ -0,0 +1,334 @@ +// Ported from nodemailer v10.0.16 (src/sendmail-transport/index.ts): a +// transport that pipes each message to the sendmail command of the system. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman +// Changed in the port: the message is composed by ./mime (DKIM, IDN, the +// transfer encodings) without a Bcc field, its blind recipients going on the +// command line only, where nodemailer kept the field for sendmail to strip; +// `--` ends the options before the recipients; success is an exit with code 0 +// alone, where nodemailer took a process killed by a signal (code null) for +// success and handed the code of `close` on as an error; what sendmail writes +// to stderr is read, bounded, into the failure; line endings are LF by +// default; an abort kills the process; and `spawn` is passed in. + +const childProcess = require('node:child_process'); +const { once } = require('node:events'); +const fsp = require('node:fs/promises'); +const nodePath = require('node:path'); +const { generateId } = require('../core/id.js'); +const { parseMessageId, pinMessageId } = require('../core/message-id.js'); +const { + createRawMessagePlan, + iterateRawMessage, + RawMessageValidationError, +} = require('../core/raw-message.js'); +const { createFailedReceipt } = require('../core/receipt.js'); +const { createLog } = require('../internal/log.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { redactAddresses } = require('../internal/redact-addresses.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); +const { composeMessage } = require('../mime/index.js'); +const { createSendmailConfig } = require('./config.js'); + +const MAX_STDERR_BYTES = 4096; +const CR = 0x0d; + +// sysexits.h: the category of a failure by the exit status of sendmail, and +// whether a later attempt may pass. A status sendmail gives says it did not +// take the message. +const EXIT_STATUSES = new Map([ + [64, { category: 'configuration', retryable: false }], + [65, { category: 'validation', retryable: false }], + [67, { category: 'rejected', retryable: false }], + [68, { category: 'rejected', retryable: false }], + [69, { category: 'service-unavailable', retryable: true }], + [75, { category: 'service-unavailable', retryable: true }], + [77, { category: 'configuration', retryable: false }], + [78, { category: 'configuration', retryable: false }], + [126, { category: 'configuration', retryable: false }], + [127, { category: 'configuration', retryable: false }], +]); + +const isCallerAbort = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +const failureOf = (error) => { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof RawMessageValidationError) { + return createFailedReceipt(message, { provider: 'sendmail', category: 'validation' }); + } + return createFailedReceipt(message, { provider: 'sendmail' }); +}; + +// An address that would read as an option, as nodemailer refuses it; `--` +// keeps the recipients from being read as one too. +const checkArgument = (address) => { + if (address.startsWith('-') || address.startsWith('"-')) { + throw new TypeError('A sendmail envelope address must not start with "-".'); + } + return address; +}; + +const argsOf = (config, envelope) => { + const args = ['-i']; + if (config.args === undefined) { + args.push('-f', envelope.from == null ? '<>' : checkArgument(envelope.from)); + } else { + for (let i = 0; i < config.args.length; i++) args.push(config.args[i]); + } + args.push('--'); + for (let i = 0; i < envelope.to.length; i++) args.push(checkArgument(envelope.to[i])); + return args; +}; + +// The message as LF lines. Every CR of what is written is the CR of a CRLF: +// ./mime writes CRLF alone, and a raw message with a bare CR is refused as it +// is read, so dropping each CR leaves the LF of each line. +const withoutCr = (chunk) => { + let crs = 0; + for (let i = 0; i < chunk.length; i++) if (chunk[i] === CR) crs++; + if (crs === 0) return chunk; + const out = new Uint8Array(chunk.length - crs); + let length = 0; + for (let i = 0; i < chunk.length; i++) if (chunk[i] !== CR) out[length++] = chunk[i]; + return out; +}; + +// Waits for room in the pipe, or for it to close with the process. +const drained = async (stdin) => { + const settled = new AbortController(); + try { + await Promise.race([ + once(stdin, 'drain', { signal: settled.signal }), + once(stdin, 'close', { signal: settled.signal }), + ]); + } finally { + settled.abort(); + } +}; + +// Writes the message to the process; an error of the content is thrown, one of +// the pipe (the process went away) left to its exit status. +const writeMessage = async (stdin, chunks, newline) => { + for await (const chunk of chunks) { + if (stdin.destroyed) return false; + if (!stdin.write(newline === 'lf' ? withoutCr(chunk) : chunk)) await drained(stdin); + } + if (stdin.destroyed) return false; + stdin.end(); + return true; +}; + +const collectStderr = (stream) => { + const chunks = []; + let size = 0; + stream?.on('data', (chunk) => { + if (size >= MAX_STDERR_BYTES) return; + const part = chunk.subarray(0, MAX_STDERR_BYTES - size); + chunks.push(part); + size += part.length; + }); + return () => Buffer.concat(chunks, size).toString('utf8').trim(); +}; + +/** + * Runs sendmail on a message and waits for it to exit. + * + * @returns `{ code, signal, stderr, written, error, contentError }`. + */ +const run = (child, chunks, newline, abort) => + new Promise((resolve, reject) => { + const outcome = { written: false }; + const stderrOf = collectStderr(child.stderr); + child.stdin?.on('error', () => {}); + const onAbort = () => { + child.kill(); + reject(abort.reason); + }; + abort?.addEventListener('abort', onAbort, { once: true }); + child.once('error', (error) => { + abort?.removeEventListener('abort', onAbort); + resolve({ ...outcome, error }); + }); + child.once('close', (code, killSignal) => { + abort?.removeEventListener('abort', onAbort); + resolve({ ...outcome, code, signal: killSignal, stderr: stderrOf() }); + }); + writeMessage(child.stdin, chunks, newline).then( + (written) => { + outcome.written = written; + return written; + }, + (error) => { + // The message could not be read: sendmail must not deliver a part. + outcome.contentError = error; + child.kill('SIGINT'); + }, + ); + }); + +const receiptOf = (result, path, messageId) => { + if (result.error !== undefined) { + const missing = result.error.code === 'ENOENT'; + const text = missing ? `sendmail was not found: ${path}` : result.error.message; + return createFailedReceipt(text, { + provider: 'sendmail', + category: missing ? 'configuration' : undefined, + retryable: missing ? false : undefined, + }); + } + if (result.contentError !== undefined) return failureOf(result.contentError); + if (result.code === 0 && result.signal == null) { + return { successful: true, messageId, provider: 'sendmail' }; + } + const said = result.stderr === '' ? '' : `: ${redactAddresses(result.stderr)}`; + if (result.code == null) { + // Killed once it had the whole message, it may have queued it. + return createFailedReceipt(`sendmail was killed by ${result.signal}${said}`, { + provider: 'sendmail', + category: 'server-error', + retryable: !result.written, + deliveryUnknown: result.written, + }); + } + const status = EXIT_STATUSES.get(result.code) ?? { category: 'server-error', retryable: false }; + return createFailedReceipt(`sendmail exited with code ${result.code}${said}`, { + provider: 'sendmail', + ...status, + }); +}; + +// The command as sendmail would be run: a path, or a name on PATH. +const findCommand = async (command) => { + const candidates = []; + if (command.includes('/') || command.includes('\\')) candidates.push(command); + else { + for (const directory of (process.env.PATH ?? '').split(nodePath.delimiter)) { + if (directory !== '') candidates.push(nodePath.join(directory, command)); + } + } + for (const candidate of candidates) { + try { + await fsp.access(candidate, fsp.constants.X_OK); + return candidate; + } catch { + // The next one. + } + } + throw new Error(`sendmail was not found: ${command}`); +}; + +/** + * A transport that pipes each message to the sendmail command of the system + * (Postfix, Exim, sendmail, msmtp, …), one process per message. + */ +class SendmailTransport { + id = 'sendmail'; + #log; + + constructor(config = {}) { + this.config = createSendmailConfig(config); + this.#log = createLog(this.config.logger, { transport: 'sendmail' }); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Composes a message and pipes it to sendmail. + * + * @param message The message. + * @param options `signal`, which kills the process. + * @returns A receipt with the Message-ID of the message. + * @throws The reason of `signal` if it aborts. + */ + async send(message, options) { + const signal = options?.signal; + signal?.throwIfAborted(); + let composed; + let pinned; + try { + refuseScheduledAt(message, 'sendmail'); + pinned = pinMessageId(message, this.config.generateId); + composed = await composeMessage(pinned, { + signal, + dkim: this.config.dkim, + generateId: this.config.generateId, + logger: this.config.logger, + }); + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + return failureOf(error); + } + const messageId = + pinned.messageId ?? parseMessageId(pinned.headers.get('Message-ID') ?? '') ?? ''; + return this.deliver(composed.envelope, composed.content(signal), messageId, signal); + } + + /** + * Sends each message in turn, a process each. + */ + async *sendMany(messages, options) { + for await (const message of messages) yield await this.send(message, options); + } + + /** + * Pipes serialized MIME to sendmail unchanged but for its line endings, to + * the recipients of its envelope. + */ + async sendRaw(message, options) { + const signal = options?.signal; + if (message?.content instanceof Promise) message.content.catch(() => {}); + signal?.throwIfAborted(); + let plan; + try { + plan = createRawMessagePlan(message); + } catch (error) { + return failureOf(error); + } + const messageId = generateId('receipt-id', this.config.generateId); + return this.deliver(plan.envelope, iterateRawMessage(plan, { signal }), messageId, signal); + } + + /** + * Checks that the command can be run: `path` itself, or a file of that name + * on PATH that is executable. With `spawn` passed in, which decides what + * runs, it checks nothing. + * + * @throws {Error} If no such command is found. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (this.config.spawn === undefined) await findCommand(this.config.path); + } + + async deliver(envelope, chunks, messageId, signal) { + let args; + let child; + try { + args = argsOf(this.config, envelope); + const spawn = this.config.spawn ?? childProcess.spawn; + child = spawn(this.config.path, args, { + stdio: ['pipe', 'ignore', 'pipe'], + windowsHide: true, + }); + } catch (error) { + return failureOf(error); + } + this.#log.write('debug', () => ({ recipients: envelope.to.length }), 'sendmail started'); + const result = await run(child, chunks, this.config.newline, signal); + const receipt = receiptOf(result, this.config.path, messageId); + this.#log.write( + 'debug', + () => ({ code: result.code, signal: result.signal, successful: receipt.successful }), + 'sendmail exited', + ); + return receipt; + } +} + +module.exports = { SendmailTransport }; diff --git a/tests/types/sendmail.test-d.ts b/tests/types/sendmail.test-d.ts new file mode 100644 index 0000000..01d3c16 --- /dev/null +++ b/tests/types/sendmail.test-d.ts @@ -0,0 +1,33 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type VerifiableTransport, +} from '../../index.js'; +import { SendmailTransport } from '../../src/sendmail/index.js'; + +const transport = new SendmailTransport({ + path: '/usr/sbin/sendmail', + args: ['-oi'], + newline: 'crlf', + generateId: () => 'id', +}); +expectType<'sendmail'>(transport.id); +expectAssignable>(transport); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>( + transport.sendRaw({ envelope: { from: null, to: ['b@example.com'] }, content: new Uint8Array() }), +); +expectType>(transport.verify()); +new SendmailTransport(); + +expectError(new SendmailTransport({ newline: 'cr' })); +expectError(new SendmailTransport({ args: '-oi' })); diff --git a/tests/unit/graph.test.js b/tests/unit/graph.test.js index 1d40cdf..8a9d2a1 100644 --- a/tests/unit/graph.test.js +++ b/tests/unit/graph.test.js @@ -91,6 +91,10 @@ const SUBPATH_EDGES = [ // The transport checks the DKIM keys when it is made. 'smtp/smtp-transport.js -> mime/dkim/sign.js', 'smtp/smtp-transport.js -> mime/dkim/types.js', + // ./sendmail composes and signs as ./mime does, and checks its DKIM + // configuration when it is made. + 'sendmail/config.js -> mime/dkim/types.js', + 'sendmail/sendmail-transport.js -> mime/index.js', ]; const subpathOf = (name) => { diff --git a/tests/unit/sendmail.test.js b/tests/unit/sendmail.test.js new file mode 100644 index 0000000..a34c255 --- /dev/null +++ b/tests/unit/sendmail.test.js @@ -0,0 +1,388 @@ +// Not ported: tests of ./sendmail, ported from nodemailer's sendmail +// transport, with a fake `spawn` and with a real process that stands in for +// sendmail. +const { test } = require('node:test'); +const assert = require('node:assert'); +const childProcess = require('node:child_process'); +const { EventEmitter } = require('node:events'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const util = require('node:util'); +const { PassThrough } = require('node:stream'); +const { createMessage } = require('../../index.js'); +const { SendmailTransport } = require('../../src/sendmail/index.js'); +const { + TEST_DKIM_PRIVATE_KEY, + TEST_DKIM_PUBLIC_KEY, +} = require('../helpers/mime/dkim-test-keys.js'); +const { verifyDkim } = require('../helpers/mime/verify-dkim.js'); + +const message = (fields = {}) => + createMessage({ + from: 'alex@example.com', + to: ['bob@example.com', 'carol@example.com'], + bcc: 'blind@example.com', + subject: 'Hello', + content: { text: 'line one\r\nline two\r\n.\r\nend' }, + ...fields, + }); + +// A fake child: what is written to stdin is kept, and `exit` decides how the +// process ends once its input ends. +const fakeSpawn = (exit = { code: 0 }) => { + const calls = []; + const spawn = (command, args, options) => { + const child = new EventEmitter(); + const chunks = []; + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.killed = []; + child.kill = (signal = 'SIGTERM') => { + child.killed.push(signal); + setImmediate(() => child.emit('close', null, signal)); + return true; + }; + child.stdin.on('data', (chunk) => chunks.push(chunk)); + child.stdin.on('end', () => { + if (exit.stderr !== undefined) child.stderr.write(exit.stderr); + if (exit.hang) return; + setImmediate(() => child.emit('close', exit.code ?? null, exit.signal ?? null)); + }); + calls.push({ command, args, options, child, input: () => Buffer.concat(chunks).toString() }); + return child; + }; + return { spawn, calls }; +}; + +test('Sendmail: pipes the composed message with -i, -f and -- before the recipients', async () => { + const { spawn, calls } = fakeSpawn(); + const transport = new SendmailTransport({ path: '/usr/sbin/sendmail', spawn }); + const receipt = await transport.send(message()); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(receipt.provider, 'sendmail'); + const [call] = calls; + assert.strictEqual(call.command, '/usr/sbin/sendmail'); + assert.deepStrictEqual(call.args, [ + '-i', + '-f', + 'alex@example.com', + '--', + 'bob@example.com', + 'carol@example.com', + 'blind@example.com', + ]); + assert.deepStrictEqual(call.options.stdio, ['pipe', 'ignore', 'pipe']); + const input = call.input(); + // LF line endings, no Bcc field, the receipt's Message-ID in the message. + assert.ok(!input.includes('\r\n')); + assert.ok(input.includes('\nline one\nline two\n.\nend')); + assert.ok(!/^Bcc:/im.test(input)); + assert.ok(input.includes(`Message-ID: <${receipt.messageId}>`)); +}); + +test('Sendmail: crlf keeps CRLF, and args replace -f', async () => { + const { spawn, calls } = fakeSpawn(); + const transport = new SendmailTransport({ newline: 'crlf', args: ['-oi', '-t'], spawn }); + assert.strictEqual((await transport.send(message({ bcc: undefined }))).successful, true); + assert.deepStrictEqual(calls[0].args, [ + '-i', + '-oi', + '-t', + '--', + 'bob@example.com', + 'carol@example.com', + ]); + assert.strictEqual(calls[0].command, 'sendmail'); + assert.ok(calls[0].input().includes('line one\r\nline two\r\n')); +}); + +test('Sendmail: sendRaw pipes the bytes, CRLF as LF across chunks, -f <> for no sender', async () => { + const { spawn, calls } = fakeSpawn(); + const transport = new SendmailTransport({ spawn, generateId: () => 'raw-1' }); + const encoder = new TextEncoder(); + const receipt = await transport.sendRaw({ + envelope: { from: null, to: ['bob@example.com'] }, + content: async function* () { + yield encoder.encode('Subject: Raw\r'); + yield encoder.encode('\nTo: bob@example.com\r\n\r\nbody\r'); + yield encoder.encode('\n'); + }, + }); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(receipt.messageId, 'raw-1'); + assert.deepStrictEqual(calls[0].args, ['-i', '-f', '<>', '--', 'bob@example.com']); + assert.strictEqual(calls[0].input(), 'Subject: Raw\nTo: bob@example.com\n\nbody\n'); + const invalid = await transport.sendRaw({ + envelope: { from: null, to: [] }, + content: new Uint8Array(), + }); + assert.strictEqual(invalid.successful, false); + assert.strictEqual(invalid.errors[0].category, 'validation'); +}); + +test('Sendmail: an address that reads as an option never reaches the command line', async () => { + const { spawn, calls } = fakeSpawn(); + const transport = new SendmailTransport({ spawn }); + for (const to of ['-oQ/tmp@example.com', '"-bv"@example.com']) { + const receipt = await transport.sendRaw({ + envelope: { from: 'a@example.com', to: [to] }, + content: new TextEncoder().encode('Subject: x\r\n\r\nx\r\n'), + }); + assert.strictEqual(receipt.successful, false, to); + assert.match(receipt.errorMessages.join(), /must not start with "-"/); + } + const sender = await transport.send(message({ from: '-f@example.com' })); + assert.match(sender.errorMessages.join(), /must not start with "-"/); + assert.strictEqual(calls.length, 0); +}); + +test('Sendmail: the exit status says what failed, with stderr, addresses redacted', async () => { + const cases = [ + [{ code: 75, stderr: 'queue full' }, 'service-unavailable', true], + [{ code: 69 }, 'service-unavailable', true], + [{ code: 67, stderr: 'bob@example.com... User unknown' }, 'rejected', false], + [{ code: 68 }, 'rejected', false], + [{ code: 65 }, 'validation', false], + [{ code: 64 }, 'configuration', false], + [{ code: 78 }, 'configuration', false], + [{ code: 127 }, 'configuration', false], + [{ code: 1, stderr: 'odd' }, 'server-error', false], + ]; + for (const [exit, category, retryable] of cases) { + const { spawn } = fakeSpawn(exit); + const receipt = await new SendmailTransport({ spawn }).send(message()); + assert.strictEqual(receipt.successful, false, `code ${exit.code}`); + assert.strictEqual(receipt.errors[0].category, category, `code ${exit.code}`); + assert.strictEqual(receipt.retryable, retryable, `code ${exit.code}`); + assert.match(receipt.errorMessages[0], new RegExp(`exited with code ${exit.code}`)); + } + const { spawn } = fakeSpawn({ + code: 67, + stderr: `bob@example.com... User unknown${'x'.repeat(9000)}`, + }); + const receipt = await new SendmailTransport({ spawn }).send(message()); + assert.ok(!receipt.errorMessages[0].includes('bob@example.com')); + assert.ok(receipt.errorMessages[0].includes('User unknown')); + assert.ok(receipt.errorMessages[0].length < 4200); +}); + +test('Sendmail: a full pipe is waited for, and a process that leaves early ends the write', async () => { + const big = message({ content: { text: 'x'.repeat(200_000) } }); + const chunks = []; + const slow = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough({ highWaterMark: 16 }); + child.stderr = new PassThrough(); + child.kill = () => true; + // A reader that takes a chunk at a time, later. + const read = () => { + const chunk = child.stdin.read(); + if (chunk !== null) chunks.push(chunk); + }; + child.stdin.on('readable', () => setImmediate(read)); + child.stdin.on('end', () => setImmediate(() => child.emit('close', 0, null))); + return child; + }; + const receipt = await new SendmailTransport({ spawn: slow }).send(big); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.ok(Buffer.concat(chunks).toString().includes('xxxx')); + const leaving = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough({ highWaterMark: 16 }); + child.stderr = new PassThrough(); + child.kill = () => true; + child.stdin.once('data', () => { + child.stdin.destroy(); + child.emit('close', 75, null); + }); + return child; + }; + const early = await new SendmailTransport({ spawn: leaving }).send(big); + assert.strictEqual(early.successful, false); + assert.strictEqual(early.retryable, true); +}); + +test('Sendmail: a process killed by a signal is a failure, unknown once it had the message', async () => { + // nodemailer took `code === null` for success. + const { spawn } = fakeSpawn({ code: null, signal: 'SIGKILL' }); + const receipt = await new SendmailTransport({ spawn }).send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.deliveryUnknown, true); + assert.strictEqual(receipt.retryable, false); + assert.match(receipt.errorMessages[0], /killed by SIGKILL/); +}); + +test('Sendmail: a command that is missing, or a spawn that throws, fails the send', async () => { + const missing = (command, args) => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = null; + child.kill = () => true; + setImmediate(() => + child.emit('error', Object.assign(new Error('spawn ENOENT'), { code: 'ENOENT' })), + ); + return child; + }; + const receipt = await new SendmailTransport({ path: '/nowhere/sendmail', spawn: missing }).send( + message(), + ); + assert.strictEqual(receipt.errorMessages[0], 'sendmail was not found: /nowhere/sendmail'); + assert.strictEqual(receipt.errors[0].category, 'configuration'); + assert.strictEqual(receipt.retryable, false); + const other = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = null; + child.kill = () => true; + setImmediate(() => child.emit('error', new Error('spawn EACCES'))); + return child; + }; + assert.match( + (await new SendmailTransport({ spawn: other }).send(message())).errorMessages[0], + /EACCES/, + ); + const throwing = () => { + throw new Error('cannot spawn'); + }; + assert.match( + (await new SendmailTransport({ spawn: throwing }).send(message())).errorMessages[0], + /cannot spawn/, + ); +}); + +test('Sendmail: an abort kills the process, and a content error keeps it from delivering', async () => { + const { spawn, calls } = fakeSpawn({ hang: true }); + const controller = new AbortController(); + const sending = new SendmailTransport({ spawn }).send(message(), { signal: controller.signal }); + setTimeout(() => controller.abort(new Error('stopped')), 10); + await assert.rejects(sending, /stopped/); + assert.deepStrictEqual(calls[0].child.killed, ['SIGTERM']); + const broken = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Broken', + content: { text: 'x' }, + attachments: { + filename: 'a.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: async function* () { + yield new Uint8Array(10); + throw new Error('disk gone'); + }, + }, + }); + const second = fakeSpawn({ hang: true }); + const receipt = await new SendmailTransport({ spawn: second.spawn }).send(broken); + assert.strictEqual(receipt.successful, false); + assert.match(receipt.errorMessages[0], /disk gone/); + assert.deepStrictEqual(second.calls[0].child.killed, ['SIGINT']); + const aborted = new AbortController(); + aborted.abort(new Error('early')); + await assert.rejects( + new SendmailTransport({ spawn }).send(message(), { signal: aborted.signal }), + /early/, + ); +}); + +test('Sendmail: a scheduled message is refused, sendMany sends in turn, DKIM signs', async () => { + const { spawn, calls } = fakeSpawn(); + const transport = new SendmailTransport({ + spawn, + dkim: { + signatures: [ + { signingDomain: 'example.com', selector: 'test', privateKey: TEST_DKIM_PRIVATE_KEY }, + ], + }, + }); + const scheduled = await transport.send(message({ scheduledAt: new Date() })); + assert.match(scheduled.errorMessages[0], /Invalid scheduledAt: sendmail cannot schedule/); + const receipts = []; + for await (const receipt of transport.sendMany([message(), message()])) receipts.push(receipt); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, true], + ); + assert.strictEqual(calls.length, 2); + verifyDkim(calls[0].input().replace(/\n/g, '\r\n'), TEST_DKIM_PUBLIC_KEY); + // The key does not show in a dump of the transport. + assert.ok(!JSON.stringify(transport).includes('PRIVATE KEY')); + assert.ok(!util.inspect(transport, { depth: 10 }).includes('PRIVATE KEY')); +}); + +test('Sendmail: a real process gets the message on stdin and the arguments', async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-sendmail-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const script = path.join(dir, 'fake-sendmail.js'); + const output = path.join(dir, 'out.json'); + fs.writeFileSync( + script, + `const chunks = []; +process.stdin.on('data', (c) => chunks.push(c)); +process.stdin.on('end', () => { + require('node:fs').writeFileSync(${JSON.stringify(output)}, JSON.stringify({ + args: process.argv.slice(2), body: Buffer.concat(chunks).toString() })); + process.stderr.write('Mail queued'); + process.exit(Number(process.env.FAKE_EXIT ?? 0)); +});`, + ); + const spawn = (command, args, options) => + childProcess.spawn(process.execPath, [script, ...args], options); + const transport = new SendmailTransport({ spawn }); + const receipt = await transport.send(message({ attachments: [] })); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + const seen = JSON.parse(fs.readFileSync(output, 'utf8')); + assert.deepStrictEqual(seen.args.slice(0, 4), ['-i', '-f', 'alex@example.com', '--']); + assert.ok(seen.body.includes('Subject: Hello\n')); + process.env.FAKE_EXIT = '75'; + t.after(() => delete process.env.FAKE_EXIT); + const temporary = await transport.send(message()); + assert.strictEqual(temporary.retryable, true); + assert.match(temporary.errorMessages[0], /code 75: Mail queued/); +}); + +test('Sendmail: verify finds the command on PATH or at its path', async (t) => { + await new SendmailTransport({ path: process.execPath }).verify(); + await assert.rejects(new SendmailTransport({ path: '/nowhere/sendmail' }).verify(), /not found/); + if (process.platform !== 'win32') { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-path-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + fs.writeFileSync(path.join(dir, 'my-sendmail'), '#!/bin/sh\n', { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'not-executable'), '', { mode: 0o644 }); + const saved = process.env.PATH; + process.env.PATH = `${dir}${path.delimiter}${saved}`; + t.after(() => { + process.env.PATH = saved; + }); + await new SendmailTransport({ path: 'my-sendmail' }).verify(); + await assert.rejects(new SendmailTransport({ path: 'not-executable' }).verify(), /not found/); + } + // With spawn passed in, what runs is the caller's to know. + await new SendmailTransport({ path: 'whatever', spawn: fakeSpawn().spawn }).verify(); + const aborted = new AbortController(); + aborted.abort(new Error('stop')); + await assert.rejects(new SendmailTransport().verify({ signal: aborted.signal }), /stop/); +}); + +test('Sendmail: the options are checked when the transport is made', () => { + const invalid = [ + [{ path: '' }, /path must name/], + [{ path: 'a\0b' }, /path must name/], + [{ newline: 'cr' }, /newline must be/], + [{ args: '-t' }, /args must be an array/], + [{ args: ['-t', 'a\0'] }, /argument at index 1/], + [{ spawn: 'sh' }, /spawn must be a function/], + [ + { + dkim: { signatures: [{ signingDomain: 'example.com', selector: 's;x', privateKey: 'k' }] }, + }, + /selector/, + ], + ]; + for (const [config, error] of invalid) { + assert.throws(() => new SendmailTransport(config), error, JSON.stringify(config)); + } + assert.strictEqual(new SendmailTransport().id, 'sendmail'); +}); From 2a8d19fe022f4fc16ac7dc1c6d8e52b54b7ca4c0 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:09:08 +0300 Subject: [PATCH 013/143] fix(mime): refuse a DKIM domain or selector with a tab or line break checkDkimConfig read a name through domainToASCII, a WHATWG host parser that removes tabs, CR and LF before it reads the name, so "exa\r\nmple.com" passed as the valid "example.com" while the name, line break and all, went into the DKIM-Signature field as given, breaking the header block. A name with a control character, whitespace, "/", "\", "?", "#" or "%" is now refused before it reaches the parser, for the signing domain and the selector. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 +++- src/mime/dkim/types.js | 10 +++++++++- tests/unit/mime-dkim.test.js | 7 +++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a490cf5..f4a9da9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -341,7 +341,9 @@ fails without it; ported tests that pinned the old behavior say "Changed after t 62. DKIM: `signingDomain`, `selector`, the names in `headerFields`, the algorithm and the canonicalization went into the DKIM-Signature field unchecked, so a domain with a line break added a header field such as Bcc to the message, and a `;` ended a tag early. - `checkDkimConfig` refuses them, from the SMTP constructor and at every composition. + `checkDkimConfig` refuses them, from the SMTP constructor and at every composition. A name + with a tab, CR or LF, which the WHATWG parser behind `domainToASCII` removes before it reads + the name, is refused before it gets there; the first version of this fix let one through. 63. DKIM (known bug 6): a PEM was stripped of its armor and imported as PKCS#8, so a PKCS#1 key (`BEGIN RSA PRIVATE KEY`) failed. `createPrivateKey` of node:crypto reads either and the key reaches Web Crypto as PKCS#8; the import error keeps its cause. diff --git a/src/mime/dkim/types.js b/src/mime/dkim/types.js index fa88eb3..a982e5c 100644 --- a/src/mime/dkim/types.js +++ b/src/mime/dkim/types.js @@ -34,10 +34,18 @@ const CANONICALIZATIONS = new Set([ 'simple/simple', ]); +// domainToASCII is a WHATWG host parser: it removes tabs, CR and LF before +// it reads a name, cuts it at "/", "?" or "#" and percent-decodes it, so +// "exa\r\nmple.com" came back as the valid "example.com" while the name, +// line break and all, went into the DKIM-Signature field as given. +// oxlint-disable-next-line no-control-regex -- the controls are what it finds +const URL_PARSER_UNSAFE = /[/\\?#%\x00-\x20\x7f]/; + const isDomainName = (value) => { if (typeof value !== 'string' || value.length === 0) return false; + if (URL_PARSER_UNSAFE.test(value)) return false; // domainToASCII gives '' for a name it cannot convert, such as one with a - // space, a semicolon or a line break. + // semicolon. const ascii = domainToASCII(value); if (ascii.length === 0 || ascii.length > 253) return false; for (const label of ascii.split('.')) { diff --git a/tests/unit/mime-dkim.test.js b/tests/unit/mime-dkim.test.js index e08c071..da99c56 100644 --- a/tests/unit/mime-dkim.test.js +++ b/tests/unit/mime-dkim.test.js @@ -117,6 +117,13 @@ test('DKIM: a domain, selector or field name that could break the signature fiel { signingDomain: '-example.com' }, { selector: 'mail\r\nX: y' }, { selector: 'a;b' }, + // The URL parser removes these, so the name read as valid while the line + // break went into the field. + { signingDomain: 'exa\r\nmple.com' }, + { signingDomain: 'example.com\n' }, + { signingDomain: 'ex\tample.com' }, + { signingDomain: 'example.com/x' }, + { selector: 'mail\r\n2026' }, { headerFields: ['from', 'subject; x=1'] }, { headerFields: ['from:to'] }, { algorithm: 'rsa-sha1' }, From 82bcc8dad7dcfc6c7c83e52063af42f4a7f27b6a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:17:05 +0300 Subject: [PATCH 014/143] feat: forward sendRaw and verify through every decorator; raw on SES and Mailgun Upstream's decorators forwarded neither sendRaw nor verify, so a transport that sends raw MIME lost it once wrapped in ./retry, ./pool, ./otel or ./logger, while nodemailer takes raw messages on every transport. Each decorator now has both: ./retry retries a raw message as a message and verifies once, ./pool sends one through its transports that have sendRaw with its strategy and failover (no selector is asked) and verifies when one of its transports verifies, ./otel traces it in a span of its own, ./logger logs it by its envelope, and ./mock keeps raw messages apart. A wrapped transport without sendRaw gives a failure of the category configuration. SES sends raw MIME as the Raw content of SendEmail and Mailgun through messages.mime, each to the envelope of the message, checked line by line as SMTP checks it. The budgets of SES and Mailgun go up. Co-Authored-By: Claude Opus 5.5 --- docs/transports/logger.md | 10 ++ docs/transports/mailgun.md | 6 + docs/transports/mock.md | 7 + docs/transports/otel.md | 9 + docs/transports/pool.md | 12 ++ docs/transports/retry.md | 8 + docs/transports/ses.md | 8 + scripts/size-budgets.json | 4 +- src/internal/capabilities.js | 21 +++ src/internal/raw.js | 54 ++++++ src/logger/index.d.ts | 15 +- src/logger/logger-transport.js | 53 +++++- src/mailgun/http-client.js | 13 ++ src/mailgun/index.d.ts | 10 +- src/mailgun/mailgun-transport.js | 27 +++ src/mock/index.d.ts | 24 ++- src/mock/mock-transport.js | 33 ++++ src/otel/attributes.js | 23 +++ src/otel/index.d.ts | 22 ++- src/otel/opentelemetry-transport.js | 61 ++++++- src/otel/tracing.js | 18 ++ src/pool/index.d.ts | 28 ++- src/pool/pool-transport.js | 77 +++++++- src/retry/index.d.ts | 15 +- src/retry/retry-transport.js | 48 ++++- src/ses/index.d.ts | 10 +- src/ses/message-converter.js | 27 ++- src/ses/ses-transport.js | 30 +++- tests/unit/raw-verify.test.js | 268 ++++++++++++++++++++++++++++ 29 files changed, 906 insertions(+), 35 deletions(-) create mode 100644 src/internal/capabilities.js create mode 100644 src/internal/raw.js create mode 100644 tests/unit/raw-verify.test.js diff --git a/docs/transports/logger.md b/docs/transports/logger.md index b01eb58..2e12d92 100644 --- a/docs/transports/logger.md +++ b/docs/transports/logger.md @@ -116,6 +116,16 @@ The outer decorator observes the complete operation performed by the inner transport, so choose the order according to which retries or failovers should appear as one logged operation. +## Raw messages and verification + +`sendRaw()` logs the lifecycle of serialized MIME as `send()` logs a message's, +with `operation: 'sendRaw'` and the number of recipients of its envelope; the +envelope itself, which holds addresses, goes in the record only when +`recordMessage` is not `false`. Without a wrapped transport the receipt is +synthetic, as for `send()`; with one that has no `sendRaw()`, it is a failure of +the category `configuration`, logged as a failure. `verify()` verifies the +wrapped transport, and has nothing to check without one. + ## Levels The lifecycle levels can be changed independently, by pino's level names diff --git a/docs/transports/mailgun.md b/docs/transports/mailgun.md index 02ac7c7..4acdf83 100644 --- a/docs/transports/mailgun.md +++ b/docs/transports/mailgun.md @@ -150,6 +150,12 @@ Tags appear in Mailgun's analytics dashboard, allowing you to track open rates, click rates, and delivery statistics for specific message categories. This helps you understand which types of emails perform best with your audience. +## Raw MIME + +`sendRaw()` sends serialized MIME through the `messages.mime` endpoint, to the +recipients of its envelope. The message is checked line by line as the SMTP +transport checks it, then read into memory; composition is bypassed. + ## Bulk email sending For sending newsletters, notifications, or other bulk emails, the Mailgun diff --git a/docs/transports/mock.md b/docs/transports/mock.md index 04255c5..bb153f0 100644 --- a/docs/transports/mock.md +++ b/docs/transports/mock.md @@ -369,6 +369,13 @@ The mock transport handles bulk sending efficiently and provides detailed verification of each message, making it perfect for testing newsletter systems, notification broadcasts, and other high-volume email features. +## Raw messages and verification + +`sendRaw()` takes serialized MIME as `send()` takes a message: it applies the +delay and answers with the next response, and keeps the message apart, in +`getSentRawMessages()`. `clearSentMessages()` clears both lists. `verify()` has +nothing to check and resolves, unless its signal is aborted. + ## Test cleanup and isolation When running multiple tests, it's important to ensure that each test starts diff --git a/docs/transports/otel.md b/docs/transports/otel.md index 4f1963d..c53fe0f 100644 --- a/docs/transports/otel.md +++ b/docs/transports/otel.md @@ -494,6 +494,15 @@ Custom attributes appear in both traces and metrics, enabling you to: - Track different email campaigns or types separately - Add environment-specific context for debugging +## Raw messages and verification + +`sendRaw()` goes through the wrapped transport in a span of its own, +`email send_raw`, whose attributes are those of the envelope (`email.to.count`, +the sender's domain or, with `recordSensitiveData`, its address) and the size of +the content, and in the metrics of a send. `verify()` verifies the wrapped +transport. A wrapped transport without `sendRaw()` gives a failure of the +category `configuration`. + ## Bulk email monitoring For applications that send large volumes of emails, the OpenTelemetry transport diff --git a/docs/transports/pool.md b/docs/transports/pool.md index f7f8580..b188f10 100644 --- a/docs/transports/pool.md +++ b/docs/transports/pool.md @@ -474,6 +474,18 @@ preventing connection leaks and ensuring graceful shutdown. Each transport is disposed of once, even when it is listed twice, and one that fails stops no other: the disposal rejects afterwards with an `AggregateError` of the failures. +## Raw messages and verification + +`sendRaw()` sends serialized MIME through the enabled transports that have +`sendRaw()`, with the configured strategy and failover; a selector is not asked, +as it judges a `Message`. With no such transport the receipt is a failure of the +category `configuration`. + +`verify()` verifies the enabled transports that can verify, all at once, and +resolves when one of them does: the pool can send when one of its transports can. +When none does, it throws an `AggregateError` of their errors; when none can +verify, it has nothing to check. + ## Testing with pools Pool transports integrate seamlessly with testing workflows using mock diff --git a/docs/transports/retry.md b/docs/transports/retry.md index 640b7e4..943e237 100644 --- a/docs/transports/retry.md +++ b/docs/transports/retry.md @@ -224,6 +224,14 @@ A `logger` in the configuration hears each retry the decorator schedules, at on its span and in its metrics, and the JMAP transport does not report its own yet. +## Raw messages and verification + +`sendRaw()` retries serialized MIME as `send()` retries a message, since the +content of a raw message can be read again. `verify()` verifies the wrapped +transport once, without retries. When the wrapped transport has no `sendRaw()`, +the receipt is a failure of the category `configuration`, not retried; when it +cannot verify, `verify()` has nothing to check and resolves. + ## Composition Retry transport composes with other decorators because it preserves the diff --git a/docs/transports/ses.md b/docs/transports/ses.md index e29ae86..135a8e9 100644 --- a/docs/transports/ses.md +++ b/docs/transports/ses.md @@ -295,6 +295,14 @@ It reads `batchSize` messages (50 by default) at a time and sends them together before it reads more, so an endless or very long iterable costs one batch of memory. +## Raw MIME + +`sendRaw()` sends serialized MIME as the Raw content of SendEmail, to the +recipients of its envelope and from its sender (none for a null sender), with +the configuration set and the default tags of the transport. The message is +checked line by line as the SMTP transport checks it, then read into memory; +composition and DKIM are bypassed, so sign the message before. + ## Configuration sets and tracking Amazon SES provides configuration sets for advanced tracking, reputation diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d9cd1dc..3e6b432 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -5,10 +5,10 @@ "./smtp": 22.5, "./sendmail": 10.5, "./jmap": 12.5, - "./ses": 6.5, + "./ses": 7, "./resend": 6, "./sendgrid": 6, - "./mailgun": 5.5, + "./mailgun": 6, "./mailtrap": 6.5, "./maileroo": 6, "./lettermint": 6, diff --git a/src/internal/capabilities.js b/src/internal/capabilities.js new file mode 100644 index 0000000..7250d04 --- /dev/null +++ b/src/internal/capabilities.js @@ -0,0 +1,21 @@ +// Not ported: the receipt of a decorator asked for what the transport it wraps +// cannot do, such as `sendRaw()` of one that composes every message itself. + +const { createFailedReceipt } = require('../core/receipt.js'); + +/** + * The failed receipt of `sendRaw()` through a decorator whose transport has + * none: a configuration error, not retried and not failed over. + * + * @param transport The transport that cannot send raw messages. + * @returns The receipt. + * @internal + */ +const refuseRaw = (transport) => + createFailedReceipt(`The transport ${transport.id} does not send raw messages.`, { + provider: transport.id, + category: 'configuration', + retryable: false, + }); + +module.exports = { refuseRaw }; diff --git a/src/internal/raw.js b/src/internal/raw.js new file mode 100644 index 0000000..662e535 --- /dev/null +++ b/src/internal/raw.js @@ -0,0 +1,54 @@ +// Not ported: the bytes of a raw message for an HTTP API that takes it whole, +// read once and checked as the SMTP transport checks it. + +const { + createRawMessagePlan, + iterateRawMessage, + RawMessageValidationError, +} = require('../core/raw-message.js'); +const { toAsciiAddress } = require('../core/address.js'); +const { createFailedReceipt } = require('../core/receipt.js'); +const { joinChunks } = require('./body.js'); + +/** + * Reads a raw message into one buffer, its lines checked as they are read, + * and its envelope with domains as A-labels. + * + * @param message The raw message. + * @param signal Cancels the read. + * @returns `{ envelope, bytes }`. + * @throws {RawMessageValidationError} If the envelope or the content is + * invalid. + * @internal + */ +const readRawMessage = async (message, signal) => { + const plan = createRawMessagePlan(message); + const chunks = []; + let size = 0; + for await (const chunk of iterateRawMessage(plan, { signal })) { + // A chunk is valid until the next read. + const copy = chunk.slice(); + chunks.push(copy); + size += copy.length; + } + const to = []; + for (let i = 0; i < plan.envelope.to.length; i++) to.push(toAsciiAddress(plan.envelope.to[i])); + const from = plan.envelope.from == null ? null : toAsciiAddress(plan.envelope.from); + return { envelope: { from, to }, bytes: joinChunks(chunks, size) }; +}; + +/** + * The receipt of a raw message that is invalid, or `undefined` for another + * error. + * + * @param provider The transport. + * @param error What was thrown. + * @returns A non-retryable `validation` receipt, or `undefined`. + * @internal + */ +const rawMessageFailure = (provider, error) => + error instanceof RawMessageValidationError + ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) + : undefined; + +module.exports = { rawMessageFailure, readRawMessage }; diff --git a/src/logger/index.d.ts b/src/logger/index.d.ts index 6faae41..1eb961a 100644 --- a/src/logger/index.d.ts +++ b/src/logger/index.d.ts @@ -9,9 +9,12 @@ import type { Logger, LoggerLevel, Message, + RawMessage, + RawTransport, Receipt, Transport, TransportOptions, + VerifiableTransport, } from '../../index.js'; // The logger types are those of the root entry, shared by every transport; @@ -131,7 +134,7 @@ type LoggerTransportConstructorArguments = [TProvide * `"logger"` in log-only mode. */ export declare class LoggerTransport - implements Transport, AsyncDisposable + implements RawTransport, VerifiableTransport, AsyncDisposable { /** Provider id used by receipts from this transport. */ readonly id: TProviderId; @@ -157,6 +160,16 @@ export declare class LoggerTransport * @throws {Error} If the wrapped transport throws an error. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME through the wrapped transport, its lifecycle logged with the envelope for the message. A wrapped + * transport without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Verifies the wrapped transport; it has nothing to check when there is none or it cannot verify. + */ + verify(options?: TransportOptions): Promise; /** * Sends multiple emails while recording each delivery lifecycle. * diff --git a/src/logger/logger-transport.js b/src/logger/logger-transport.js index 28e8b6e..7393d30 100644 --- a/src/logger/logger-transport.js +++ b/src/logger/logger-transport.js @@ -7,6 +7,11 @@ const { generateId } = require('../core/id.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { resolveLoggerTransportOptions } = require('./config.js'); const { PendingQueue } = require('./pending-queue.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('../internal/capabilities.js'); + +// A raw message has its envelope where a message has its recipients. +const isRawMessage = (message) => message.envelope != null && message.recipients == null; /** * Transport that records email delivery lifecycle events through a logger the @@ -59,6 +64,34 @@ class LoggerTransport { return this.sendOne(message, options, 'send'); } + /** + * Sends serialized MIME through the wrapped transport while recording its + * lifecycle, the envelope standing for the message. A wrapped transport + * without `sendRaw()` gives a failed receipt of the category + * `configuration`, logged as a failure. + * + * @param message The serialized message and its envelope. + * @param options Optional transport options, including cancellation. + * @returns The receipt. + */ + sendRaw(message, options) { + return this.sendOne(message, options, 'sendRaw', (raw, sendOptions) => + isRawTransport(this.wrappedTransport) + ? this.wrappedTransport.sendRaw(raw, sendOptions) + : refuseRaw(this.wrappedTransport), + ); + } + + /** + * Verifies the wrapped transport, which has nothing to check when it is + * absent or cannot verify. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const transport = this.wrappedTransport; + if (transport != null && isVerifiableTransport(transport)) await transport.verify(options); + } + /** * Sends multiple emails while recording each delivery lifecycle. * @@ -127,7 +160,12 @@ class LoggerTransport { } } - async sendOne(message, options, operation) { + async sendOne( + message, + options, + operation, + deliver = (sent, sendOptions) => this.wrappedTransport.send(sent, sendOptions), + ) { options?.signal?.throwIfAborted(); const startedAt = performance.now(); this.logSending(message, operation); @@ -136,10 +174,7 @@ class LoggerTransport { const receipt = this.wrappedTransport == null ? this.createSyntheticReceipt() - : await this.wrappedTransport.send( - message, - this.withRetryLogging(options, message, operation), - ); + : await deliver(message, this.withRetryLogging(options, message, operation)); this.logReceipt(receipt, { message, startedAt }, operation); return receipt; } catch (error) { @@ -293,6 +328,12 @@ class LoggerTransport { } getMessageProperties(message) { + if (isRawMessage(message)) { + return { + recipientCount: message.envelope.to.length, + ...(this.config.recordMessage === false ? {} : { envelope: message.envelope }), + }; + } return { recipientCount: message.recipients.length, ccRecipientCount: message.ccRecipients.length, @@ -307,7 +348,7 @@ class LoggerTransport { // placeholders of a template; a pino-like logger does not, so the text is // built here. getMessageText(lifecycleMessage, message) { - if (this.config.recordMessage !== 'inline' || message == null) { + if (this.config.recordMessage !== 'inline' || message == null || isRawMessage(message)) { return lifecycleMessage; } diff --git a/src/mailgun/http-client.js b/src/mailgun/http-client.js index 8c767e1..9816063 100644 --- a/src/mailgun/http-client.js +++ b/src/mailgun/http-client.js @@ -44,6 +44,19 @@ class MailgunHttpClient { ); } + /** + * Sends serialized MIME to the `messages.mime` endpoint. + * + * @param formData The `to` fields and the `message` file. + * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. + * @returns Promise that resolves to the Mailgun response. + */ + sendMime(formData, signal, onRetry) { + const url = `${this.config.baseUrl}/${encodeURIComponent(this.config.domain)}/messages.mime`; + return this.makeRequest(url, { method: 'POST', body: formData, signal }, onRetry); + } + /** * Makes an HTTP request to Mailgun API with retry logic. * diff --git a/src/mailgun/index.d.ts b/src/mailgun/index.d.ts index 65a085c..ee85378 100644 --- a/src/mailgun/index.d.ts +++ b/src/mailgun/index.d.ts @@ -7,6 +7,8 @@ import type { Transport, TransportDependencies, TransportOptions, + RawMessage, + RawTransport, } from '../../index.js'; /** @@ -150,7 +152,7 @@ type ResolvedMailgunConfig = Required { +export declare class MailgunTransport implements RawTransport<'mailgun'> { readonly id = 'mailgun'; /** * The resolved Mailgun configuration used by this transport. @@ -196,6 +198,12 @@ export declare class MailgunTransport implements Transport<'mailgun'> { * failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME through the `messages.mime` endpoint, to the recipients of its + * envelope. It is checked as SMTP checks it and read into memory; + * composition is bypassed. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; /** * Sends multiple email messages efficiently via Mailgun API. * diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index 1cf4e6d..2cc7cce 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -5,6 +5,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createMailgunConfig } = require('./config.js'); const { MailgunApiError, MailgunHttpClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); @@ -133,6 +134,32 @@ class MailgunTransport { } } + /** + * Sends serialized MIME through the `messages.mime` endpoint, to the + * recipients of its envelope. The message is checked as SMTP checks it + * and read into memory; composition is bypassed. + * + * @param message The serialized message and its envelope. + * @param options Optional abort signal and retry callback. + * @returns The receipt. + */ + async sendRaw(message, options) { + if (message?.content instanceof Promise) message.content.catch(() => {}); + options?.signal?.throwIfAborted(); + try { + const raw = await readRawMessage(message, options?.signal); + const formData = new FormData(); + for (const address of raw.envelope.to) formData.append('to', address); + formData.append('message', new Blob([raw.bytes], { type: 'message/rfc822' }), 'message.mime'); + const response = await this.httpClient.sendMime(formData, options?.signal, options?.onRetry); + return { successful: true, messageId: response.id, provider: 'mailgun' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + const errorMessage = error instanceof Error ? error.message : String(error); + return rawMessageFailure('mailgun', error) ?? createMailgunFailure(errorMessage, error); + } + } + /** * Sends multiple email messages efficiently via Mailgun API. * diff --git a/src/mock/index.d.ts b/src/mock/index.d.ts index f6fe51b..b8a75c3 100644 --- a/src/mock/index.d.ts +++ b/src/mock/index.d.ts @@ -1,7 +1,14 @@ // Types ported from upyo v0.7.0 (packages/mock/src) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -import type { Message, Receipt, Transport, TransportOptions } from '../../index.js'; +import type { + Message, + RawMessage, + RawTransport, + Receipt, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; /** * Configuration interface for Mock transport. @@ -95,7 +102,7 @@ type ResolvedMockConfig = Required; * comprehensive testing capabilities including message verification, * behavior simulation, and async utilities. */ -export declare class MockTransport implements Transport<'mock'> { +export declare class MockTransport implements RawTransport<'mock'>, VerifiableTransport<'mock'> { readonly id = 'mock'; /** * The resolved configuration used by this mock transport. @@ -119,6 +126,19 @@ export declare class MockTransport implements Transport<'mock'> { * @throws {DOMException} When the operation is aborted via `AbortSignal`. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Takes serialized MIME as `send()` takes a message: it is kept apart from + * the messages and answered with the next response. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Has nothing to verify, and resolves unless the signal is aborted. + */ + verify(options?: TransportOptions): Promise; + /** + * The raw messages sent through {@link sendRaw}, in order. + */ + getSentRawMessages(): readonly RawMessage[]; /** * Sends multiple email messages through the mock transport. * diff --git a/src/mock/mock-transport.js b/src/mock/mock-transport.js index 94bed9c..ea4a794 100644 --- a/src/mock/mock-transport.js +++ b/src/mock/mock-transport.js @@ -29,6 +29,7 @@ class MockTransport { config; sentMessages = []; + sentRawMessages = []; nextResponse = null; messageIdCounter = 1; @@ -68,6 +69,37 @@ class MockTransport { return response; } + /** + * Takes serialized MIME as `send()` takes a message: it is kept, apart from + * the messages, and answered with the next response. + * + * @param message The serialized message and its envelope. + * @param options Transport options including abort signal. + * @returns The receipt. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + await this.applyDelay(options?.signal); + this.sentRawMessages.push(message); + return this.getResponse(); + } + + /** + * Has nothing to verify, and resolves, unless the signal is aborted. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + } + + /** + * The raw messages sent through {@link sendRaw}, in order. + * + * @returns A copy of the list. + */ + getSentRawMessages() { + return [...this.sentRawMessages]; + } + /** * Sends multiple email messages through the mock transport. * @@ -121,6 +153,7 @@ class MockTransport { */ clearSentMessages() { this.sentMessages = []; + this.sentRawMessages = []; } /** diff --git a/src/otel/attributes.js b/src/otel/attributes.js index d91e8ea..deeccf3 100644 --- a/src/otel/attributes.js +++ b/src/otel/attributes.js @@ -96,6 +96,29 @@ class EmailAttributeExtractor { return attributes; } + /** + * Extracts attributes for the send of serialized MIME: its envelope and + * size, with the sender's address under the same rule as a message's. + */ + extractRawMessageAttributes(message, messageSize) { + const from = message?.envelope?.from; + const attributes = { + 'operation.name': 'email.send_raw', + 'operation.type': 'email', + 'email.to.count': message?.envelope?.to?.length ?? 0, + 'mailer.transport.name': this.transportName, + 'mailer.retry.count': 0, + 'mailer.batch.size': 1, + 'email.message.size': messageSize, + }; + if (this.transportVersion) attributes['mailer.transport.version'] = this.transportVersion; + if (typeof from === 'string') { + if (this.recordSensitiveData) attributes['email.from.address'] = from; + else attributes['email.from.domain'] = this.extractDomain(from); + } + return attributes; + } + /** * Extracts attributes for batch send operations, from the totals of the * messages the batch read. diff --git a/src/otel/index.d.ts b/src/otel/index.d.ts index cc93fc4..168d0dc 100644 --- a/src/otel/index.d.ts +++ b/src/otel/index.d.ts @@ -3,7 +3,15 @@ /// -import type { Message, Receipt, Transport, TransportOptions } from '../../index.js'; +import type { + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; // The parts of the OpenTelemetry API this subpath calls, as structural types: // the providers come from the application's own setup, and nothing here @@ -277,7 +285,7 @@ export declare function defaultErrorClassifier(error: unknown): string; * ``` */ export declare class OpenTelemetryTransport - implements Transport, AsyncDisposable + implements RawTransport, VerifiableTransport, AsyncDisposable { readonly id: TProviderId; /** @@ -302,6 +310,16 @@ export declare class OpenTelemetryTransport * @returns A promise that resolves to a receipt indicating success or failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME through the wrapped transport, in a span of its own (`send_raw`) and the metrics of a send. A wrapped + * transport without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Verifies the wrapped transport; it has nothing to check when it cannot verify. + */ + verify(options?: TransportOptions): Promise; /** * Sends multiple email messages with OpenTelemetry observability. * diff --git a/src/otel/opentelemetry-transport.js b/src/otel/opentelemetry-transport.js index d34a7e9..8e91fa3 100644 --- a/src/otel/opentelemetry-transport.js +++ b/src/otel/opentelemetry-transport.js @@ -5,6 +5,8 @@ const { createOpenTelemetryConfig, defaultErrorClassifier } = require('./config. const { MetricsCollector } = require('./metrics.js'); const { TracingCollector } = require('./tracing.js'); const { estimateMessageSize } = require('./message-size.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('../internal/capabilities.js'); // How many error messages of a failed batch its span keeps; the count says how // many there were in all. @@ -144,6 +146,54 @@ class OpenTelemetryTransport { this.metricsCollector?.recordSendStart(this.transportName); this.metricsCollector?.recordMessage(message, this.transportName, messageSize); + return this.observe( + { startTime, span, messageSize, priority: message.priority }, + (sendOptions) => this.wrappedTransport.send(message, sendOptions), + options, + ); + } + + /** + * Sends serialized MIME through the wrapped transport, in a span of its own + * (`send_raw`) and in the metrics of a send. A wrapped transport without + * `sendRaw()` gives a failed receipt of the category `configuration`. + * + * @param message The serialized message and its envelope. + * @param options Optional transport options including abort signal. + * @returns The receipt of the wrapped transport. + */ + async sendRaw(message, options) { + if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const startTime = performance.now(); + const messageSize = message?.content instanceof Uint8Array ? message.content.byteLength : 0; + const span = this.tracingCollector?.createRawSendSpan( + message, + this.extractNetworkAttributes(), + messageSize, + ); + this.metricsCollector?.recordSendStart(this.transportName); + return this.observe( + { startTime, span, messageSize, priority: 'normal' }, + (sendOptions) => this.wrappedTransport.sendRaw(message, sendOptions), + options, + ); + } + + /** + * Verifies the wrapped transport, which has nothing to check when it + * cannot verify. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** + * Runs one send in its span and metrics: the custom attributes, the + * outcome, the error and the end. + */ + async observe(context, deliver, options) { + const { startTime, span, messageSize, priority } = context; try { // Apply custom attributes if configured if (this.config.attributeExtractor && span) { @@ -157,16 +207,9 @@ class OpenTelemetryTransport { } // Delegate to wrapped transport - const receipt = await this.wrappedTransport.send( - message, - this.withRetryRecording(options, span), - ); + const receipt = await deliver(this.withRetryRecording(options, span)); const duration = (performance.now() - startTime) / 1000; // Convert to seconds - this.metricsCollector?.recordMessageOutcome( - this.transportName, - message.priority, - receipt.successful, - ); + this.metricsCollector?.recordMessageOutcome(this.transportName, priority, receipt.successful); // Record success/failure if (receipt.successful) { diff --git a/src/otel/tracing.js b/src/otel/tracing.js index 499f7ef..c27b35c 100644 --- a/src/otel/tracing.js +++ b/src/otel/tracing.js @@ -70,6 +70,24 @@ class TracingCollector { return new LiveEmailSpan(span, this.attributeExtractor); } + /** + * Creates a span for the send of serialized MIME, whose envelope is all + * that is known of it. + */ + createRawSendSpan(message, networkAttributes, messageSize) { + if (!this.shouldSample()) { + return new NoOpEmailSpan(); + } + const span = this.tracer.startSpan(`${this.config.spanPrefix} send_raw`, { + kind: SPAN_KIND_CLIENT, + }); + span.setAttributes(this.attributeExtractor.extractRawMessageAttributes(message, messageSize)); + if (networkAttributes) { + span.setAttributes(networkAttributes); + } + return new LiveEmailSpan(span, this.attributeExtractor); + } + /** * Creates a span for a batch email send operation. Its batch attributes * come at its end (`recordBatchTotals`), once the batch has been read. diff --git a/src/pool/index.d.ts b/src/pool/index.d.ts index 54dafca..9ac03d4 100644 --- a/src/pool/index.d.ts +++ b/src/pool/index.d.ts @@ -3,7 +3,17 @@ /// -import { IdOptions, Logger, Message, Receipt, Transport, TransportOptions } from '../../index.js'; +import { + IdOptions, + Logger, + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; /** * Result of transport selection by a strategy. @@ -199,7 +209,10 @@ export interface ResolvedTransportEntry { * ``` */ export declare class PoolTransport - implements Transport, AsyncDisposable + implements + RawTransport, + VerifiableTransport, + AsyncDisposable { readonly id = 'pool'; /** @@ -225,6 +238,17 @@ export declare class PoolTransport * @returns A promise that resolves to a receipt indicating success or failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME through the transports that send raw messages, + * with the strategy and the failover of `send()`; a selector is not asked, + * as it judges a `Message`. With no such transport, the receipt is a + * failure of the category `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Verifies the enabled transports that can verify: it resolves when one of them verifies, and throws an `AggregateError` of their errors when none does; it has nothing to check when none can verify. + */ + verify(options?: TransportOptions): Promise; /** * Sends multiple email messages using the pool strategy. * diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index b67b21a..75041a9 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -3,6 +3,8 @@ const { combineSignals } = require('../core/abort-signal.js'); const { pinMessageId } = require('../core/message-id.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('../internal/capabilities.js'); const { createLog } = require('../internal/log.js'); const { createFailedReceipt, createReceiptError } = require('../core/receipt.js'); const { createPoolConfig } = require('./config.js'); @@ -98,6 +100,71 @@ class PoolTransport { */ async send(message, options) { const pinned = pinMessageId(message, this.config.generateId); + return this.deliver( + pinned, + this.config.transports, + (transport, sendOptions) => transport.send(pinned, sendOptions), + options, + ); + } + + /** + * Sends serialized MIME through the transports that send raw messages, with + * the strategy and the failover of `send()`. A selector is not asked, as it + * judges a `Message`; with no such transport the receipt is a failure of + * the category `configuration`. + * + * @param message The serialized message and its envelope. + * @param options Optional transport options including abort signal. + * @returns The receipt of the first transport that sent it, or a failure. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + const entries = []; + let enabled = 0; + for (const entry of this.config.transports) { + const canSend = entry.enabled && isRawTransport(entry.transport); + if (canSend) enabled++; + entries.push({ ...entry, enabled: canSend, selector: undefined }); + } + if (enabled === 0) return refuseRaw(this); + return this.deliver( + message, + entries, + (transport, sendOptions) => transport.sendRaw(message, sendOptions), + options, + ); + } + + /** + * Verifies the enabled transports that can verify, all at once: the pool + * can send when one of them can. It throws an `AggregateError` of their + * errors when none verifies, and has nothing to check when none can verify. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const verifying = []; + for (const entry of this.config.transports) { + if (entry.enabled && isVerifiableTransport(entry.transport)) { + verifying.push(entry.transport.verify(options)); + } + } + if (verifying.length === 0) return; + const results = await Promise.allSettled(verifying); + options?.signal?.throwIfAborted(); + const errors = []; + for (const result of results) { + if (result.status === 'fulfilled') return; + errors.push(result.reason); + } + throw new AggregateError(errors, 'No transport of the pool verified.'); + } + + /** + * Sends a message through the transports the strategy picks, one after + * another until one sends it or failover has to stop. + */ + async deliver(message, entries, sendOnce, options) { const attemptedIndices = new Set(); // Failover stops when another transport could only deliver the message a // second time (delivery unknown) or would refuse it too (the message @@ -112,12 +179,12 @@ class PoolTransport { for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) { options?.signal?.throwIfAborted(); - const selection = this.strategy.select(pinned, this.config.transports, attemptedIndices); + const selection = this.strategy.select(message, entries, attemptedIndices); // No more transports available if (!selection) break; attemptedIndices.add(selection.index); - const receipt = await this.attempt(selection, pinned, options, failure); + const receipt = await this.attempt(selection, sendOnce, options, failure); if (receipt?.successful) return receipt; if (failure.deliveryUnknown || failure.isRejected) break; if (receipt !== undefined) { @@ -148,18 +215,18 @@ class PoolTransport { * timeout, and records a failure. * * @param selection The selected transport and its index. - * @param pinned The message, with its Message-ID pinned. + * @param sendOnce Sends the message through a transport, once. * @param options The options of the caller. * @param failure What the failed attempts have gathered, updated in place. * @returns The receipt of the transport, or `undefined` when it threw. * @throws The caller's abort reason when the caller aborts. */ - async attempt(selection, pinned, options, failure) { + async attempt(selection, sendOnce, options, failure) { const transportId = selection.entry.transport.id; const sendOptions = this.createSendOptions(options); let receipt; try { - receipt = await selection.entry.transport.send(pinned, sendOptions.options); + receipt = await sendOnce(selection.entry.transport, sendOptions.options); } catch (error) { if (sendOptions.abortedByCaller() && isCallerAbort(error, options?.signal)) throw error; recordThrown(failure, error, transportId); diff --git a/src/retry/index.d.ts b/src/retry/index.d.ts index 83972f1..88cf727 100644 --- a/src/retry/index.d.ts +++ b/src/retry/index.d.ts @@ -11,6 +11,9 @@ import type { ReceiptError, Transport, TransportOptions, + RawMessage, + RawTransport, + VerifiableTransport, } from '../../index.js'; /** @@ -198,7 +201,7 @@ interface ResolvedRetryConfig { * Transport decorator that retries transient delivery failures. */ export declare class RetryTransport - implements Transport, AsyncDisposable + implements RawTransport, VerifiableTransport, AsyncDisposable { readonly id: TProviderId; /** @@ -222,6 +225,16 @@ export declare class RetryTransport * @throws {DOMException} If the operation is aborted. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME through the wrapped transport, retrying retryable failures as `send()` does. A wrapped + * transport without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Verifies the wrapped transport once, without retries; it has nothing to check when it cannot verify. + */ + verify(options?: TransportOptions): Promise; /** * Sends messages with per-message retry behavior. * diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index be112d5..4611dd8 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -8,6 +8,8 @@ const { } = require('../core/receipt.js'); const { combineSignals } = require('../core/abort-signal.js'); const { pinMessageId } = require('../core/message-id.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('../internal/capabilities.js'); const { createLog } = require('../internal/log.js'); const { createRetryConfig } = require('./config.js'); @@ -62,13 +64,57 @@ class RetryTransport { options?.signal?.throwIfAborted(); const pinned = pinMessageId(message, this.config.generateId); + return this.withRetries( + (sendOptions) => this.wrappedTransport.send(pinned, sendOptions), + options, + ); + } + + /** + * Sends serialized MIME through the wrapped transport, retrying retryable + * failures as `send()` does; the content of a raw message can be read + * again. A wrapped transport without `sendRaw()` gives a failed receipt of + * the category `configuration`. + * + * @param message The serialized message and its envelope. + * @param options Optional transport options. + * @returns The final delivery receipt. + * @throws {DOMException} If the operation is aborted. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + return this.withRetries( + (sendOptions) => this.wrappedTransport.sendRaw(message, sendOptions), + options, + ); + } + + /** + * Verifies the wrapped transport once, without retries; it has nothing to + * check when it cannot verify. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** + * Sends once, then again while the failure is retryable and attempts are + * left, waiting between attempts. + * + * @param sendOnce Sends the message once with the given options. + * @param options Optional transport options. + * @returns The final delivery receipt. + */ + async withRetries(sendOnce, options) { let lastThrownError; for (let attempt = 1; attempt <= this.config.maxAttempts; attempt++) { options?.signal?.throwIfAborted(); let receipt; try { - receipt = await this.wrappedTransport.send(pinned, options); + receipt = await sendOnce(options); } catch (error) { options?.signal?.throwIfAborted(); // A bug of the wrapped transport is not a delivery failure to retry diff --git a/src/ses/index.d.ts b/src/ses/index.d.ts index 3e13dd2..1713519 100644 --- a/src/ses/index.d.ts +++ b/src/ses/index.d.ts @@ -7,6 +7,8 @@ import type { Transport, TransportDependencies, TransportOptions, + RawMessage, + RawTransport, } from '../../index.js'; /** @@ -200,7 +202,7 @@ type ResolvedSesConfig = Required< * } * ``` */ -export declare class SesTransport implements Transport<'ses'> { +export declare class SesTransport implements RawTransport<'ses'> { readonly id = 'ses'; /** Resolved configuration with defaults applied */ config: ResolvedSesConfig; @@ -238,6 +240,12 @@ export declare class SesTransport implements Transport<'ses'> { * @returns A promise that resolves to a receipt with the result. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends serialized MIME as the Raw content of SendEmail, to the recipients of its + * envelope. It is checked as SMTP checks it and read into memory; + * composition is bypassed. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; /** * Sends multiple email messages concurrently through Amazon SES. * diff --git a/src/ses/message-converter.js b/src/ses/message-converter.js index 1547d0c..5aef5a0 100644 --- a/src/ses/message-converter.js +++ b/src/ses/message-converter.js @@ -89,6 +89,31 @@ const convertMessage = async (message, config, signal) => { return sesMessage; }; +/** + * The request of a raw message: its bytes as Raw content, its envelope as the + * destination and the sender. + * + * @param raw The envelope and bytes of `readRawMessage`. + * @param config The configuration. + * @returns The SES v2 SendEmail request. + */ +const convertRawMessage = (raw, config) => { + const sesMessage = { + Destination: { ToAddresses: raw.envelope.to }, + Content: { Raw: { Data: toBase64(raw.bytes) } }, + }; + if (raw.envelope.from != null) sesMessage.FromEmailAddress = raw.envelope.from; + if (config.configurationSetName) sesMessage.ConfigurationSetName = config.configurationSetName; + const tags = []; + for (const name in config.defaultTags) { + if (Object.hasOwn(config.defaultTags, name)) { + tags.push({ Name: name, Value: config.defaultTags[name] }); + } + } + if (tags.length > 0) sesMessage.Tags = tags; + return sesMessage; +}; + const createSimpleContent = async (message, signal) => { const content = { Subject: { @@ -199,4 +224,4 @@ const convertAttachment = async (attachment, signal) => { return converted; }; -module.exports = { convertMessage }; +module.exports = { convertMessage, convertRawMessage }; diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 3110b02..42c56d3 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -5,7 +5,8 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { createSesConfig } = require('./config.js'); const { SesApiError, SesHttpClient } = require('./http-client.js'); -const { convertMessage } = require('./message-converter.js'); +const { convertMessage, convertRawMessage } = require('./message-converter.js'); +const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); @@ -140,6 +141,33 @@ class SesTransport { } } + /** + * Sends serialized MIME as the Raw content of SendEmail, to the recipients + * of its envelope and from its sender. The message is checked as SMTP + * checks it and read into memory; DKIM and composition are bypassed. + * + * @param message The serialized message and its envelope. + * @param options Optional abort signal and retry callback. + * @returns The receipt. + */ + async sendRaw(message, options) { + if (message?.content instanceof Promise) message.content.catch(() => {}); + options?.signal?.throwIfAborted(); + try { + const raw = await readRawMessage(message, options?.signal); + const response = await this.httpClient.sendMessage( + convertRawMessage(raw, this.config), + options?.signal, + options?.onRetry, + ); + return { successful: true, messageId: this.extractMessageId(response), provider: 'ses' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + const errorMessage = error instanceof Error ? error.message : String(error); + return rawMessageFailure('ses', error) ?? createSesFailure(errorMessage, error); + } + } + /** * Sends multiple email messages concurrently through Amazon SES. * diff --git a/tests/unit/raw-verify.test.js b/tests/unit/raw-verify.test.js new file mode 100644 index 0000000..6e4fa6f --- /dev/null +++ b/tests/unit/raw-verify.test.js @@ -0,0 +1,268 @@ +// Not ported: `sendRaw()` and `verify()` through every decorator, which +// upstream forwarded through none, and raw MIME on SES and Mailgun. +const { test } = require('node:test'); +const assert = require('node:assert'); +const PostalMime = require('postal-mime'); +const { isRawTransport, isVerifiableTransport } = require('../../index.js'); +const { RetryTransport } = require('../../src/retry/index.js'); +const { PoolTransport } = require('../../src/pool/index.js'); +const { OpenTelemetryTransport } = require('../../src/otel/index.js'); +const { LoggerTransport } = require('../../src/logger/index.js'); +const { MockTransport } = require('../../src/mock/index.js'); +const { SesTransport } = require('../../src/ses/index.js'); +const { MailgunTransport } = require('../../src/mailgun/index.js'); +const { MockMeterProvider, MockTracerProvider } = require('../helpers/otel/mock-otel-providers.js'); + +const encoder = new TextEncoder(); +const raw = (to = ['bob@example.com']) => ({ + envelope: { from: 'alex@bücher.de', to }, + content: encoder.encode( + 'From: alex@example.com\r\nTo: bob@example.com\r\nSubject: Raw\r\n\r\nHello\r\n', + ), +}); +// A transport that composes every message itself: no sendRaw, no verify. +const plain = (id = 'plain') => ({ + id, + send: async () => ({ successful: true, messageId: 'p', provider: id }), + async *sendMany() {}, +}); +const failure = (retryable) => ({ + successful: false, + errorMessages: ['busy'], + errors: [{ message: 'busy', category: 'server-error', code: 'x', retryable }], + retryable, +}); +const off = { tracing: { enabled: false }, metrics: { enabled: false } }; +const assertRefused = (receipt, id) => { + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(receipt.errors[0].category, 'configuration'); + assert.strictEqual(receipt.errorMessages[0], `The transport ${id} does not send raw messages.`); +}; + +test('Raw: every decorator and the mock send raw messages and verify', () => { + const mock = new MockTransport(); + const decorators = [ + new RetryTransport(mock), + new PoolTransport({ strategy: 'priority', transports: [{ transport: mock }] }), + new OpenTelemetryTransport(mock, { tracing: { enabled: false }, metrics: { enabled: false } }), + new LoggerTransport({ + transport: mock, + logger: { info() {}, debug() {}, warn() {}, error() {} }, + }), + mock, + new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'a', secretAccessKey: 's' }, + }), + new MailgunTransport({ apiKey: 'k', domain: 'mg.example.com' }), + ]; + for (const transport of decorators) assert.ok(isRawTransport(transport), transport.id); + for (const transport of decorators.slice(0, 5)) { + assert.ok(isVerifiableTransport(transport), transport.id); + } +}); + +test('Raw: the mock keeps raw messages apart, and verify resolves', async () => { + const mock = new MockTransport(); + const receipt = await mock.sendRaw(raw()); + assert.strictEqual(receipt.successful, true); + assert.strictEqual(mock.getSentRawMessages().length, 1); + assert.strictEqual(mock.getSentMessagesCount(), 0); + await mock.verify(); + mock.clearSentMessages(); + assert.strictEqual(mock.getSentRawMessages().length, 0); + const aborted = new AbortController(); + aborted.abort(new Error('stop')); + await assert.rejects(mock.verify({ signal: aborted.signal }), /stop/); + await assert.rejects(mock.sendRaw(raw(), { signal: aborted.signal }), /stop/); +}); + +test('Raw: retry retries a raw message as a message, and verifies once', async () => { + const mock = new MockTransport(); + mock.setNextResponse(failure(true)); + const retry = new RetryTransport(mock, { + maxAttempts: 3, + backoff: { baseDelayMilliseconds: 1, maxDelayMilliseconds: 1 }, + }); + const receipt = await retry.sendRaw(raw()); + assert.strictEqual(receipt.successful, true); + assert.strictEqual(receipt.attempts, 2); + assert.strictEqual(mock.getSentRawMessages().length, 2); + assertRefused(await new RetryTransport(plain()).sendRaw(raw()), 'plain'); + let verified = 0; + const verifiable = { ...plain(), verify: async () => verified++ }; + await new RetryTransport(verifiable).verify(); + await new RetryTransport(plain()).verify(); + assert.strictEqual(verified, 1); +}); + +test('Raw: the pool sends through the transports that can, failing over among them', async () => { + const first = new MockTransport(); + first.setNextResponse(failure(true)); + const second = new MockTransport(); + let asked = 0; + const pool = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: plain(), priority: 3 }, + { transport: first, priority: 2, selector: () => asked++ > -1 }, + { transport: second, priority: 1 }, + ], + }); + const receipt = await pool.sendRaw(raw()); + assert.strictEqual(receipt.successful, true); + assert.strictEqual(first.getSentRawMessages().length, 1); + assert.strictEqual(second.getSentRawMessages().length, 1); + assert.strictEqual(asked, 0); + const none = new PoolTransport({ strategy: 'priority', transports: [{ transport: plain() }] }); + assertRefused(await none.sendRaw(raw()), 'pool'); + const disabled = new PoolTransport({ + strategy: 'priority', + transports: [{ transport: new MockTransport(), enabled: false }, { transport: plain() }], + }); + assertRefused(await disabled.sendRaw(raw()), 'pool'); +}); + +test('Raw: the pool verifies when one transport verifies', async () => { + const fails = (text) => ({ ...plain(), verify: async () => Promise.reject(new Error(text)) }); + const passes = { ...plain(), verify: async () => {} }; + const pool = (...transports) => + new PoolTransport({ + strategy: 'priority', + transports: transports.map((transport) => ({ transport })), + }); + await pool(fails('down'), passes).verify(); + await pool(plain()).verify(); + await assert.rejects(pool(fails('a'), fails('b')).verify(), (error) => { + assert.ok(error instanceof AggregateError); + assert.deepStrictEqual( + error.errors.map((each) => each.message), + ['a', 'b'], + ); + return true; + }); + const aborted = new AbortController(); + aborted.abort(new Error('stop')); + await assert.rejects(pool(passes).verify({ signal: aborted.signal }), /stop/); +}); + +test('Raw: OpenTelemetry records a raw send in a span of its own', async () => { + const tracerProvider = new MockTracerProvider(); + const meterProvider = new MockMeterProvider(); + const mock = new MockTransport(); + const otel = new OpenTelemetryTransport(mock, { + tracerProvider, + meterProvider, + tracing: { enabled: true }, + metrics: { enabled: true }, + }); + const receipt = await otel.sendRaw(raw(['b@example.com', 'c@example.com'])); + assert.strictEqual(receipt.successful, true); + const span = tracerProvider.getAllSpans().at(-1); + assert.strictEqual(span.name, 'email send_raw'); + assert.strictEqual(span.attributes['operation.name'], 'email.send_raw'); + assert.strictEqual(span.attributes['email.to.count'], 2); + assert.strictEqual(span.attributes['email.from.domain'], 'bücher.de'); + assert.strictEqual(span.attributes['email.from.address'], undefined); + assert.ok(span.attributes['email.message.size'] > 0); + assertRefused(await new OpenTelemetryTransport(plain(), off).sendRaw(raw()), 'plain'); + await otel.verify(); + await new OpenTelemetryTransport(plain(), off).verify(); +}); + +test('Raw: the logger logs a raw send by its envelope, addresses only when asked', async () => { + const records = []; + const logger = {}; + for (const level of ['debug', 'info', 'warn', 'error']) { + logger[level] = (properties, text) => records.push({ level, properties, text }); + } + const mock = new MockTransport(); + const transport = new LoggerTransport({ transport: mock, logger, recordMessage: false }); + const receipt = await transport.sendRaw(raw(['b@example.com', 'c@example.com'])); + assert.strictEqual(receipt.successful, true); + assert.deepStrictEqual( + records.map((record) => [record.properties.event, record.properties.recipientCount]), + [ + ['email.sending', 2], + ['email.sent', 2], + ], + ); + assert.strictEqual(records[0].properties.envelope, undefined); + assert.strictEqual(records[0].properties.operation, 'sendRaw'); + records.length = 0; + await new LoggerTransport({ transport: mock, logger, recordMessage: 'inline' }).sendRaw(raw()); + assert.deepStrictEqual(records[0].properties.envelope.to, ['bob@example.com']); + assert.strictEqual(records[0].text, 'Sending email.'); + records.length = 0; + assertRefused(await new LoggerTransport({ transport: plain(), logger }).sendRaw(raw()), 'plain'); + assert.strictEqual(records.at(-1).properties.event, 'email.failed'); + const logOnly = await new LoggerTransport({ logger }).sendRaw(raw()); + assert.strictEqual(logOnly.successful, true); + await new LoggerTransport({ logger }).verify(); + await new LoggerTransport({ transport: mock, logger }).verify(); +}); + +test('Raw: SES sends a raw message as Raw content, to its envelope', async (t) => { + const requests = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + requests.push({ url: String(input), body: JSON.parse(init.body) }); + return Response.json({ MessageId: 'ses-1' }); + }); + const ses = new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'a', secretAccessKey: 's' }, + configurationSetName: 'tracking', + defaultTags: { app: 'test' }, + retries: 0, + }); + const receipt = await ses.sendRaw(raw(['b@example.com', 'c@пример.рф'])); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(receipt.messageId, 'ses-1'); + const { body, url } = requests[0]; + assert.match(url, /\/v2\/email\/outbound-emails$/); + assert.deepStrictEqual(body.Destination, { + ToAddresses: ['b@example.com', 'c@xn--e1afmkfd.xn--p1ai'], + }); + assert.strictEqual(body.FromEmailAddress, 'alex@xn--bcher-kva.de'); + assert.strictEqual(body.ConfigurationSetName, 'tracking'); + assert.deepStrictEqual(body.Tags, [{ Name: 'app', Value: 'test' }]); + const parsed = await PostalMime.parse(Buffer.from(body.Content.Raw.Data, 'base64')); + assert.strictEqual(parsed.subject, 'Raw'); + const invalid = await ses.sendRaw({ + envelope: { from: null, to: ['b@example.com'] }, + content: encoder.encode('Subject: x\r\n\r\nbare\rcr\r\n'), + }); + assert.strictEqual(invalid.successful, false); + assert.strictEqual(invalid.errors[0].category, 'validation'); + assert.strictEqual(requests.length, 1); + const nullSender = await ses.sendRaw({ + ...raw(), + envelope: { from: null, to: ['b@example.com'] }, + }); + assert.strictEqual(nullSender.successful, true); + assert.strictEqual(requests[1].body.FromEmailAddress, undefined); +}); + +test('Raw: Mailgun sends a raw message to messages.mime', async (t) => { + const requests = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + requests.push({ url: String(input), form: init.body }); + return Response.json({ id: '', message: 'Queued' }); + }); + const mailgun = new MailgunTransport({ apiKey: 'k', domain: 'mg.example.com', retries: 0 }); + const receipt = await mailgun.sendRaw(raw(['b@example.com', 'c@example.com'])); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(receipt.messageId, ''); + assert.match(requests[0].url, /\/v3\/mg\.example\.com\/messages\.mime$/); + assert.deepStrictEqual(requests[0].form.getAll('to'), ['b@example.com', 'c@example.com']); + const file = requests[0].form.get('message'); + assert.strictEqual(file.name, 'message.mime'); + assert.match(await file.text(), /^From: alex@example\.com\r\n/); + const aborted = new AbortController(); + aborted.abort(new Error('stop')); + await assert.rejects(mailgun.sendRaw(raw(), { signal: aborted.signal }), /stop/); + const invalid = await mailgun.sendRaw({ + envelope: { from: null, to: [] }, + content: new Uint8Array(), + }); + assert.strictEqual(invalid.errors[0].category, 'validation'); +}); From 77cbc2bc65f5d0a098a024750b60f950656c1bfa Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:20:44 +0300 Subject: [PATCH 015/143] feat: verify() on the HTTP providers that can check a key SES, SendGrid, Resend and Mailgun get verify(), as sently's transports have, each one authenticated read of the API through the shared retry layer, retried as a read and never taken for a delivery: SES GetAccount, SendGrid's scopes with mail.send required, Resend's domains and Mailgun's sending domain. A key known to the provider but not allowed to read (403, or Resend's restricted_api_key) is verified too, so a key that may only send is no false negative; a 401 or another status throws an error with the statusCode of the answer. Mailtrap, Maileroo, Lettermint and Plunk offer no request that checks a sending key without sending, and say so in their pages. The SES budget goes to 7.5 KB. Co-Authored-By: Claude Opus 5.5 --- docs/transports/lettermint.md | 7 ++ docs/transports/maileroo.md | 7 ++ docs/transports/mailgun.md | 7 ++ docs/transports/mailtrap.md | 7 ++ docs/transports/plunk.md | 7 ++ docs/transports/resend.md | 6 ++ docs/transports/sendgrid.md | 7 ++ docs/transports/ses.md | 7 ++ scripts/size-budgets.json | 2 +- src/internal/verify.js | 63 ++++++++++++ src/mailgun/index.d.ts | 10 +- src/mailgun/mailgun-transport.js | 22 +++++ src/resend/index.d.ts | 8 +- src/resend/resend-transport.js | 31 ++++++ src/sendgrid/index.d.ts | 10 +- src/sendgrid/sendgrid-transport.js | 34 +++++++ src/ses/index.d.ts | 8 +- src/ses/ses-transport.js | 21 ++++ tests/unit/provider-verify.test.js | 154 +++++++++++++++++++++++++++++ 19 files changed, 413 insertions(+), 5 deletions(-) create mode 100644 src/internal/verify.js create mode 100644 tests/unit/provider-verify.test.js diff --git a/docs/transports/lettermint.md b/docs/transports/lettermint.md index 8040d80..7d0fd88 100644 --- a/docs/transports/lettermint.md +++ b/docs/transports/lettermint.md @@ -212,3 +212,10 @@ const message = createMessage({ await transport.send(message); ``` + +## Verification + +This transport has no `verify()`: Lettermint offers no request that checks a sending +key without sending a message. `isVerifiableTransport()` is `false` for it, and +`./pool`, `./retry`, `./otel` and `./logger` have nothing to check when they wrap +it. diff --git a/docs/transports/maileroo.md b/docs/transports/maileroo.md index 8401d1f..4f6601b 100644 --- a/docs/transports/maileroo.md +++ b/docs/transports/maileroo.md @@ -227,3 +227,10 @@ const receipt = await transport.send(message, { signal: controller.signal, }); ``` + +## Verification + +This transport has no `verify()`: Maileroo offers no request that checks a sending +key without sending a message. `isVerifiableTransport()` is `false` for it, and +`./pool`, `./retry`, `./otel` and `./logger` have nothing to check when they wrap +it. diff --git a/docs/transports/mailgun.md b/docs/transports/mailgun.md index 4acdf83..68b996a 100644 --- a/docs/transports/mailgun.md +++ b/docs/transports/mailgun.md @@ -248,6 +248,13 @@ while retry configuration determines how many times to retry failed requests. The transport uses exponential backoff for retries, reducing load on Mailgun's servers during temporary outages. +## Verifying the configuration + +`verify()` reads the sending domain (`GET /v3/domains/`), so it checks the +key, the region and the domain. A domain sending key, which may not read the +domain (403), is verified too; a domain the account does not have fails with +404. + ## Development and testing For development and testing, you can use Mailgun's [sandbox domain] or configure diff --git a/docs/transports/mailtrap.md b/docs/transports/mailtrap.md index 95466b2..9df7f3e 100644 --- a/docs/transports/mailtrap.md +++ b/docs/transports/mailtrap.md @@ -183,3 +183,10 @@ for await (const receipt of transport.sendMany(messages)) { - `retries`: Number of retry attempts for transient failures. Default: `3`. - `headers`: Additional HTTP headers to include with every request. + +## Verification + +This transport has no `verify()`: Mailtrap offers no request that checks a sending +key without sending a message. `isVerifiableTransport()` is `false` for it, and +`./pool`, `./retry`, `./otel` and `./logger` have nothing to check when they wrap +it. diff --git a/docs/transports/plunk.md b/docs/transports/plunk.md index 7a7d75c..2fff4ee 100644 --- a/docs/transports/plunk.md +++ b/docs/transports/plunk.md @@ -301,3 +301,10 @@ delivery without affecting production systems. > sending test emails to real users. For self-hosted instances, consider > setting up separate testing environments with isolated email delivery > to prevent accidental sends to production email addresses. + +## Verification + +This transport has no `verify()`: Plunk offers no request that checks a sending +key without sending a message. `isVerifiableTransport()` is `false` for it, and +`./pool`, `./retry`, `./otel` and `./logger` have nothing to check when they wrap +it. diff --git a/docs/transports/resend.md b/docs/transports/resend.md index eca582f..e186b76 100644 --- a/docs/transports/resend.md +++ b/docs/transports/resend.md @@ -210,6 +210,12 @@ while retry configuration determines how many times to retry failed requests. The transport uses exponential backoff for retries, reducing load on Resend's servers during temporary outages and improving overall reliability. +## Verifying the configuration + +`verify()` reads the domains of the account (`GET /domains`). A key restricted to +sending, which Resend answers with `restricted_api_key`, is verified too; an +invalid key throws an error with the `statusCode` of the answer. + ## Development and testing For development and testing, you can configure the Resend transport for diff --git a/docs/transports/sendgrid.md b/docs/transports/sendgrid.md index cff53e0..38a83d3 100644 --- a/docs/transports/sendgrid.md +++ b/docs/transports/sendgrid.md @@ -207,6 +207,13 @@ while retry configuration determines how many times to retry failed requests. The transport uses exponential backoff for retries, reducing load on SendGrid's servers during temporary outages and improving overall reliability. +## Verifying the configuration + +`verify()` reads the scopes of the API key (`GET /v3/scopes`) and checks that one +of them is `mail.send`, so a restricted key that may not send fails before the +first message. It throws an error with the `statusCode` of the answer when +SendGrid refuses the key. + ## Development and testing For development and testing, you can configure the SendGrid transport for diff --git a/docs/transports/ses.md b/docs/transports/ses.md index 135a8e9..efb0f7d 100644 --- a/docs/transports/ses.md +++ b/docs/transports/ses.md @@ -378,6 +378,13 @@ configuration determines how many times to retry failed requests. The transport uses exponential backoff for retries, reducing load on SES during temporary issues and improving overall reliability. +## Verifying the configuration + +`verify()` reads the account with GetAccount, signed as a send is, so it checks +the credentials and the region. A key whose policy allows sending but not +`ses:GetAccount` gets AccessDenied, and is verified too: the signature it made was +accepted. It throws an error with the `statusCode` of the answer otherwise. + ## Development and testing For development and testing, you can configure the SES transport for testing diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 3e6b432..da9b035 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -5,7 +5,7 @@ "./smtp": 22.5, "./sendmail": 10.5, "./jmap": 12.5, - "./ses": 7, + "./ses": 7.5, "./resend": 6, "./sendgrid": 6, "./mailgun": 6, diff --git a/src/internal/verify.js b/src/internal/verify.js new file mode 100644 index 0000000..f1c1476 --- /dev/null +++ b/src/internal/verify.js @@ -0,0 +1,63 @@ +// Not ported: `verify()` of an HTTP provider, one authenticated read of its +// API, retried as a read is and never taken for a delivery. + +const { readText } = require('./body.js'); +const { requestWithRetry } = require('./http.js'); + +const MAX_DETAIL = 200; + +const detailOf = (text) => { + const trimmed = text.trim(); + return trimmed.length > MAX_DETAIL ? `${trimmed.slice(0, MAX_DETAIL)}…` : trimmed; +}; + +const failureOf = (provider, status, text) => { + const detail = detailOf(text); + const reason = + status === 401 ? 'refused the credentials' : `answered the check with HTTP ${status}`; + const error = new Error(`${provider} ${reason}${detail === '' ? '.' : `: ${detail}`}`); + error.statusCode = status; + return error; +}; + +/** + * Verifies the credentials of an HTTP provider with one read of its API. A + * 2xx verifies them; so does a 403, which says the key is known but may not + * read what was asked (a key that may only send); a 401, or any other + * status, fails. + * + * @param request `provider`, the name in the error; `send(signal)`, the + * authenticated request; `check(status, body)`, which may throw on a 2xx or + * return `true` to accept another status; and `retries`, `timeoutMs`, + * `signal` and `log` of the transport. + * @throws {Error} With the `statusCode` of the answer, if the check fails. + * @internal + */ +const verifyByRequest = async (request) => { + let answer; + try { + answer = await requestWithRetry({ + log: request.log, + signal: request.signal, + retries: request.retries, + timeoutMs: request.timeoutMs, + isIdempotent: true, + send: request.send, + read: async (response) => ({ status: response.status, body: await readText(response) }), + reject: async (response) => { + const text = await readText(response); + if (response.status === 403 || request.check?.(response.status, text) === true) { + // Not retried, as its status says, and caught below. + return Object.assign(new Error('Verified.'), { verified: true, statusCode: 403 }); + } + return failureOf(request.provider, response.status, text); + }, + }); + } catch (error) { + if (error?.verified === true) return; + throw error; + } + request.check?.(answer.status, answer.body); +}; + +module.exports = { verifyByRequest }; diff --git a/src/mailgun/index.d.ts b/src/mailgun/index.d.ts index ee85378..a2192ef 100644 --- a/src/mailgun/index.d.ts +++ b/src/mailgun/index.d.ts @@ -9,6 +9,7 @@ import type { TransportOptions, RawMessage, RawTransport, + VerifiableTransport, } from '../../index.js'; /** @@ -152,7 +153,9 @@ type ResolvedMailgunConfig = Required { +export declare class MailgunTransport + implements RawTransport<'mailgun'>, VerifiableTransport<'mailgun'> +{ readonly id = 'mailgun'; /** * The resolved Mailgun configuration used by this transport. @@ -198,6 +201,11 @@ export declare class MailgunTransport implements RawTransport<'mailgun'> { * failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Verifies the API key and the sending domain; a key of that domain that + * may only send is verified too, a domain the account does not have fails. + */ + verify(options?: TransportOptions): Promise; /** * Sends serialized MIME through the `messages.mime` endpoint, to the recipients of its * envelope. It is checked as SMTP checks it and read into memory; diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index 2cc7cce..3d6faec 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -3,6 +3,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createMailgunConfig } = require('./config.js'); +const { verifyByRequest } = require('../internal/verify.js'); const { MailgunApiError, MailgunHttpClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); @@ -71,6 +72,27 @@ class MailgunTransport { return inspectRedacted(this, options, inspect); } + /** + * Verifies the API key and the sending domain with `GET /v3/domains/`. + * A key of that domain that may only send (403) is verified too; a domain + * the account does not have fails. + * + * @param options Optional abort signal. + * @throws {Error} If Mailgun refuses the key or knows no such domain. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const url = `${this.config.baseUrl}/domains/${encodeURIComponent(this.config.domain)}`; + await verifyByRequest({ + provider: 'Mailgun', + send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } + /** * Sends a single email message via Mailgun API. * diff --git a/src/resend/index.d.ts b/src/resend/index.d.ts index e535e51..44c0b8a 100644 --- a/src/resend/index.d.ts +++ b/src/resend/index.d.ts @@ -7,6 +7,7 @@ import type { Transport, TransportDependencies, TransportOptions, + VerifiableTransport, } from '../../index.js'; /** @@ -109,7 +110,7 @@ export type ResolvedResendConfig = Required { +export declare class ResendTransport implements Transport<'resend'>, VerifiableTransport<'resend'> { readonly id = 'resend'; /** * The resolved Resend configuration used by this transport. @@ -156,6 +157,11 @@ export declare class ResendTransport implements Transport<'resend'> { * failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Verifies the API key with `GET /domains`; a key that may only send is + * verified too. + */ + verify(options?: TransportOptions): Promise; /** * Sends multiple email messages efficiently via Resend API. * diff --git a/src/resend/resend-transport.js b/src/resend/resend-transport.js index 90035cc..4b39eb2 100644 --- a/src/resend/resend-transport.js +++ b/src/resend/resend-transport.js @@ -3,6 +3,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createResendConfig } = require('./config.js'); +const { verifyByRequest } = require('../internal/verify.js'); const { ResendApiError, ResendHttpClient } = require('./http-client.js'); const { convertMessage, @@ -78,6 +79,27 @@ class ResendTransport { return inspectRedacted(this, options, inspect); } + /** + * Verifies the API key with `GET /domains`. A key that may only send, + * which Resend answers with `restricted_api_key`, is verified too. + * + * @param options Optional abort signal. + * @throws {Error} If Resend refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'Resend', + send: (signal) => + this.httpClient.fetchWithAuth(`${this.config.baseUrl}/domains`, { method: 'GET' }, signal), + check: (status, body) => status === 401 && isRestrictedKey(body), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } + /** * Sends a single email message via Resend API. * @@ -343,4 +365,13 @@ const isCallerAbort = (error, signal) => { return signal?.aborted === true && (isAbortError(error) || error === signal.reason); }; +// Resend answers a key that may only send with `restricted_api_key`. +const isRestrictedKey = (body) => { + try { + return JSON.parse(body).name === 'restricted_api_key'; + } catch { + return false; + } +}; + module.exports = { ResendTransport }; diff --git a/src/sendgrid/index.d.ts b/src/sendgrid/index.d.ts index c898608..cc8095d 100644 --- a/src/sendgrid/index.d.ts +++ b/src/sendgrid/index.d.ts @@ -7,6 +7,7 @@ import type { Transport, TransportDependencies, TransportOptions, + VerifiableTransport, } from '../../index.js'; /** @@ -141,7 +142,9 @@ type ResolvedSendGridConfig = Required { +export declare class SendGridTransport + implements Transport<'sendgrid'>, VerifiableTransport<'sendgrid'> +{ readonly id = 'sendgrid'; /** * The resolved SendGrid configuration used by this transport. @@ -187,6 +190,11 @@ export declare class SendGridTransport implements Transport<'sendgrid'> { * failure. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Verifies the API key with the scopes it has, and that one of them is + * `mail.send`. + */ + verify(options?: TransportOptions): Promise; /** * Sends multiple email messages efficiently via SendGrid API. * diff --git a/src/sendgrid/sendgrid-transport.js b/src/sendgrid/sendgrid-transport.js index 6498801..9287046 100644 --- a/src/sendgrid/sendgrid-transport.js +++ b/src/sendgrid/sendgrid-transport.js @@ -4,6 +4,7 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { createSendGridConfig } = require('./config.js'); +const { verifyByRequest } = require('../internal/verify.js'); const { SendGridApiError, SendGridHttpClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); @@ -71,6 +72,39 @@ class SendGridTransport { return inspectRedacted(this, options, inspect); } + /** + * Verifies the API key with the scopes it has (`GET /v3/scopes`), and that + * one of them is `mail.send`. + * + * @param options Optional abort signal. + * @throws {Error} If SendGrid refuses the key, or the key may not send. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'SendGrid', + send: (signal) => + this.httpClient.fetchWithAuth(`${this.config.baseUrl}/scopes`, { method: 'GET' }, signal), + check: (status, body) => { + if (status < 200 || status > 299) return false; + let scopes; + try { + scopes = JSON.parse(body).scopes; + } catch { + scopes = undefined; + } + if (!Array.isArray(scopes) || !scopes.includes('mail.send')) { + throw new Error('The SendGrid API key has no mail.send scope, so it may not send mail.'); + } + return true; + }, + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } + /** * Sends a single email message via SendGrid API. * diff --git a/src/ses/index.d.ts b/src/ses/index.d.ts index 1713519..88e435c 100644 --- a/src/ses/index.d.ts +++ b/src/ses/index.d.ts @@ -9,6 +9,7 @@ import type { TransportOptions, RawMessage, RawTransport, + VerifiableTransport, } from '../../index.js'; /** @@ -202,7 +203,7 @@ type ResolvedSesConfig = Required< * } * ``` */ -export declare class SesTransport implements RawTransport<'ses'> { +export declare class SesTransport implements RawTransport<'ses'>, VerifiableTransport<'ses'> { readonly id = 'ses'; /** Resolved configuration with defaults applied */ config: ResolvedSesConfig; @@ -240,6 +241,11 @@ export declare class SesTransport implements RawTransport<'ses'> { * @returns A promise that resolves to a receipt with the result. */ send(message: Message, options?: TransportOptions): Promise>; + /** + * Verifies the credentials and the region with GetAccount; a key that may + * send but not read the account (AccessDenied) is verified too. + */ + verify(options?: TransportOptions): Promise; /** * Sends serialized MIME as the Raw content of SendEmail, to the recipients of its * envelope. It is checked as SMTP checks it and read into memory; diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 42c56d3..01d4ad3 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -10,6 +10,7 @@ const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); +const { verifyByRequest } = require('../internal/verify.js'); /** * Amazon SES email transport implementation. @@ -82,6 +83,26 @@ class SesTransport { return inspectRedacted(this, options, inspect); } + /** + * Verifies the credentials and the region with GetAccount. A key that may + * send but not read the account (AccessDenied) is verified too. + * + * @param options Optional abort signal. + * @throws {Error} If SES refuses the credentials or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const url = `https://email.${this.config.region}.amazonaws.com/v2/email/account`; + await verifyByRequest({ + provider: 'SES', + send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } + /** * Sends a single email message through Amazon SES. * diff --git a/tests/unit/provider-verify.test.js b/tests/unit/provider-verify.test.js new file mode 100644 index 0000000..bda886c --- /dev/null +++ b/tests/unit/provider-verify.test.js @@ -0,0 +1,154 @@ +// Not ported: `verify()` of the HTTP providers (after sently's), one +// authenticated read each, with `fetch` stubbed. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { isVerifiableTransport } = require('../../index.js'); +const { SesTransport } = require('../../src/ses/index.js'); +const { SendGridTransport } = require('../../src/sendgrid/index.js'); +const { ResendTransport } = require('../../src/resend/index.js'); +const { MailgunTransport } = require('../../src/mailgun/index.js'); + +// Answers each request with the next of `answers`, and keeps the requests. +const stub = (t, ...answers) => { + const requests = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + requests.push({ url: String(input), method: init.method, headers: init.headers }); + const answer = answers.shift(); + if (answer instanceof Error) throw answer; + return answer(); + }); + return requests; +}; +const json = (status, body) => () => Response.json(body, { status }); + +const ses = () => + new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, + region: 'eu-west-1', + retries: 1, + }); + +test('Verify: every HTTP provider with a check is verifiable', () => { + for (const transport of [ + ses(), + new SendGridTransport({ apiKey: 'SG.key' }), + new ResendTransport({ apiKey: 're_key' }), + new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com' }), + ]) { + assert.ok(isVerifiableTransport(transport), transport.id); + } +}); + +test('Verify: SES reads the account, signed, and takes AccessDenied for a known key', async (t) => { + const requests = stub( + t, + json(200, { SendQuota: {} }), + json(403, { message: 'User is not authorized to perform: ses:GetAccount' }), + json(401, { message: 'The security token included in the request is invalid.' }), + ); + await ses().verify(); + assert.strictEqual(requests[0].url, 'https://email.eu-west-1.amazonaws.com/v2/email/account'); + assert.strictEqual(requests[0].method, 'GET'); + assert.match( + requests[0].headers.authorization ?? requests[0].headers.Authorization, + /^AWS4-HMAC-SHA256 /, + ); + await ses().verify(); + await assert.rejects(ses().verify(), (error) => { + assert.strictEqual(error.statusCode, 401); + assert.match(error.message, /^SES refused the credentials: .*token .* is invalid/); + return true; + }); +}); + +test('Verify: a read is retried after a network failure, never taken for a delivery', async (t) => { + // The shortest jittered wait: half of the first delay. + t.mock.method(Math, 'random', () => 0); + const requests = stub( + t, + Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } }), + json(200, {}), + ); + await ses().verify(); + assert.strictEqual(requests.length, 2); + stub(t, json(500, { message: 'oops' })); + const once = new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, + retries: 0, + }); + await assert.rejects( + once.verify(), + (error) => error.deliveryUnknown !== true && /500|oops/.test(error.message), + ); +}); + +test('Verify: SendGrid wants the mail.send scope', async (t) => { + const requests = stub( + t, + json(200, { scopes: ['mail.send', 'user.profile.read'] }), + json(200, { scopes: ['user.profile.read'] }), + json(401, { errors: [{ message: 'The provided authorization grant is invalid' }] }), + () => new Response('not json', { status: 200 }), + ); + const sendgrid = new SendGridTransport({ apiKey: 'SG.key', retries: 0 }); + await sendgrid.verify(); + assert.strictEqual(requests[0].url, 'https://api.sendgrid.com/v3/scopes'); + await assert.rejects(sendgrid.verify(), /no mail\.send scope/); + await assert.rejects(sendgrid.verify(), /SendGrid refused the credentials/); + await assert.rejects(sendgrid.verify(), /no mail\.send scope/); +}); + +test('Verify: Resend takes a sending-only key, refuses an invalid one', async (t) => { + const requests = stub( + t, + json(200, { data: [] }), + json(401, { + name: 'restricted_api_key', + message: 'This API key is restricted to only send emails', + }), + json(401, { name: 'validation_error', message: 'API key is invalid' }), + () => new Response('', { status: 401 }), + ); + const resend = new ResendTransport({ apiKey: 're_key', retries: 0 }); + await resend.verify(); + assert.strictEqual(requests[0].url, 'https://api.resend.com/domains'); + await resend.verify(); + await assert.rejects(resend.verify(), /Resend refused the credentials: .*API key is invalid/); + await assert.rejects(resend.verify(), /Resend refused the credentials/); +}); + +test('Verify: Mailgun reads the domain, takes a sending key, fails on an unknown domain', async (t) => { + const requests = stub( + t, + json(200, { domain: { name: 'mg.example.com' } }), + json(403, { message: 'Forbidden' }), + json(404, { message: 'Domain not found' }), + ); + const mailgun = new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com', retries: 0 }); + await mailgun.verify(); + assert.strictEqual(requests[0].url, 'https://api.mailgun.net/v3/domains/mg.example.com'); + await mailgun.verify(); + await assert.rejects(mailgun.verify(), (error) => { + assert.strictEqual(error.statusCode, 404); + assert.match(error.message, /Mailgun answered the check with HTTP 404: .*Domain not found/); + return true; + }); + const long = 'x'.repeat(500); + stub(t, json(400, { message: long })); + await assert.rejects(mailgun.verify(), (error) => error.message.length < 300); +}); + +test('Verify: an aborted signal rejects before any request', async (t) => { + const requests = stub(t); + const controller = new AbortController(); + controller.abort(new Error('stop')); + for (const transport of [ + ses(), + new SendGridTransport({ apiKey: 'SG.key' }), + new ResendTransport({ apiKey: 're_key' }), + new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com' }), + ]) { + await assert.rejects(transport.verify({ signal: controller.signal }), /stop/); + } + assert.strictEqual(requests.length, 0); +}); From 1ac2057817dc1d7a738f484312f4e73a0b9e0545 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:21:56 +0300 Subject: [PATCH 016/143] test(mailpit): an HTTP CONNECT proxy and sendmail against Mailpit A message goes to Mailpit through a local HTTP proxy with createHttpProxyConnector, and through ./sendmail into Mailpit's own sendmail command in its container when MAILPIT_SENDMAIL names it, with its blind recipient on the envelope. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 +- tests/mailpit/features.mailpit.test.js | 79 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index f4a9da9..3391ad8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -723,7 +723,8 @@ change. **This file wins** if they ever disagree. real provider in `pnpm test`. - **Opt-in suites**, outside the gates: `tests/mailpit/` (`pnpm test:mailpit`, serial, as the inbox is shared; `docker compose -f tests/mailpit/docker-compose.yml up -d` and - `MAILPIT_SMTP_HOST`/`MAILPIT_API_URL`) and `tests/e2e/` (`pnpm test:e2e`, each suite skipped + `MAILPIT_SMTP_HOST`/`MAILPIT_API_URL`, plus `MAILPIT_SENDMAIL="docker exec -i mailer-mailpit + /mailpit sendmail"` for the sendmail case) and `tests/e2e/` (`pnpm test:e2e`, each suite skipped until its provider's variables are set; Stalwart for JMAP in `tests/e2e/jmap/`). - **Platform-dependent tests:** the mock SMTP server takes a client that has gone as no failure: a reset (`ECONNRESET`; macOS and Windows send an RST where Linux sends a FIN) and a reply diff --git a/tests/mailpit/features.mailpit.test.js b/tests/mailpit/features.mailpit.test.js index 7c1621c..b1702c2 100644 --- a/tests/mailpit/features.mailpit.test.js +++ b/tests/mailpit/features.mailpit.test.js @@ -100,3 +100,82 @@ test( assert.equal(delivered.Attachments[0].FileName, 'data.csv'); }, ); + +test('Mailpit: a message goes through an HTTP CONNECT proxy', { skip }, async (t) => { + const http = require('node:http'); + const net = require('node:net'); + const { createHttpProxyConnector } = require('../../src/smtp/index.js'); + const tunnels = new Set(); + let connects = 0; + const proxy = http.createServer(); + proxy.on('connect', (request, client, head) => { + connects++; + tunnels.add(client); + const [host, port] = [ + request.url.slice(0, request.url.lastIndexOf(':')), + request.url.slice(request.url.lastIndexOf(':') + 1), + ]; + const upstream = net.connect(Number(port), host, () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head.length > 0) upstream.write(head); + upstream.pipe(client); + client.pipe(upstream); + }); + upstream.on('error', () => client.destroy()); + client.on('error', () => upstream.destroy()); + }); + await new Promise((resolve) => proxy.listen(0, '127.0.0.1', resolve)); + t.after(() => { + for (const tunnel of tunnels) tunnel.destroy(); + proxy.close(); + }); + const { delivered } = await deliver( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Through a proxy', + content: { text: 'Tunneled' }, + }), + { connect: createHttpProxyConnector({ url: `http://127.0.0.1:${proxy.address().port}` }) }, + ); + assert.equal(connects, 1); + assert.equal(delivered.Text.trim(), 'Tunneled'); +}); + +// MAILPIT_SENDMAIL, such as "docker exec -i mailer-mailpit /mailpit sendmail", +// runs Mailpit's own sendmail in its container. +const sendmailCommand = process.env.MAILPIT_SENDMAIL; + +test( + 'Mailpit: sendmail delivers through mailpit sendmail', + { skip: skip || !sendmailCommand }, + async () => { + const childProcess = require('node:child_process'); + const { SendmailTransport } = require('../../src/sendmail/index.js'); + const [command, ...prefix] = sendmailCommand.split(' '); + const transport = new SendmailTransport({ + spawn: (path, args, options) => childProcess.spawn(command, [...prefix, ...args], options), + }); + const config = getTestConfig(); + const client = new MailpitClient(config.mailpit); + await client.deleteAllMessages(); + const receipt = await transport.send( + createMessage({ + from: 'app@example.com', + to: 'user@example.com', + bcc: 'archive@example.com', + subject: 'Through sendmail', + content: { text: 'Piped' }, + }), + ); + assert.ok(receipt.successful, JSON.stringify(receipt)); + const delivered = await waitForMailpitDelivery(client, { subject: 'Through sendmail' }); + assert.equal(delivered.Text.trim(), 'Piped'); + assert.deepEqual( + delivered.Bcc.map((address) => address.Address), + ['archive@example.com'], + ); + // Mailpit writes a Bcc field of its own into what it stores, from the + // envelope; that the message has none is a unit test of ./sendmail. + }, +); From b0c63c9a25ec13b7fe2073367af525872d8f25ad Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:31:28 +0300 Subject: [PATCH 017/143] feat(postmark): a transport over the Postmark API A new subpath, ported from the Postmark drivers of unemail and sently: a message goes to /email, a batch of up to 500 to /email/batch with the receipt of each message's own answer, and verify reads the server of the token. The first tag is Postmark's Tag (a second is refused), metadata its Metadata, tracking TrackOpens and TrackLinks; List-* fields, priority and threading go in Headers; an ErrorCode becomes the category of the receipt (406 rejected, 10 auth, sender signatures configuration). The providers of 1.1 share src/internal/api.js: their API error, an authenticated JSON request on requestWithRetry, the receipt of a failure, attachments in base64 and header fields with names capitalized (moved to internal/headers.js from ./mime). Fixed in the port: unemail's fetch followed redirects with the credentials, read bodies whole and retried after the provider may have taken the message (U4); sently sent an inline part without `cid:` and parsed JSON before the status (S2). The shared modules grew, and the budgets of ./smtp, Resend, SendGrid and Mailgun with them. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 9 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/postmark.md | 82 ++++++++ package.json | 4 + scripts/size-budgets.json | 9 +- src/internal/api.js | 326 +++++++++++++++++++++++++++++ src/internal/config.js | 32 ++- src/internal/headers.js | 22 +- src/internal/redact.js | 9 + src/mime/message.js | 16 +- src/postmark/config.js | 40 ++++ src/postmark/http-client.js | 64 ++++++ src/postmark/index.d.ts | 69 ++++++ src/postmark/index.js | 7 + src/postmark/message-converter.js | 80 +++++++ src/postmark/postmark-transport.js | 142 +++++++++++++ tests/helpers/providers/fetch.js | 27 +++ tests/types/postmark.test-d.ts | 26 +++ tests/unit/api.test.js | 134 ++++++++++++ tests/unit/postmark.test.js | 250 ++++++++++++++++++++++ 21 files changed, 1331 insertions(+), 19 deletions(-) create mode 100644 docs/transports/postmark.md create mode 100644 src/internal/api.js create mode 100644 src/postmark/config.js create mode 100644 src/postmark/http-client.js create mode 100644 src/postmark/index.d.ts create mode 100644 src/postmark/index.js create mode 100644 src/postmark/message-converter.js create mode 100644 src/postmark/postmark-transport.js create mode 100644 tests/helpers/providers/fetch.js create mode 100644 tests/types/postmark.test-d.ts create mode 100644 tests/unit/api.test.js create mode 100644 tests/unit/postmark.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 3391ad8..0ec2d0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -496,6 +496,15 @@ fails on upstream's behavior): strip. `./sendmail` succeeds on an exit with 0 alone, reports a signal after the whole message as `deliveryUnknown`, classifies the sysexits codes, reads stderr bounded and redacted, ends the options with `--`, and puts blind recipients on the command line only. +- U4. unemail (`src/drivers/_fetch.ts`), behind every HTTP driver ported: redirects were followed, + so `X-Postmark-Server-Token`, `api-key` and the other custom credentials went wherever one + pointed; the body was read whole, with no bound; and a timeout or a 5xx after which the + provider may have taken the message was retried by the middleware. The providers of 1.1 go + through `createApiClient` (`src/internal/api.js`) on `requestWithRetry`, as the ported ones do. +- S2. sently (`src/transports/postmark.ts`): an inline part went with its Content-ID alone, while + Postmark wants `cid:` before it, and the JSON of an answer was parsed before its status, so a + non-JSON 5xx failed with a `SyntaxError` and no status. `./postmark` writes `cid:`, and the + shared client reads the status first. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index b595ddf..cb0986a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -84,6 +84,7 @@ const guide = [ { text: 'Maileroo', link: '/transports/maileroo' }, { text: 'Lettermint', link: '/transports/lettermint' }, { text: 'Plunk', link: '/transports/plunk' }, + { text: 'Postmark', link: '/transports/postmark' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 7fb86b8..60837bf 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -53,6 +53,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/maileroo` | `MailerooTransport`, `createMailerooConfig` and its error classes | [Maileroo](/transports/maileroo) | | `@alexify/mailer/lettermint` | `LettermintTransport`, `createLettermintConfig` and its error classes | [Lettermint](/transports/lettermint) | | `@alexify/mailer/plunk` | `PlunkTransport` | [Plunk](/transports/plunk) | +| `@alexify/mailer/postmark` | `PostmarkTransport` | [Postmark](/transports/postmark) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/postmark.md b/docs/transports/postmark.md new file mode 100644 index 0000000..6a2d4a4 --- /dev/null +++ b/docs/transports/postmark.md @@ -0,0 +1,82 @@ +# Postmark transport + +The Postmark transport sends through the API of [Postmark], from the Postmark +drivers of unemail and sently. + +[Postmark]: https://postmarkapp.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { PostmarkTransport } = require('@alexify/mailer/postmark'); + +const transport = new PostmarkTransport({ serverToken: process.env.POSTMARK_SERVER_TOKEN }); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

', text: 'On its way.' }, + tags: ['orders'], + metadata: { orderId: '42' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); // Postmark's MessageID +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `serverToken` | (required) | The Server API token, not the account token. | +| `messageStream` | Postmark's default | The message stream, such as `broadcast` for bulk mail. | +| `openTracking`, `clickTracking` | the server's | Tracking, unless a message sets its own `tracking`. | +| `baseUrl` | `https://api.postmarkapp.com` | HTTPS, or HTTP to this machine. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before Postmark took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- The sender and recipients go as mailboxes, display names quoted when they need + it and domains as A-labels. +- The first tag is Postmark's `Tag`; Postmark has one, so a second is refused. +- `metadata` is `Metadata`, `tracking` is `TrackOpens` and `TrackLinks` + (`HtmlAndText` or `None`). +- List-* fields, priority and threading go in `Headers`. +- An inline attachment goes with `cid:` before its Content-ID, as Postmark wants + it. +- Postmark cannot schedule a message, so `scheduledAt` is refused with a + `validation` receipt, and it has no idempotency key. + +## Errors + +Postmark answers with an `ErrorCode`, which the receipt carries as +`postmark.`, with a category: 10 is `auth`, 406 (an inactive recipient) +`rejected`, 400, 401 and 405 (sender signatures, sending not allowed) +`configuration`, and any other `validation`; none is retried. A server error or a +connection that was never made is retried; a failure after Postmark may have taken +the message is `deliveryUnknown`. + +## Batches + +`sendMany()` sends up to 500 messages a request through `/email/batch`, reading +its input a batch at a time. Each message has the receipt of its own answer, so an +inactive recipient fails its message alone; a message that cannot be converted +fails alone too. + +## Verifying the configuration + +`verify()` reads the server of the token (`GET /server`), and throws when Postmark +refuses the token. diff --git a/package.json b/package.json index 42fc702..9098be4 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,10 @@ "types": "./src/plunk/index.d.ts", "default": "./src/plunk/index.js" }, + "./postmark": { + "types": "./src/postmark/index.d.ts", + "default": "./src/postmark/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index da9b035..d9dd321 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -2,17 +2,18 @@ ".": 10, "./mime": 7.5, "./attachments": 4, - "./smtp": 22.5, + "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, "./ses": 7.5, - "./resend": 6, - "./sendgrid": 6, - "./mailgun": 6, + "./resend": 6.5, + "./sendgrid": 6.5, + "./mailgun": 6.5, "./mailtrap": 6.5, "./maileroo": 6, "./lettermint": 6, "./plunk": 5.5, + "./postmark": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/internal/api.js b/src/internal/api.js new file mode 100644 index 0000000..cb5ad29 --- /dev/null +++ b/src/internal/api.js @@ -0,0 +1,326 @@ +// Not ported: what the HTTP providers added in 1.1 share, once rather than a +// copy in each: the error of their API, an authenticated JSON request through +// `requestWithRetry`, the receipt of a failure, and the conversion of a batch +// in which a message that cannot be converted fails alone. + +const { readAttachmentContent } = require('../core/attachment.js'); +const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); +const { toBase64 } = require('./base64.js'); +const { readText } = require('./body.js'); +const { canonicalHeaderName } = require('./headers.js'); +const { convertAttachments, visitCustomHeaders, withCalendarAttachment } = require('./provider.js'); +const { isDeliveryUnknown } = require('./delivery-unknown.js'); +const { requestWithRetry } = require('./http.js'); +const { createLog } = require('./log.js'); + +const MAX_ERROR_TEXT = 500; + +/** + * An error of a provider's API: its status, its own code and the category the + * provider gives it, the wait it asks for, and the attempts made. + * + * @internal + */ +class ProviderApiError extends Error { + constructor(provider, message, details = {}) { + super(message, details.cause === undefined ? undefined : { cause: details.cause }); + this.name = 'ProviderApiError'; + this.provider = provider; + this.statusCode = details.statusCode; + this.retryAfterMilliseconds = details.retryAfterMilliseconds; + this.attempts = details.attempts; + this.category = details.category; + this.retryable = details.retryable; + this.providerDetails = details.providerDetails; + // The code of the receipt, apart from `code`, which classifyReceiptError + // reads as a system error code. + this.receiptCode = details.code; + } +} + +const parseJson = (text) => { + try { + return JSON.parse(text); + } catch { + return undefined; + } +}; + +const shorten = (text) => + text.length > MAX_ERROR_TEXT ? `${text.slice(0, MAX_ERROR_TEXT)}…` : text; + +// The message an API gives for an error, in the fields most of them use. +const defaultMessageOf = (json) => { + if (json == null || typeof json !== 'object') return undefined; + if (typeof json.message === 'string') return json.message; + if (typeof json.Message === 'string') return json.Message; + if (typeof json.error === 'string') return json.error; + if (typeof json.error?.message === 'string') return json.error.message; + const first = Array.isArray(json.errors) ? json.errors[0] : undefined; + if (typeof first === 'string') return first; + if (typeof first?.message === 'string') return first.message; + return undefined; +}; + +/** + * Makes the client of a provider's API. + * + * @param config The resolved configuration: `fetch`, `headers`, `retries`, + * `timeout` and `logger`. + * @param provider `id`, the provider's id; `label`, its name in messages; + * `authorize(headers, request)`, which adds its credentials; optional + * `messageOf(json, text)`, the message of an error answer, and + * `classify(status, json)`, `{ category, retryable, code }` of one. + * @returns `{ log, fetchWithAuth, request }`. + * @internal + */ +const createApiClient = (config, provider) => { + const log = createLog(config.logger, { transport: provider.id }); + + const fetchWithAuth = async (url, options, signal) => { + const headers = new Headers(options.headers); + await provider.authorize(headers, { url, method: options.method ?? 'GET', body: options.body }); + for (const name in config.headers) { + if (Object.hasOwn(config.headers, name)) headers.set(name, config.headers[name]); + } + // A redirect comes back as a response rather than being followed, so the + // credentials in the headers never reach another origin. + const fetch = config.fetch ?? globalThis.fetch; + return fetch(url, { ...options, headers, signal, redirect: 'manual' }); + }; + + const rejectionOf = (response, text, attempt) => { + const json = parseJson(text); + const message = + (provider.messageOf ?? defaultMessageOf)(json, text) ?? + (text.trim() === '' ? `HTTP ${response.status}` : shorten(text.trim())); + const classified = provider.classify?.(response.status, json) ?? {}; + return new ProviderApiError(provider.id, `${provider.label}: ${message}`, { + statusCode: response.status, + retryAfterMilliseconds: parseRetryAfter(response.headers.get('Retry-After')), + attempts: attempt + 1, + category: classified.category, + retryable: classified.retryable, + code: classified.code, + providerDetails: json ?? undefined, + }); + }; + + /** + * Sends one request and reads its answer: `{ status, headers, json }` of a + * 2xx, an empty body as `null`. A 2xx whose body is not JSON fails as of + * unknown delivery, as the provider may have taken the request. + */ + const request = (options) => { + const isForm = options.body instanceof FormData; + const headers = { Accept: 'application/json', ...options.headers }; + let body; + if (isForm) body = options.body; + else if (options.body !== undefined) { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify(options.body); + } + return requestWithRetry({ + log, + signal: options.signal, + onRetry: options.onRetry, + retries: config.retries, + timeoutMs: config.timeout, + hasIdempotencyKey: options.hasIdempotencyKey === true, + isIdempotent: options.isIdempotent === true, + send: (signal) => + fetchWithAuth(options.url, { method: options.method ?? 'POST', headers, body }, signal), + read: async (response) => { + const text = await readText(response); + const json = text.trim() === '' ? null : parseJson(text); + if (json === undefined) { + throw new ProviderApiError(provider.id, `${provider.label} answered with no JSON.`, { + statusCode: response.status, + }); + } + return { status: response.status, headers: response.headers, json }; + }, + reject: async (response, attempt) => rejectionOf(response, await readText(response), attempt), + createTimeoutError: () => + new ProviderApiError( + provider.id, + `${provider.label} API request timed out after ${config.timeout} ms.`, + ), + finalError: (error, attempts) => + error instanceof ProviderApiError + ? error + : new ProviderApiError(provider.id, `${provider.label}: ${error.message}`, { + attempts, + cause: error, + }), + }); + }; + + return { log, fetchWithAuth, request }; +}; + +/** + * The receipt of a failure, with what the error says of it. + * + * @param provider The provider's id. + * @param error What was thrown. + * @returns A failed receipt. + * @internal + */ +const providerFailure = (provider, error) => { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof ProviderApiError) { + return createFailedReceipt(message, { + provider, + deliveryUnknown: isDeliveryUnknown(error), + statusCode: error.statusCode, + retryAfterMilliseconds: error.retryAfterMilliseconds, + attempts: error.attempts, + category: error.category, + retryable: error.retryable, + code: error.receiptCode, + providerDetails: error.providerDetails, + }); + } + return createFailedReceipt(message, { provider, deliveryUnknown: isDeliveryUnknown(error) }); +}; + +/** + * Whether an error is the abort of the caller's own signal. + * + * @internal + */ +const isCallerAbort = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +/** + * Converts each message of a batch, one that fails to convert failing alone. + * + * @param messages The messages. + * @param convert Converts one message. + * @param provider The provider's id, for the receipt of a failure. + * @param signal Rethrows the caller's abort. + * @returns `{ payloads, slots }`: a slot is the index of a payload, or the + * receipt of a message that did not convert. + * @internal + */ +const convertBatch = async (messages, convert, provider, signal) => { + const payloads = []; + const slots = []; + for (let i = 0; i < messages.length; i++) { + try { + payloads.push(await convert(messages[i])); + slots.push(payloads.length - 1); + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + slots.push(providerFailure(provider, error)); + } + } + return { payloads, slots }; +}; + +/** + * The receipts of a batch in the order of its messages: those of + * `receiptOf(index)` for the payloads that were sent, and the failures of the + * others. + * + * @internal + */ +const mergeBatch = function* (slots, receiptOf) { + for (let i = 0; i < slots.length; i++) { + yield typeof slots[i] === 'number' ? receiptOf(slots[i]) : slots[i]; + } +}; + +/** + * Checks the credential a provider needs when the transport is made. + * + * @internal + */ +const checkCredential = (value, name, label) => { + if (typeof value !== 'string' || value === '') { + throw new TypeError(`${label} needs ${name}.`); + } + return value; +}; + +/** + * The attachments of a message, an iCalendar part among them, each with its + * content in base64, read concurrently. + * + * @param message The message. + * @param signal Cancels the reads. + * @returns `{ attachment, content }` of each. + * @internal + */ +const base64Attachments = (message, signal) => + convertAttachments( + withCalendarAttachment(message), + async (attachment, readSignal) => ({ + attachment, + content: toBase64(await readAttachmentContent(attachment.content, readSignal)), + }), + signal, + ); + +// The fields an API sets from its own parameters. +const STANDARD_HEADERS = new Set([ + 'from', + 'to', + 'cc', + 'bcc', + 'reply-to', + 'subject', + 'content-type', + 'content-transfer-encoding', + 'mime-version', +]); +const isStandardHeader = (name) => STANDARD_HEADERS.has(name.toLowerCase()); + +const PRIORITY_HEADERS = { + high: [ + ['X-Priority', '1'], + ['X-MSMail-Priority', 'High'], + ['Importance', 'High'], + ], + low: [ + ['X-Priority', '5'], + ['X-MSMail-Priority', 'Low'], + ['Importance', 'Low'], + ], +}; + +/** + * The header fields a provider is given besides those it sets itself: the + * priority, the custom fields (List-* among them) and the threading fields, + * names capitalized. + * + * @param message The message. + * @returns `{ name, value }` of each. + * @internal + */ +const headerFieldsOf = (message) => { + const fields = []; + const priority = PRIORITY_HEADERS[message.priority]; + if (priority !== undefined) { + for (let i = 0; i < priority.length; i++) { + fields.push({ name: priority[i][0], value: priority[i][1] }); + } + } + visitCustomHeaders(message, isStandardHeader, (name, value) => { + fields.push({ name: canonicalHeaderName(name.toLowerCase()), value }); + }); + return fields; +}; + +module.exports = { + ProviderApiError, + base64Attachments, + checkCredential, + convertBatch, + createApiClient, + headerFieldsOf, + isCallerAbort, + mergeBatch, + providerFailure, +}; diff --git a/src/internal/config.js b/src/internal/config.js index a3050fb..599d7c0 100644 --- a/src/internal/config.js +++ b/src/internal/config.js @@ -60,4 +60,34 @@ const checkConcurrency = (concurrency) => { } }; -module.exports = { checkConcurrency, checkRequestOptions, checkSecureUrl }; +/** + * Checks an option that is a non-empty string when it is given. + * + * @internal + */ +const checkOptionalString = (value, name) => { + if (value !== undefined && (typeof value !== 'string' || value === '')) { + throw new TypeError(`${name} must be a non-empty string.`); + } + return value; +}; + +/** + * Checks an option that is a boolean when it is given. + * + * @internal + */ +const checkOptionalBoolean = (value, name) => { + if (value !== undefined && typeof value !== 'boolean') { + throw new TypeError(`${name} must be a boolean.`); + } + return value; +}; + +module.exports = { + checkConcurrency, + checkOptionalBoolean, + checkOptionalString, + checkRequestOptions, + checkSecureUrl, +}; diff --git a/src/internal/headers.js b/src/internal/headers.js index cd73e5b..8223dfe 100644 --- a/src/internal/headers.js +++ b/src/internal/headers.js @@ -1,5 +1,8 @@ // Not ported: small helpers of header names that the converters of // the HTTP providers share. +// Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the case of a +// field name, `_normalizeHeaderKey`. +// https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman /** * The names, in lower case, as a set built in one pass. @@ -14,4 +17,21 @@ const lowerCaseNames = (names) => { return set; }; -module.exports = { lowerCaseNames }; +// Each word of a field name capitalized, as nodemailer writes it, the +// abbreviations its readers know upper-cased: `Headers` keeps names in lower +// case, and `list-unsubscribe-post` reads as a field nobody named. +const UPPER_CASE_PARTS = + /^x-smtpapi$|^(?:mime|dkim|arc|bimi)\b|^[a-z]|-(?:spf|fbl|id|md5)$|-[a-z]/g; +/** + * A field name as nodemailer writes it. + * + * @param name A field name in lower case, as `Headers` keeps it. + * @returns The name capitalized. + * @internal + */ +const canonicalHeaderName = (name) => + name === 'content-features' + ? 'Content-features' + : name.replace(UPPER_CASE_PARTS, (part) => part.toUpperCase()); + +module.exports = { canonicalHeaderName, lowerCaseNames }; diff --git a/src/internal/redact.js b/src/internal/redact.js index c3c1345..99f9e84 100644 --- a/src/internal/redact.js +++ b/src/internal/redact.js @@ -13,13 +13,17 @@ const REDACTED = '[REDACTED]'; // The credential fields of the transports' configurations, and the header // names that carry credentials, compared in lower case. const SECRET_KEYS = new Set([ + 'accesskey', 'accesstoken', + 'api-key', 'api-token', 'apikey', + 'apisecret', 'apitoken', 'authorization', 'bearertoken', 'clientsecret', + 'connectionstring', 'cookie', 'pass', 'password', @@ -28,10 +32,15 @@ const SECRET_KEYS = new Set([ 'refreshtoken', 'secret', 'secretaccesskey', + 'secretkey', + 'servertoken', 'sessiontoken', 'token', 'x-api-key', + 'x-auth-token', 'x-lettermint-token', + 'x-postmark-server-token', + 'x-smtp2go-api-key', ]); // Plain objects and class instances are copied; built-in values such as a diff --git a/src/mime/message.js b/src/mime/message.js index 9b2dd2a..7f33d65 100644 --- a/src/mime/message.js +++ b/src/mime/message.js @@ -1,8 +1,7 @@ // Ported from upyo v0.7.0 (packages/mime/src/message.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -// Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the case of -// a custom field name, `_normalizeHeaderKey`, and the transfer encoding of a -// text body, `getTransferEncoding` and `_getTextEncoding`. +// Also ported from nodemailer v10.0.16 (src/mime-node/index.ts): the transfer +// encoding of a text body, `getTransferEncoding` and `_getTextEncoding`. // https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman const { Buffer } = require('node:buffer'); @@ -15,6 +14,7 @@ const { resolveThreadingHeaders, } = require('../core/message-id.js'); const { resolveCalendarContent } = require('../core/calendar.js'); +const { canonicalHeaderName } = require('../internal/headers.js'); const { createLog } = require('../internal/log.js'); const { checkDkimConfig } = require('./dkim/types.js'); const { resolvePrivateKey } = require('./dkim/sign.js'); @@ -247,16 +247,6 @@ const pushHeaderFields = (lines, message, generateId) => { } }; -// Each word of a field name capitalized, as nodemailer writes it, the -// abbreviations its readers know upper-cased: `Headers` keeps names in lower -// case, and `list-unsubscribe-post` reads as a field nobody named. -const UPPER_CASE_PARTS = - /^x-smtpapi$|^(?:mime|dkim|arc|bimi)\b|^[a-z]|-(?:spf|fbl|id|md5)$|-[a-z]/g; -const canonicalHeaderName = (name) => - name === 'content-features' - ? 'Content-features' - : name.replace(UPPER_CASE_PARTS, (part) => part.toUpperCase()); - const encodeAddressList = (addresses) => { let list = ''; for (let i = 0; i < addresses.length; i++) { diff --git a/src/postmark/config.js b/src/postmark/config.js new file mode 100644 index 0000000..65989c9 --- /dev/null +++ b/src/postmark/config.js @@ -0,0 +1,40 @@ +// Ported from unemail v0.7.0 (src/drivers/postmark.ts): the options of the +// Postmark driver, the server token and the message stream. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkOptionalBoolean, + checkOptionalString, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +/** + * Resolves the options of the Postmark transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createPostmarkConfig = (config = {}) => { + const resolved = { + serverToken: checkCredential(config.serverToken, 'serverToken', 'PostmarkTransport'), + baseUrl: config.baseUrl ?? 'https://api.postmarkapp.com', + messageStream: checkOptionalString(config.messageStream, 'messageStream'), + openTracking: checkOptionalBoolean(config.openTracking, 'openTracking'), + clickTracking: checkOptionalBoolean(config.clickTracking, 'clickTracking'), + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createPostmarkConfig }; diff --git a/src/postmark/http-client.js b/src/postmark/http-client.js new file mode 100644 index 0000000..ad209a3 --- /dev/null +++ b/src/postmark/http-client.js @@ -0,0 +1,64 @@ +// Ported from unemail v0.7.0 (src/drivers/postmark.ts): the requests of the +// Postmark API and the category of its error codes. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers, so a +// redirect is not followed, the body is bounded, and a failure after Postmark +// may have taken the message is of unknown delivery rather than retried. + +const { createApiClient } = require('../internal/api.js'); + +// What an ErrorCode of Postmark says, where it says more than its status. +const ERROR_CODES = new Map([ + [10, { category: 'auth', retryable: false }], + [300, { category: 'validation', retryable: false }], + [400, { category: 'configuration', retryable: false }], + [401, { category: 'configuration', retryable: false }], + [405, { category: 'configuration', retryable: false }], + [406, { category: 'rejected', retryable: false }], +]); + +/** + * The category of a Postmark ErrorCode, a non-retryable `validation` for one + * it does not name. + * + * @internal + */ +const classifyErrorCode = (errorCode) => + ERROR_CODES.get(errorCode) ?? { category: 'validation', retryable: false }; + +/** + * The client of the Postmark API. + * + * @internal + */ +const createPostmarkClient = (config) => { + const client = createApiClient(config, { + id: 'postmark', + label: 'Postmark', + authorize: (headers) => headers.set('X-Postmark-Server-Token', config.serverToken), + classify: (status, json) => { + const code = json?.ErrorCode; + if (typeof code !== 'number' || code === 0) return undefined; + return { ...classifyErrorCode(code), code: `postmark.${code}` }; + }, + }); + return { + ...client, + send: (payload, options) => + client.request({ + url: `${config.baseUrl}/email`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }), + sendBatch: (payloads, options) => + client.request({ + url: `${config.baseUrl}/email/batch`, + body: payloads, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { classifyErrorCode, createPostmarkClient }; diff --git a/src/postmark/index.d.ts b/src/postmark/index.d.ts new file mode 100644 index 0000000..a4286b8 --- /dev/null +++ b/src/postmark/index.d.ts @@ -0,0 +1,69 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link PostmarkTransport}. + */ +export interface PostmarkConfig extends TransportDependencies { + /** The Server API token, not the account token. */ + readonly serverToken: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.postmarkapp.com" + */ + readonly baseUrl?: string; + /** + * The message stream, such as `broadcast` for bulk mail; Postmark's default + * transactional stream when it is not set. + */ + readonly messageStream?: string; + /** Open tracking, unless a message sets its own `tracking`. */ + readonly openTracking?: boolean; + /** Link tracking (HTML and text), unless a message sets its own. */ + readonly clickTracking?: boolean; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before Postmark took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * A transport over the API of Postmark. A message goes to `/email`, a batch + * of up to 500 to `/email/batch`, each with the receipt of its own answer. + * The first tag is Postmark's `Tag` (a second is refused), `metadata` its + * `Metadata`, an inline part a `cid:` ContentID. Postmark cannot schedule a + * message, and has no idempotency key. + * + * @example + * ```js + * const { PostmarkTransport } = require('@alexify/mailer/postmark'); + * const transport = new PostmarkTransport({ serverToken: process.env.POSTMARK_TOKEN }); + * ``` + */ +export declare class PostmarkTransport implements VerifiableTransport<'postmark'> { + readonly id = 'postmark'; + constructor(config: PostmarkConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the server token by reading its server (`GET /server`). */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/postmark/index.js b/src/postmark/index.js new file mode 100644 index 0000000..17634a7 --- /dev/null +++ b/src/postmark/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/postmark.ts): the entry of the +// Postmark transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { PostmarkTransport } = require('./postmark-transport.js'); + +module.exports = { PostmarkTransport }; diff --git a/src/postmark/message-converter.js b/src/postmark/message-converter.js new file mode 100644 index 0000000..705af92 --- /dev/null +++ b/src/postmark/message-converter.js @@ -0,0 +1,80 @@ +// Ported from unemail v0.7.0 (src/drivers/postmark.ts): a message as the +// request of Postmark's email API, `toPayload`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/postmark.ts): the inline +// attachments and the reply-to addresses. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: display names are quoted when they need it; an inline +// part goes with `cid:` before its Content-ID, which sently left out; a second +// tag is refused rather than sent as metadata, as Postmark has one `Tag`; and +// a scheduled message is refused. + +const { formatMailbox } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { formatAddresses } = require('../internal/provider.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +const addressList = (addresses) => formatAddresses(addresses, formatMailbox).join(', '); + +const trackingOf = (message, config) => { + const opens = message.tracking?.opens ?? config.openTracking; + const clicks = message.tracking?.clicks ?? config.clickTracking; + return { opens, clicks }; +}; + +/** + * Converts a message to a request of Postmark's `/email`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {TypeError} If the message is scheduled. + * @throws {RangeError} If it has more than one tag. + */ +const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'postmark'); + if (message.tags.length > 1) throw new RangeError('Postmark takes one tag per message.'); + const payload = { + From: formatMailbox(message.sender), + To: addressList(message.recipients), + Subject: message.subject, + }; + if (message.ccRecipients.length > 0) payload.Cc = addressList(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.Bcc = addressList(message.bccRecipients); + if (message.replyRecipients.length > 0) payload.ReplyTo = addressList(message.replyRecipients); + if ('html' in message.content) { + payload.HtmlBody = message.content.html; + if (message.content.text) payload.TextBody = message.content.text; + } else { + payload.TextBody = message.content.text; + } + const fields = headerFieldsOf(message); + if (fields.length > 0) { + payload.Headers = []; + for (const field of fields) payload.Headers.push({ Name: field.name, Value: field.value }); + } + if (message.tags.length === 1) payload.Tag = message.tags[0]; + if (message.metadata !== undefined) payload.Metadata = { ...message.metadata }; + const tracking = trackingOf(message, config); + if (tracking.opens !== undefined) payload.TrackOpens = tracking.opens; + if (tracking.clicks !== undefined) payload.TrackLinks = tracking.clicks ? 'HtmlAndText' : 'None'; + if (config.messageStream !== undefined) payload.MessageStream = config.messageStream; + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.Attachments = []; + for (const item of attachments) { + const attachment = item.attachment; + const part = { + Name: attachment.filename, + Content: item.content, + ContentType: attachment.contentType, + }; + if (attachment.inline) part.ContentID = `cid:${attachment.contentId}`; + payload.Attachments.push(part); + } + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js new file mode 100644 index 0000000..598f054 --- /dev/null +++ b/src/postmark/postmark-transport.js @@ -0,0 +1,142 @@ +// Ported from unemail v0.7.0 (src/drivers/postmark.ts): a transport over the +// Postmark API, single and batch sends. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/postmark.ts): `verify`, the +// server of the token. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: a failure in the answer of one message of a batch is +// its receipt alone, a message that cannot be converted fails alone too, and +// sently's JSON parse before the status check is gone with the shared client. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { chunks } = require('../internal/batch.js'); +const { convertBatch, isCallerAbort, mergeBatch, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createPostmarkConfig } = require('./config.js'); +const { classifyErrorCode, createPostmarkClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +// The most messages one request of `/email/batch` takes. +const MAX_BATCH_SIZE = 500; + +// The receipt of an answer of Postmark: an `ErrorCode` other than 0 fails it, +// though it came in a 200. +const receiptOf = (entry) => { + if (entry == null || typeof entry !== 'object') { + return createFailedReceipt('Postmark gave no answer for the message.', { + provider: 'postmark', + deliveryUnknown: true, + }); + } + if (entry.ErrorCode !== 0 && entry.ErrorCode !== undefined) { + return createFailedReceipt(`Postmark: ${entry.Message ?? `ErrorCode ${entry.ErrorCode}`}`, { + provider: 'postmark', + ...classifyErrorCode(entry.ErrorCode), + code: `postmark.${entry.ErrorCode}`, + providerDetails: entry, + }); + } + return { successful: true, messageId: entry.MessageID, provider: 'postmark' }; +}; + +/** + * A transport over the API of Postmark (postmarkapp.com). + */ +class PostmarkTransport { + id = 'postmark'; + + constructor(config) { + this.config = createPostmarkConfig(config); + this.httpClient = createPostmarkClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/email`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with Postmark's MessageID. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const answer = await this.httpClient.send(payload, options); + return receiptOf(answer.json); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('postmark', error); + } + } + + /** + * Sends messages with `/email/batch`, 500 a request, reading the input a + * batch at a time; each message has the receipt of its own answer. + * + * @param messages The messages. + * @param options Optional abort signal and retry callback. + * @returns The receipts, in the order of the messages. + */ + async *sendMany(messages, options) { + const signal = options?.signal; + for await (const batch of chunks(messages, MAX_BATCH_SIZE)) { + signal?.throwIfAborted(); + const converted = await convertBatch( + batch, + (message) => convertMessage(message, this.config, signal), + 'postmark', + signal, + ); + if (converted.payloads.length === 0) { + yield* mergeBatch(converted.slots, () => undefined); + continue; + } + let answers; + let failure; + try { + const answer = await this.httpClient.sendBatch(converted.payloads, options); + answers = Array.isArray(answer.json) ? answer.json : []; + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + failure = providerFailure('postmark', error); + } + yield* mergeBatch(converted.slots, (index) => failure ?? receiptOf(answers[index])); + } + } + + /** + * Verifies the server token by reading its server (`GET /server`). + * + * @param options Optional abort signal. + * @throws {Error} If Postmark refuses the token or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'Postmark', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/server`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { PostmarkTransport }; diff --git a/tests/helpers/providers/fetch.js b/tests/helpers/providers/fetch.js new file mode 100644 index 0000000..50a2797 --- /dev/null +++ b/tests/helpers/providers/fetch.js @@ -0,0 +1,27 @@ +// Not ported: a stubbed fetch for the tests of the HTTP providers of 1.1, +// keeping each request with its headers and its body. + +const stubFetch = (t, respond) => { + const requests = []; + t.mock.method(globalThis, 'fetch', async (input, init = {}) => { + const request = { + url: String(input), + method: init.method ?? 'GET', + headers: new Headers(init.headers), + redirect: init.redirect, + body: init.body, + json: typeof init.body === 'string' ? JSON.parse(init.body) : undefined, + }; + requests.push(request); + return respond(request, requests.length - 1); + }); + return requests; +}; + +const json = (body, status = 200, headers = {}) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json', ...headers }, + }); + +module.exports = { json, stubFetch }; diff --git a/tests/types/postmark.test-d.ts b/tests/types/postmark.test-d.ts new file mode 100644 index 0000000..3e42bc1 --- /dev/null +++ b/tests/types/postmark.test-d.ts @@ -0,0 +1,26 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { PostmarkTransport } from '../../src/postmark/index.js'; + +const transport = new PostmarkTransport({ + serverToken: 'token', + messageStream: 'broadcast', + openTracking: true, + clickTracking: false, + timeout: 10000, + retries: 2, + headers: { 'X-Team': 'mail' }, +}); +expectType<'postmark'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new PostmarkTransport({})); +expectError(new PostmarkTransport({ serverToken: 'token', openTracking: 'yes' })); diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js new file mode 100644 index 0000000..a1342e5 --- /dev/null +++ b/tests/unit/api.test.js @@ -0,0 +1,134 @@ +// Not ported: the shared client of the HTTP providers of 1.1 +// (src/internal/api.js), against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { + ProviderApiError, + checkCredential, + createApiClient, + providerFailure, +} = require('../../src/internal/api.js'); +const { isDeliveryUnknown } = require('../../src/internal/delivery-unknown.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const client = (config = {}, provider = {}) => + createApiClient( + { retries: 0, timeout: 1000, headers: {}, ...config }, + { + id: 'demo', + label: 'Demo', + authorize: (headers) => headers.set('Authorization', 'Bearer key'), + ...provider, + }, + ); + +test('API: the message of an error comes from the fields most APIs use', async (t) => { + const answers = [ + json({ message: 'a' }, 400), + json({ Message: 'b' }, 400), + json({ error: 'c' }, 400), + json({ error: { message: 'd' } }, 400), + json({ errors: ['e'] }, 400), + json({ errors: [{ message: 'f' }] }, 400), + json({ other: true }, 400), + new Response('plain text', { status: 400 }), + new Response('', { status: 418 }), + new Response('x'.repeat(700), { status: 400 }), + ]; + stubFetch(t, () => answers.shift()); + const messages = []; + for (let i = 0; i < 10; i++) { + await client() + .request({ url: 'https://api.example.com/send', body: {} }) + .catch((error) => messages.push(error.message)); + } + assert.deepStrictEqual(messages.slice(0, 9), [ + 'Demo: a', + 'Demo: b', + 'Demo: c', + 'Demo: d', + 'Demo: e', + 'Demo: f', + 'Demo: {"other":true}', + 'Demo: plain text', + 'Demo: HTTP 418', + ]); + assert.ok(messages[9].length < 520); +}); + +test('API: headers of the config, a form body, an empty answer', async (t) => { + const requests = stubFetch(t, () => new Response(null, { status: 202 })); + const form = new FormData(); + form.append('to', 'a@example.com'); + const answer = await client({ headers: { 'X-Team': 'mail' } }).request({ + url: 'https://api.example.com/send', + body: form, + headers: { 'X-One': '1' }, + }); + assert.deepStrictEqual({ status: answer.status, json: answer.json }, { status: 202, json: null }); + const [request] = requests; + assert.strictEqual(request.body, form); + assert.strictEqual(request.headers.get('x-team'), 'mail'); + assert.strictEqual(request.headers.get('x-one'), '1'); + assert.strictEqual(request.headers.get('authorization'), 'Bearer key'); + assert.strictEqual(request.headers.get('content-type'), null); +}); + +test('API: a timeout, and a network failure once retries are spent', async (t) => { + t.mock.method( + globalThis, + 'fetch', + (input, init) => + new Promise((resolve, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }), + ); + await assert.rejects( + client({ timeout: 20 }).request({ url: 'https://api.example.com/send', body: {} }), + (error) => { + assert.ok(error instanceof ProviderApiError); + assert.strictEqual(error.message, 'Demo API request timed out after 20 ms.'); + return true; + }, + ); + const refused = Object.assign(new TypeError('fetch failed'), { + cause: Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' }), + }); + t.mock.method(globalThis, 'fetch', async () => { + throw refused; + }); + await assert.rejects( + client().request({ url: 'https://api.example.com/send', body: {}, isIdempotent: true }), + (error) => { + assert.ok(error instanceof ProviderApiError); + assert.strictEqual(error.message, 'Demo: fetch failed'); + assert.strictEqual(error.cause, refused); + assert.strictEqual(error.attempts, 1); + return true; + }, + ); +}); + +test('API: the receipt of a failure carries what the error says', () => { + const error = new ProviderApiError('demo', 'Demo: refused', { + statusCode: 422, + category: 'rejected', + retryable: false, + code: 'demo.7', + providerDetails: { reason: 'x' }, + }); + const receipt = providerFailure('demo', error); + assert.strictEqual(receipt.errors[0].category, 'rejected'); + assert.strictEqual(receipt.errors[0].code, 'demo.7'); + assert.strictEqual(receipt.errors[0].statusCode, 422); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(isDeliveryUnknown(error), false); + const plain = providerFailure('demo', 'just text'); + assert.deepStrictEqual(plain.errorMessages, ['just text']); + assert.throws( + () => checkCredential(undefined, 'apiKey', 'DemoTransport'), + /DemoTransport needs apiKey/, + ); + assert.throws(() => checkCredential('', 'apiKey', 'DemoTransport'), TypeError); + assert.strictEqual(checkCredential('k', 'apiKey', 'DemoTransport'), 'k'); +}); diff --git a/tests/unit/postmark.test.js b/tests/unit/postmark.test.js new file mode 100644 index 0000000..095f6cb --- /dev/null +++ b/tests/unit/postmark.test.js @@ -0,0 +1,250 @@ +// Not ported: tests of ./postmark, ported from unemail's and sently's Postmark +// drivers, against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { PostmarkTransport } = require('../../src/postmark/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const transport = (config = {}) => + new PostmarkTransport({ serverToken: 'server-token', retries: 0, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop, Inc.', address: 'shop@example.com' }, + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const ok = (id = 'pm-1') => json({ ErrorCode: 0, Message: 'OK', MessageID: id }); + +test('Postmark: a message is the request of /email, with the token and no redirect', async (t) => { + const requests = stubFetch(t, () => ok()); + const receipt = await transport({ messageStream: 'outbound', openTracking: true }).send( + message({ + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: 'support@example.com', + priority: 'high', + tags: ['orders'], + metadata: { orderId: '42' }, + tracking: { clicks: true }, + list: { unsubscribe: 'https://example.com/u', oneClick: true }, + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1, 2]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'pm-1', provider: 'postmark' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.postmarkapp.com/email'); + assert.strictEqual(request.method, 'POST'); + assert.strictEqual(request.redirect, 'manual'); + assert.strictEqual(request.headers.get('x-postmark-server-token'), 'server-token'); + assert.strictEqual(request.headers.get('accept'), 'application/json'); + const body = request.json; + assert.strictEqual(body.From, '"Shop, Inc." '); + assert.strictEqual(body.To, 'bob@example.com, Carol '); + assert.strictEqual(body.Cc, 'dave@example.com'); + assert.strictEqual(body.Bcc, 'erin@example.com'); + assert.strictEqual(body.ReplyTo, 'support@example.com'); + assert.strictEqual(body.TextBody, 'Shipped'); + assert.strictEqual(body.HtmlBody, '

Shipped

'); + assert.strictEqual(body.Tag, 'orders'); + assert.deepStrictEqual(body.Metadata, { orderId: '42' }); + assert.strictEqual(body.TrackOpens, true); + assert.strictEqual(body.TrackLinks, 'HtmlAndText'); + assert.strictEqual(body.MessageStream, 'outbound'); + const headers = Object.fromEntries(body.Headers.map((header) => [header.Name, header.Value])); + assert.strictEqual(headers['X-Priority'], '1'); + assert.strictEqual(headers['List-Unsubscribe'], ''); + assert.strictEqual(headers['List-Unsubscribe-Post'], 'List-Unsubscribe=One-Click'); + assert.deepStrictEqual(body.Attachments, [ + { Name: 'r.txt', Content: 'aGk=', ContentType: 'text/plain' }, + // sently sent the Content-ID without `cid:`, which Postmark needs. + { + Name: 'logo.png', + Content: 'AQI=', + ContentType: 'image/png', + ContentID: 'cid:logo@example.com', + }, + ]); +}); + +test('Postmark: a text-only message, a calendar, and tracking turned off', async (t) => { + const requests = stubFetch(t, () => ok()); + await transport({ clickTracking: true }).send( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Meeting', + content: { text: 'See you' }, + tracking: { clicks: false }, + calendar: { + content: + 'BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//t//EN\r\nMETHOD:REQUEST\r\nBEGIN:VEVENT\r\nUID:1\r\nDTSTAMP:20261017T000000Z\r\nDTSTART:20261018T090000Z\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n', + }, + }), + ); + const body = requests[0].json; + assert.strictEqual(body.TextBody, 'See you'); + assert.strictEqual(body.HtmlBody, undefined); + assert.strictEqual(body.TrackLinks, 'None'); + assert.strictEqual(body.Attachments[0].Name, 'invite.ics'); + assert.strictEqual(body.Headers, undefined); +}); + +test('Postmark: an ErrorCode says what failed', async (t) => { + const answers = [ + json( + { + ErrorCode: 406, + Message: 'You tried to send to recipient(s) that have been marked as inactive.', + }, + 422, + ), + json({ ErrorCode: 300, Message: 'Invalid email request' }, 422), + json({ ErrorCode: 10, Message: 'No Account or Server API tokens were supplied' }, 401), + json({ ErrorCode: 401, Message: 'Sender signature not confirmed' }, 422), + ]; + stubFetch(t, () => answers.shift()); + const inactive = await transport().send(message()); + assert.strictEqual(inactive.errors[0].category, 'rejected'); + assert.strictEqual(inactive.errors[0].code, 'postmark.406'); + assert.strictEqual(inactive.retryable, false); + assert.match(inactive.errorMessages[0], /^Postmark: You tried to send/); + assert.strictEqual((await transport().send(message())).errors[0].category, 'validation'); + assert.strictEqual((await transport().send(message())).errors[0].category, 'auth'); + assert.strictEqual((await transport().send(message())).errors[0].category, 'configuration'); +}); + +test('Postmark: a server error is retried, a broken 2xx is of unknown delivery', async (t) => { + t.mock.method(Math, 'random', () => 0); + const answers = [ + json({ Message: 'oops' }, 503), + ok('pm-2'), + new Response('', { status: 200 }), + ]; + const requests = stubFetch(t, () => answers.shift()); + const retried = await transport({ retries: 1 }).send(message()); + assert.strictEqual(retried.messageId, 'pm-2'); + assert.strictEqual(requests.length, 2); + const broken = await transport().send(message()); + assert.strictEqual(broken.successful, false); + assert.strictEqual(broken.deliveryUnknown, true); + assert.strictEqual(broken.retryable, false); +}); + +test('Postmark: a batch goes to /email/batch, each message with its own answer', async (t) => { + const requests = stubFetch(t, (request) => + json( + request.json.map((payload, index) => + index === 1 + ? { ErrorCode: 406, Message: 'Inactive recipient', To: payload.To } + : { ErrorCode: 0, Message: 'OK', MessageID: `pm-${index}` }, + ), + ), + ); + const receipts = []; + for await (const receipt of transport().sendMany([ + message(), + message({ tags: ['a', 'b'] }), + message(), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, false, true], + ); + assert.strictEqual(receipts[0].messageId, 'pm-0'); + assert.match(receipts[1].errorMessages[0], /one tag per message/); + assert.strictEqual(receipts[2].errors[0].category, 'rejected'); + assert.strictEqual(receipts[3].messageId, 'pm-2'); + assert.strictEqual(requests.length, 1); + assert.strictEqual(requests[0].url, 'https://api.postmarkapp.com/email/batch'); + assert.strictEqual(requests[0].json.length, 3); +}); + +test('Postmark: 500 a batch, a failed request fails its batch, a missing answer is unknown', async (t) => { + let calls = 0; + const requests = stubFetch(t, (request) => { + calls++; + if (calls === 2) return json({ Message: 'Too many batch messages', ErrorCode: 410 }, 422); + return json(request.json.slice(1).map(() => ({ ErrorCode: 0, MessageID: 'x' }))); + }); + const messages = []; + for (let i = 0; i < 502; i++) messages.push(message()); + const receipts = []; + for await (const receipt of transport().sendMany(messages)) receipts.push(receipt); + assert.deepStrictEqual( + requests.map((request) => request.json.length), + [500, 2], + ); + // The first answer leaves out one message: unknown, not failed for sure. + assert.strictEqual(receipts[499].deliveryUnknown, true); + assert.strictEqual(receipts[500].successful, false); + assert.strictEqual(receipts[500].errors[0].code, 'postmark.410'); + for await (const receipt of transport().sendMany([message({ tags: ['a', 'b'] })])) { + assert.match(receipt.errorMessages[0], /one tag/); + } + assert.strictEqual(requests.length, 2); +}); + +test('Postmark: a scheduled message and two tags are refused before any request', async (t) => { + const requests = stubFetch(t, () => ok()); + const scheduled = await transport().send(message({ scheduledAt: new Date() })); + assert.match(scheduled.errorMessages[0], /Invalid scheduledAt: postmark/); + const tagged = await transport().send(message({ tags: ['a', 'b'] })); + assert.match(tagged.errorMessages[0], /one tag per message/); + assert.strictEqual(requests.length, 0); +}); + +test('Postmark: verify reads the server of the token', async (t) => { + const answers = [ + json({ ID: 1, Name: 'Server' }), + json({ ErrorCode: 10, Message: 'Bad token' }, 401), + ]; + const requests = stubFetch(t, () => answers.shift()); + await transport().verify(); + assert.strictEqual(requests[0].url, 'https://api.postmarkapp.com/server'); + assert.strictEqual(requests[0].method, 'GET'); + await assert.rejects(transport().verify(), /Postmark refused the credentials/); +}); + +test('Postmark: the options are checked, and the token never shows', () => { + assert.throws(() => new PostmarkTransport({}), /needs serverToken/); + assert.throws(() => transport({ baseUrl: 'http://api.example.com' }), /HTTPS/); + assert.throws(() => transport({ messageStream: '' }), /messageStream must be a non-empty string/); + assert.throws(() => transport({ openTracking: 'yes' }), /openTracking must be a boolean/); + assert.throws(() => transport({ timeout: -1 }), /timeout/); + const made = transport(); + assert.strictEqual(made.id, 'postmark'); + assert.ok(!JSON.stringify(made).includes('server-token')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('server-token')); +}); + +test('Postmark: an abort rejects with its reason', async (t) => { + stubFetch(t, () => ok()); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(transport().send(message(), { signal: controller.signal }), /stop/); + await assert.rejects(transport().verify({ signal: controller.signal }), /stop/); + const sending = transport().sendMany([message()], { signal: controller.signal }); + await assert.rejects(sending.next(), /stop/); +}); From 343ef24e41cf51a81a625b9a3f44bb9998eeee08 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:34:15 +0300 Subject: [PATCH 018/143] feat(brevo): a transport over the Brevo API A new subpath, ported from the Brevo drivers of unemail and sently: /v3/smtp/email with the api-key, scheduledAt as Brevo's scheduledAt, tags as its tags, metadata in X-Mailin-custom, List-* fields, priority and threading in its headers, a sandbox switch, and verify reading the account. Every request carries an idempotency key among the headers of the message, as Brevo's documentation places it, as a UUID (toUuid, from unemail), generated when the message has none, so a request whose delivery is unknown is retried safely; Brevo's duplicate_parameter is reported as deliveryUnknown. A second reply-to address and an inline attachment, which Brevo cannot take, are refused rather than dropped. Fixed in the port (U5): unemail grouped a batch into messageVersions by the file names and lengths of the attachments, so one recipient could get another's file, and its batch keys could collide. sendMany sends single requests, `concurrency` at a time. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 ++ docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/brevo.md | 81 ++++++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/brevo/brevo-transport.js | 117 ++++++++++++++++++++ src/brevo/config.js | 40 +++++++ src/brevo/http-client.js | 49 +++++++++ src/brevo/index.d.ts | 72 +++++++++++++ src/brevo/index.js | 7 ++ src/brevo/message-converter.js | 77 ++++++++++++++ src/internal/uuid.js | 28 +++++ tests/types/brevo.test-d.ts | 18 ++++ tests/unit/brevo.test.js | 188 +++++++++++++++++++++++++++++++++ 15 files changed, 690 insertions(+) create mode 100644 docs/transports/brevo.md create mode 100644 src/brevo/brevo-transport.js create mode 100644 src/brevo/config.js create mode 100644 src/brevo/http-client.js create mode 100644 src/brevo/index.d.ts create mode 100644 src/brevo/index.js create mode 100644 src/brevo/message-converter.js create mode 100644 src/internal/uuid.js create mode 100644 tests/types/brevo.test-d.ts create mode 100644 tests/unit/brevo.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 0ec2d0a..ee8f675 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -505,6 +505,12 @@ fails on upstream's behavior): Postmark wants `cid:` before it, and the JSON of an answer was parsed before its status, so a non-JSON 5xx failed with a `SyntaxError` and no status. `./postmark` writes `cid:`, and the shared client reads the status first. +- U5. unemail (`src/drivers/brevo.ts`): a batch went as the `messageVersions` of one request + when its messages agreed on the file names and lengths of their attachments, so a recipient + could get another's file, and the idempotency key of a batch folded only the keys present, so + two batches could share one. `./brevo` sends single requests, each with its own key, generated + when the message has none, so a retry after an unknown delivery cannot send twice; Brevo's + `duplicate_parameter` is reported as `deliveryUnknown`. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index cb0986a..c61ac92 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -85,6 +85,7 @@ const guide = [ { text: 'Lettermint', link: '/transports/lettermint' }, { text: 'Plunk', link: '/transports/plunk' }, { text: 'Postmark', link: '/transports/postmark' }, + { text: 'Brevo', link: '/transports/brevo' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 60837bf..af5c138 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -54,6 +54,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/lettermint` | `LettermintTransport`, `createLettermintConfig` and its error classes | [Lettermint](/transports/lettermint) | | `@alexify/mailer/plunk` | `PlunkTransport` | [Plunk](/transports/plunk) | | `@alexify/mailer/postmark` | `PostmarkTransport` | [Postmark](/transports/postmark) | +| `@alexify/mailer/brevo` | `BrevoTransport` | [Brevo](/transports/brevo) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/brevo.md b/docs/transports/brevo.md new file mode 100644 index 0000000..6608cec --- /dev/null +++ b/docs/transports/brevo.md @@ -0,0 +1,81 @@ +# Brevo transport + +The Brevo transport sends through the transactional email API of [Brevo] +(formerly Sendinblue), from the Brevo drivers of unemail and sently. + +[Brevo]: https://www.brevo.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { BrevoTransport } = require('@alexify/mailer/brevo'); + +const transport = new BrevoTransport({ apiKey: process.env.BREVO_API_KEY }); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + tags: ['orders'], + metadata: { orderId: '42' }, + idempotencyKey: 'order-42-shipped', + }), +); +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `apiKey` | (required) | A transactional API key (`xkeysib-…`). | +| `sandbox` | `false` | Brevo checks each request in full and drops it (`X-Sib-Sandbox: drop`). | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `baseUrl` | `https://api.brevo.com` | HTTPS, or HTTP to this machine. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- `scheduledAt` schedules the message (`scheduledAt` of Brevo). +- `tags` are Brevo's tags, and `metadata` goes in `X-Mailin-custom`, which Brevo + returns in its webhook events. +- List-* fields, priority and threading go in `headers`. +- Brevo has no per-message tracking switch, so `tracking` is not sent. +- Brevo takes one reply-to address and no inline attachment, so a second reply-to + address and an inline attachment are refused with a failed receipt rather than + dropped. + +## Idempotency + +Every request carries an idempotency key among the headers of the message, where +Brevo's documentation puts it: the message's `idempotencyKey`, or one generated +with `generateId`, turned into the UUID Brevo takes (the same key, the same UUID). +So a request that timed out, which Brevo may have taken, is retried safely, and +Brevo answers a repeated key with `duplicate_parameter`, which the receipt reports +as `deliveryUnknown` with the code `brevo.duplicate_parameter`: the message went +with the earlier request and is not sent again. + +## Errors and batches + +A `code` of Brevo becomes the receipt's `brevo.`: `unauthorized` and +`permission_denied` are `auth`, `not_enough_credits` and +`account_under_validation` `configuration`. `sendMany()` sends `concurrency` +single requests at a time, each with its own key and receipt, rather than the +`messageVersions` of one request, which share the attachments of the request. + +## Verifying the configuration + +`verify()` reads the account of the key (`GET /v3/account`). diff --git a/package.json b/package.json index 9098be4..4abbd9f 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,10 @@ "types": "./src/postmark/index.d.ts", "default": "./src/postmark/index.js" }, + "./brevo": { + "types": "./src/brevo/index.d.ts", + "default": "./src/brevo/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d9dd321..742394e 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -14,6 +14,7 @@ "./lettermint": 6, "./plunk": 5.5, "./postmark": 7, + "./brevo": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/brevo/brevo-transport.js b/src/brevo/brevo-transport.js new file mode 100644 index 0000000..81837c0 --- /dev/null +++ b/src/brevo/brevo-transport.js @@ -0,0 +1,117 @@ +// Ported from unemail v0.7.0 (src/drivers/brevo.ts): a transport over the +// Brevo API. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/brevo.ts): `verify`, the +// account of the key. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: a batch goes as concurrent single requests, each with +// its idempotency key, rather than as the `messageVersions` of one request, +// whose grouping unemail took by file name and length of the attachments, so +// one recipient could get another's file; and a repeated idempotency key is a +// delivery already made, not a failure to retry. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createBrevoConfig } = require('./config.js'); +const { createBrevoClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +const failureOf = (error) => { + if (error?.receiptCode === 'brevo.duplicate_parameter') { + // Brevo took a request with this key before: the message went then. + return createFailedReceipt( + 'Brevo already took a message with this idempotency key; it is not sent again.', + { provider: 'brevo', deliveryUnknown: true, code: 'brevo.duplicate_parameter' }, + ); + } + return providerFailure('brevo', error); +}; + +/** + * A transport over the transactional email API of Brevo (formerly + * Sendinblue). + */ +class BrevoTransport { + id = 'brevo'; + + constructor(config) { + this.config = createBrevoConfig(config); + this.httpClient = createBrevoClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/v3/smtp/email`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with Brevo's messageId. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const answer = await this.httpClient.send(payload, options); + const messageId = answer.json?.messageId ?? answer.json?.messageIds?.[0]; + if (typeof messageId !== 'string') { + return createFailedReceipt('Brevo answered without a messageId.', { + provider: 'brevo', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'brevo' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return failureOf(error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the API key by reading its account (`GET /v3/account`). + * + * @param options Optional abort signal. + * @throws {Error} If Brevo refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'Brevo', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/v3/account`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { BrevoTransport }; diff --git a/src/brevo/config.js b/src/brevo/config.js new file mode 100644 index 0000000..c459f01 --- /dev/null +++ b/src/brevo/config.js @@ -0,0 +1,40 @@ +// Ported from unemail v0.7.0 (src/drivers/brevo.ts): the options of the Brevo +// driver. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkConcurrency, + checkOptionalBoolean, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +/** + * Resolves the options of the Brevo transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createBrevoConfig = (config = {}) => { + const resolved = { + apiKey: checkCredential(config.apiKey, 'apiKey', 'BrevoTransport'), + baseUrl: config.baseUrl ?? 'https://api.brevo.com', + sandbox: checkOptionalBoolean(config.sandbox, 'sandbox') ?? false, + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createBrevoConfig }; diff --git a/src/brevo/http-client.js b/src/brevo/http-client.js new file mode 100644 index 0000000..c3026f7 --- /dev/null +++ b/src/brevo/http-client.js @@ -0,0 +1,49 @@ +// Ported from unemail v0.7.0 (src/drivers/brevo.ts): the requests of the Brevo +// API and the category of its error codes. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers, a +// request with an idempotency key retried after a failure that leaves its +// delivery unknown. + +const { createApiClient } = require('../internal/api.js'); + +// What a `code` of Brevo says, where it says more than its status. +const ERROR_CODES = new Map([ + ['unauthorized', { category: 'auth', retryable: false }], + ['permission_denied', { category: 'auth', retryable: false }], + ['account_under_validation', { category: 'configuration', retryable: false }], + ['not_enough_credits', { category: 'configuration', retryable: false }], +]); + +/** + * The client of the Brevo API. + * + * @internal + */ +const createBrevoClient = (config) => { + const client = createApiClient(config, { + id: 'brevo', + label: 'Brevo', + authorize: (headers) => headers.set('api-key', config.apiKey), + classify: (status, json) => { + const code = typeof json?.code === 'string' ? json.code : undefined; + if (code === undefined) return undefined; + return { ...ERROR_CODES.get(code), code: `brevo.${code}` }; + }, + }); + return { + ...client, + send: (payload, options) => + client.request({ + url: `${config.baseUrl}/v3/smtp/email`, + body: payload, + // The key is in the headers of the message: Brevo answers its repeat + // with duplicate_parameter rather than sending again. + hasIdempotencyKey: true, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createBrevoClient }; diff --git a/src/brevo/index.d.ts b/src/brevo/index.d.ts new file mode 100644 index 0000000..897e187 --- /dev/null +++ b/src/brevo/index.d.ts @@ -0,0 +1,72 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link BrevoTransport}. + */ +export interface BrevoConfig extends TransportDependencies { + /** A transactional API key (`xkeysib-…`). */ + readonly apiKey: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.brevo.com" + */ + readonly baseUrl?: string; + /** + * Brevo's sandbox: each request is checked in full and then dropped, + * nothing delivered (`X-Sib-Sandbox: drop`). + * @default false + */ + readonly sandbox?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request; with its idempotency key, one whose delivery is + * unknown is retried too. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * A transport over the transactional email API of Brevo. Every request + * carries an idempotency key, the message's or a generated one, as the UUID + * Brevo takes; `scheduledAt` schedules the message, `tags` are Brevo's tags + * and `metadata` goes in `X-Mailin-custom`. A second reply-to address and an + * inline attachment, which Brevo cannot take, are refused. + * + * @example + * ```js + * const { BrevoTransport } = require('@alexify/mailer/brevo'); + * const transport = new BrevoTransport({ apiKey: process.env.BREVO_API_KEY }); + * ``` + */ +export declare class BrevoTransport implements VerifiableTransport<'brevo'> { + readonly id = 'brevo'; + constructor(config: BrevoConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the API key by reading its account (`GET /v3/account`). */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/brevo/index.js b/src/brevo/index.js new file mode 100644 index 0000000..5e135c3 --- /dev/null +++ b/src/brevo/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/brevo.ts): the entry of the Brevo +// transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { BrevoTransport } = require('./brevo-transport.js'); + +module.exports = { BrevoTransport }; diff --git a/src/brevo/message-converter.js b/src/brevo/message-converter.js new file mode 100644 index 0000000..c41ad2a --- /dev/null +++ b/src/brevo/message-converter.js @@ -0,0 +1,77 @@ +// Ported from unemail v0.7.0 (src/drivers/brevo.ts): a message as the request +// of Brevo's transactional email API, `toPayload`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/brevo.ts): the reply-to +// address. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the idempotency key goes among the headers of the +// message, where Brevo's documentation puts it, as a UUID, generated when the +// message has none so that a retry cannot send twice; a second reply-to +// address and an inline attachment, which Brevo cannot take, are refused +// rather than dropped (sently dropped the custom headers too); `metadata` +// goes in `X-Mailin-custom`, which Brevo returns in its events. + +const { generateId } = require('../core/id.js'); +const { toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { toUuid } = require('../internal/uuid.js'); + +const addressOf = (address) => { + const email = toAsciiAddress(address.address); + return address.name ? { email, name: address.name } : { email }; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +/** + * Converts a message to a request of Brevo's `/v3/smtp/email`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message has two reply-to addresses or an + * inline attachment. + */ +const convertMessage = async (message, config, signal) => { + if (message.replyRecipients.length > 1) { + throw new RangeError('Brevo takes one reply-to address.'); + } + const payload = { + sender: addressOf(message.sender), + to: addressesOf(message.recipients), + subject: message.subject, + }; + if (message.ccRecipients.length > 0) payload.cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.bcc = addressesOf(message.bccRecipients); + if (message.replyRecipients.length === 1) payload.replyTo = addressOf(message.replyRecipients[0]); + if ('html' in message.content) payload.htmlContent = message.content.html; + if (message.content.text) payload.textContent = message.content.text; + const headers = {}; + for (const field of headerFieldsOf(message)) headers[field.name] = field.value; + if (message.metadata !== undefined) headers['X-Mailin-custom'] = JSON.stringify(message.metadata); + if (config.sandbox) headers['X-Sib-Sandbox'] = 'drop'; + headers.idempotencyKey = toUuid( + message.idempotencyKey ?? generateId('idempotency-key', config.generateId), + ); + payload.headers = headers; + if (message.tags.length > 0) payload.tags = [...message.tags]; + if (message.scheduledAt !== undefined) payload.scheduledAt = message.scheduledAt.toISOString(); + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.attachment = []; + for (const item of attachments) { + if (item.attachment.inline) { + throw new RangeError('Brevo cannot send an inline attachment, for a cid: URL.'); + } + payload.attachment.push({ name: item.attachment.filename, content: item.content }); + } + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/internal/uuid.js b/src/internal/uuid.js new file mode 100644 index 0000000..a3a9473 --- /dev/null +++ b/src/internal/uuid.js @@ -0,0 +1,28 @@ +// Ported from unemail v0.7.0 (src/drivers/_uuid.ts): an idempotency key as a +// UUID, for the providers that take no other (Brevo, Azure). +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the hash is the synchronous SHA-256 of node:crypto. + +const { createHash } = require('node:crypto'); + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * A key as a UUID: a UUID as it is, anything else hashed into a well-formed + * version 4 UUID, the same for the same key and another for any other. + * + * @param key An idempotency key. + * @returns The UUID. + * @internal + */ +const toUuid = (key) => { + if (UUID.test(key)) return key.toLowerCase(); + const bytes = createHash('sha256').update(key, 'utf8').digest(); + // The version and variant nibbles, as a provider that checks the shape wants. + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString('hex', 0, 16); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +}; + +module.exports = { toUuid }; diff --git a/tests/types/brevo.test-d.ts b/tests/types/brevo.test-d.ts new file mode 100644 index 0000000..45d5175 --- /dev/null +++ b/tests/types/brevo.test-d.ts @@ -0,0 +1,18 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { BrevoTransport } from '../../src/brevo/index.js'; + +const transport = new BrevoTransport({ apiKey: 'xkeysib-key', sandbox: true, concurrency: 3 }); +expectType<'brevo'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new BrevoTransport({})); +expectError(new BrevoTransport({ apiKey: 'k', sandbox: 'yes' })); diff --git a/tests/unit/brevo.test.js b/tests/unit/brevo.test.js new file mode 100644 index 0000000..48400ae --- /dev/null +++ b/tests/unit/brevo.test.js @@ -0,0 +1,188 @@ +// Not ported: tests of ./brevo, ported from unemail's and sently's Brevo +// drivers, against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { BrevoTransport } = require('../../src/brevo/index.js'); +const { toUuid } = require('../../src/internal/uuid.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const transport = (config = {}) => + new BrevoTransport({ + apiKey: 'xkeysib-secret', + retries: 0, + generateId: () => 'gen-1', + ...config, + }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const created = (id = '<1@smtp-relay.mailin.fr>') => json({ messageId: id }, 201); + +test('Brevo: a message is the request of /v3/smtp/email, with the key', async (t) => { + const requests = stubFetch(t, () => created()); + const scheduledAt = new Date('2026-10-18T09:00:00Z'); + const receipt = await transport({ sandbox: true }).send( + message({ + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: 'support@example.com', + priority: 'low', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + idempotencyKey: 'order-42', + scheduledAt, + list: { unsubscribe: 'https://example.com/u' }, + attachments: { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + }), + ); + assert.deepStrictEqual(receipt, { + successful: true, + messageId: '<1@smtp-relay.mailin.fr>', + provider: 'brevo', + }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.brevo.com/v3/smtp/email'); + assert.strictEqual(request.headers.get('api-key'), 'xkeysib-secret'); + assert.strictEqual(request.redirect, 'manual'); + const body = request.json; + assert.deepStrictEqual(body.sender, { email: 'shop@example.com', name: 'Shop' }); + assert.deepStrictEqual(body.to, [ + { email: 'bob@example.com' }, + { email: 'carol@xn--bcher-kva.de', name: 'Carol' }, + ]); + assert.deepStrictEqual(body.cc, [{ email: 'dave@example.com' }]); + assert.deepStrictEqual(body.bcc, [{ email: 'erin@example.com' }]); + assert.deepStrictEqual(body.replyTo, { email: 'support@example.com' }); + assert.strictEqual(body.htmlContent, '

Shipped

'); + assert.strictEqual(body.textContent, 'Shipped'); + assert.deepStrictEqual(body.tags, ['orders', 'eu']); + assert.strictEqual(body.scheduledAt, '2026-10-18T09:00:00.000Z'); + assert.deepStrictEqual(body.attachment, [{ name: 'r.txt', content: 'aGk=' }]); + assert.strictEqual(body.headers['X-Priority'], '5'); + assert.strictEqual(body.headers['List-Unsubscribe'], ''); + assert.strictEqual(body.headers['X-Mailin-custom'], '{"orderId":"42"}'); + assert.strictEqual(body.headers['X-Sib-Sandbox'], 'drop'); + // The key of the message, as the UUID Brevo takes, among its headers. + assert.strictEqual(body.headers.idempotencyKey, toUuid('order-42')); +}); + +test('Brevo: a message without a key gets one, the same through retries', async (t) => { + t.mock.method(Math, 'random', () => 0); + const answers = [json({ code: 'internal', message: 'busy' }, 504), created('<2@x>')]; + const requests = stubFetch(t, () => answers.shift()); + const receipt = await transport({ retries: 1 }).send(message()); + assert.strictEqual(receipt.messageId, '<2@x>'); + assert.strictEqual(requests.length, 2); + assert.strictEqual(requests[0].json.headers.idempotencyKey, toUuid('gen-1')); + assert.strictEqual(requests[1].json.headers.idempotencyKey, toUuid('gen-1')); +}); + +test('Brevo: a repeated key is a delivery already made, not a failure to retry', async (t) => { + stubFetch(t, () => + json({ code: 'duplicate_parameter', message: 'idempotencyKey already used' }, 400), + ); + const receipt = await transport().send(message({ idempotencyKey: 'order-42' })); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.deliveryUnknown, true); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(receipt.errors[0].code, 'brevo.duplicate_parameter'); +}); + +test('Brevo: an error code says what failed', async (t) => { + const answers = [ + json({ code: 'unauthorized', message: 'Key not found' }, 401), + json({ code: 'not_enough_credits', message: 'No credits' }, 402), + json({ code: 'invalid_parameter', message: 'email is not valid in to' }, 400), + json({ messageIds: ['<3@x>'] }, 201), + json({}, 201), + ]; + stubFetch(t, () => answers.shift()); + const auth = await transport().send(message()); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errors[0].code, 'brevo.unauthorized'); + assert.strictEqual((await transport().send(message())).errors[0].category, 'configuration'); + const invalid = await transport().send(message()); + assert.strictEqual(invalid.errors[0].category, 'validation'); + assert.match(invalid.errorMessages[0], /^Brevo: email is not valid in to/); + assert.strictEqual((await transport().send(message())).messageId, '<3@x>'); + const empty = await transport().send(message()); + assert.strictEqual(empty.deliveryUnknown, true); +}); + +test('Brevo: a second reply-to and an inline part are refused, not dropped', async (t) => { + const requests = stubFetch(t, () => created()); + const replies = await transport().send(message({ replyTo: ['a@example.com', 'b@example.com'] })); + assert.match(replies.errorMessages[0], /one reply-to address/); + const inline = await transport().send( + message({ + attachments: { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array(1), + }, + }), + ); + assert.match(inline.errorMessages[0], /cannot send an inline attachment/); + assert.strictEqual(requests.length, 0); +}); + +test('Brevo: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => created(`<${index++}@x>`)); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ replyTo: ['a@example.com', 'b@example.com'] }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('Brevo: verify reads the account, the options are checked, the key never shows', async (t) => { + const answers = [json({ email: 'me@example.com' }), json({ code: 'unauthorized' }, 401)]; + const requests = stubFetch(t, () => answers.shift()); + await transport().verify(); + assert.strictEqual(requests[0].url, 'https://api.brevo.com/v3/account'); + await assert.rejects(transport().verify(), /Brevo refused the credentials/); + assert.throws(() => new BrevoTransport({}), /needs apiKey/); + assert.throws(() => transport({ sandbox: 1 }), /sandbox must be a boolean/); + assert.throws(() => transport({ concurrency: 0 }), /concurrency/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('xkeysib-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('xkeysib-secret')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); + +test('UUID: a key becomes a stable version 4 UUID, a UUID stays as it is', () => { + const uuid = toUuid('order-42'); + assert.match(uuid, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + assert.strictEqual(toUuid('order-42'), uuid); + assert.notStrictEqual(toUuid('order-43'), uuid); + assert.strictEqual( + toUuid('3F2504E0-4F89-11D3-9A0C-0305E82C3301'), + '3f2504e0-4f89-11d3-9a0c-0305e82c3301', + ); +}); From c8b0913e033617e6f051510846d521271183ac0a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:37:14 +0300 Subject: [PATCH 019/143] feat(mailjet): a transport over the Mailjet Send API v3.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new subpath, ported from unemail's Mailjet driver: Messages is the batch, sent in requests of at most 50 recipients as Mailjet counts them, and a 400 carries the answer of every message, so each gets its own receipt, those that went succeeding. The first tag is the CustomCampaign (a second is refused), metadata the EventPayload, tracking TrackOpens and TrackClicks only with an HTML part, an inline part an InlinedAttachment; the fields Mailjet sets itself (List-Id, X-Mailer, …), a second reply-to address, a subject over 255 characters and a scheduled message are refused before the request. The key pair goes as UTF-8 Basic authentication; verify reads the profile. Co-Authored-By: Claude Opus 5.5 --- docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/mailjet.md | 78 +++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/mailjet/config.js | 40 ++++++ src/mailjet/http-client.js | 46 +++++++ src/mailjet/index.d.ts | 75 +++++++++++ src/mailjet/index.js | 7 + src/mailjet/mailjet-transport.js | 199 +++++++++++++++++++++++++++ src/mailjet/message-converter.js | 167 +++++++++++++++++++++++ tests/types/mailjet.test-d.ts | 22 +++ tests/unit/mailjet.test.js | 224 +++++++++++++++++++++++++++++++ 13 files changed, 865 insertions(+) create mode 100644 docs/transports/mailjet.md create mode 100644 src/mailjet/config.js create mode 100644 src/mailjet/http-client.js create mode 100644 src/mailjet/index.d.ts create mode 100644 src/mailjet/index.js create mode 100644 src/mailjet/mailjet-transport.js create mode 100644 src/mailjet/message-converter.js create mode 100644 tests/types/mailjet.test-d.ts create mode 100644 tests/unit/mailjet.test.js diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c61ac92..f6088ba 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -86,6 +86,7 @@ const guide = [ { text: 'Plunk', link: '/transports/plunk' }, { text: 'Postmark', link: '/transports/postmark' }, { text: 'Brevo', link: '/transports/brevo' }, + { text: 'Mailjet', link: '/transports/mailjet' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index af5c138..2360f04 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -55,6 +55,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/plunk` | `PlunkTransport` | [Plunk](/transports/plunk) | | `@alexify/mailer/postmark` | `PostmarkTransport` | [Postmark](/transports/postmark) | | `@alexify/mailer/brevo` | `BrevoTransport` | [Brevo](/transports/brevo) | +| `@alexify/mailer/mailjet` | `MailjetTransport` | [Mailjet](/transports/mailjet) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/mailjet.md b/docs/transports/mailjet.md new file mode 100644 index 0000000..72a1160 --- /dev/null +++ b/docs/transports/mailjet.md @@ -0,0 +1,78 @@ +# Mailjet transport + +The Mailjet transport sends through the Send API v3.1 of [Mailjet], from the +Mailjet driver of unemail. + +[Mailjet]: https://www.mailjet.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { MailjetTransport } = require('@alexify/mailer/mailjet'); + +const transport = new MailjetTransport({ + apiKey: process.env.MJ_APIKEY_PUBLIC, + apiSecret: process.env.MJ_APIKEY_PRIVATE, +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Autumn sale', + content: { html: '

…

', text: '…' }, + tags: ['autumn-sale'], + metadata: { segment: 'returning' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); // the MessageUUID +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `apiKey`, `apiSecret` | (required) | The key pair, sent with Basic authentication. | +| `sandbox` | `false` | `SandboxMode`: each message is checked and dropped; the receipt id is `sandbox-…`. | +| `openTracking`, `clickTracking` | the account's | Tracking of an HTML message, unless it sets its own `tracking`. | +| `baseUrl` | `https://api.mailjet.com` | HTTPS, or HTTP to this machine. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before Mailjet took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- The first tag is the `CustomCampaign`; Mailjet has one, so a second is refused. +- `metadata` is the `EventPayload`, which Mailjet returns in its events. +- `tracking` is `TrackOpens` and `TrackClicks`, sent only with an HTML part, as + Mailjet refuses them without one. +- An inline attachment goes in `InlinedAttachments` with its Content-ID. +- Mailjet sets some fields itself and fails a message that gives one (`send-0011`): + `List-Id`, `X-Mailer`, `Date` and others are refused before the request, so a + message with `list.id` cannot go through Mailjet. +- Mailjet takes one reply-to address, a subject of at most 255 characters and 50 + recipients a message; it cannot schedule a message. Each is refused with a + failed receipt. + +## Batches and errors + +`Messages` is Mailjet's batch: `sendMany()` sends requests of at most 50 +recipients, as Mailjet counts them over the whole request, reading its input a +batch at a time. When one message of a request fails, Mailjet answers 400 with +the answers of all of them, so each message gets the receipt of its own answer: +those that went succeed, the others fail with the code Mailjet gives +(`mailjet.send-0003`) and the fields it names. + +## Verifying the configuration + +`verify()` reads the profile of the key pair (`GET /v3/REST/myprofile`). diff --git a/package.json b/package.json index 4abbd9f..09d0a1f 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,10 @@ "types": "./src/brevo/index.d.ts", "default": "./src/brevo/index.js" }, + "./mailjet": { + "types": "./src/mailjet/index.d.ts", + "default": "./src/mailjet/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 742394e..5c66b35 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -15,6 +15,7 @@ "./plunk": 5.5, "./postmark": 7, "./brevo": 7, + "./mailjet": 8, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/mailjet/config.js b/src/mailjet/config.js new file mode 100644 index 0000000..979dd96 --- /dev/null +++ b/src/mailjet/config.js @@ -0,0 +1,40 @@ +// Ported from unemail v0.7.0 (src/drivers/mailjet.ts): the options of the +// Mailjet driver, the key pair and the sandbox. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkOptionalBoolean, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +/** + * Resolves the options of the Mailjet transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createMailjetConfig = (config = {}) => { + const resolved = { + apiKey: checkCredential(config.apiKey, 'apiKey', 'MailjetTransport'), + apiSecret: checkCredential(config.apiSecret, 'apiSecret', 'MailjetTransport'), + baseUrl: config.baseUrl ?? 'https://api.mailjet.com', + sandbox: checkOptionalBoolean(config.sandbox, 'sandbox') ?? false, + openTracking: checkOptionalBoolean(config.openTracking, 'openTracking'), + clickTracking: checkOptionalBoolean(config.clickTracking, 'clickTracking'), + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createMailjetConfig }; diff --git a/src/mailjet/http-client.js b/src/mailjet/http-client.js new file mode 100644 index 0000000..0ab0831 --- /dev/null +++ b/src/mailjet/http-client.js @@ -0,0 +1,46 @@ +// Ported from unemail v0.7.0 (src/drivers/mailjet.ts): the requests of the +// Mailjet Send API v3.1. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers, with +// the key pair encoded as UTF-8 for Basic authentication. + +const { Buffer } = require('node:buffer'); +const { createApiClient } = require('../internal/api.js'); + +// The message of an answer: the errors of its first failed message. +const messageOf = (json) => { + const messages = Array.isArray(json?.Messages) ? json.Messages : []; + for (const entry of messages) { + if (entry?.Status === 'success') continue; + const error = Array.isArray(entry?.Errors) ? entry.Errors[0] : undefined; + if (error?.ErrorMessage) return error.ErrorMessage; + } + return json?.ErrorMessage; +}; + +/** + * The client of the Mailjet API. + * + * @internal + */ +const createMailjetClient = (config) => { + const authorization = `Basic ${Buffer.from(`${config.apiKey}:${config.apiSecret}`, 'utf8').toString('base64')}`; + const client = createApiClient(config, { + id: 'mailjet', + label: 'Mailjet', + authorize: (headers) => headers.set('Authorization', authorization), + messageOf, + }); + return { + ...client, + send: (messages, options) => + client.request({ + url: `${config.baseUrl}/v3.1/send`, + body: config.sandbox ? { Messages: messages, SandboxMode: true } : { Messages: messages }, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createMailjetClient }; diff --git a/src/mailjet/index.d.ts b/src/mailjet/index.d.ts new file mode 100644 index 0000000..5c8e1fc --- /dev/null +++ b/src/mailjet/index.d.ts @@ -0,0 +1,75 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link MailjetTransport}. + */ +export interface MailjetConfig extends TransportDependencies { + /** The API key, `MJ_APIKEY_PUBLIC`. */ + readonly apiKey: string; + /** The secret key, `MJ_APIKEY_PRIVATE`. */ + readonly apiSecret: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.mailjet.com" + */ + readonly baseUrl?: string; + /** + * Mailjet's sandbox: each message is checked and dropped, nothing + * delivered (`SandboxMode`). + * @default false + */ + readonly sandbox?: boolean; + /** Open tracking of an HTML message, unless it sets its own `tracking`. */ + readonly openTracking?: boolean; + /** Click tracking of an HTML message, unless it sets its own. */ + readonly clickTracking?: boolean; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before Mailjet took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * A transport over the Send API v3.1 of Mailjet. A batch goes in requests of + * up to 50 recipients, each message with the receipt of its own answer. The + * first tag is the `CustomCampaign` (a second is refused), `metadata` the + * `EventPayload`; the fields Mailjet sets itself (`List-Id`, `Message-ID`, …) + * are refused, and so is a scheduled message. + * + * @example + * ```js + * const { MailjetTransport } = require('@alexify/mailer/mailjet'); + * const transport = new MailjetTransport({ + * apiKey: process.env.MJ_APIKEY_PUBLIC, + * apiSecret: process.env.MJ_APIKEY_PRIVATE, + * }); + * ``` + */ +export declare class MailjetTransport implements VerifiableTransport<'mailjet'> { + readonly id = 'mailjet'; + constructor(config: MailjetConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the key pair by reading its profile (`GET /v3/REST/myprofile`). */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/mailjet/index.js b/src/mailjet/index.js new file mode 100644 index 0000000..486f61e --- /dev/null +++ b/src/mailjet/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/mailjet.ts): the entry of the +// Mailjet transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { MailjetTransport } = require('./mailjet-transport.js'); + +module.exports = { MailjetTransport }; diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js new file mode 100644 index 0000000..8968c30 --- /dev/null +++ b/src/mailjet/mailjet-transport.js @@ -0,0 +1,199 @@ +// Ported from unemail v0.7.0 (src/drivers/mailjet.ts): a transport over the +// Mailjet Send API v3.1, whose `Messages` are its batch, each with its own +// answer, a 400 carrying the answers of those that went too. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: requests are split by the recipients they carry, as +// unemail split them, while the input is read a batch at a time, and a +// message that cannot be converted fails alone. + +const { generateId } = require('../core/id.js'); +const { createFailedReceipt } = require('../core/receipt.js'); +const { chunks } = require('../internal/batch.js'); +const { + ProviderApiError, + convertBatch, + isCallerAbort, + mergeBatch, + providerFailure, +} = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createMailjetConfig } = require('./config.js'); +const { createMailjetClient } = require('./http-client.js'); +const { MAX_RECIPIENTS, convertMessage, recipientCount } = require('./message-converter.js'); + +// Mailjet names every problem of a message at once, with the fields it is +// about. +const describeErrors = (errors) => { + let text = ''; + for (const error of errors) { + const message = error?.ErrorMessage ?? error?.ErrorCode ?? ''; + const related = Array.isArray(error?.ErrorRelatedTo) ? error.ErrorRelatedTo : []; + let part = message; + if (related.length > 0) part = `${related.join(', ')}: ${message}`; + if (part !== '') text += text === '' ? part : `; ${part}`; + } + return text; +}; + +// The answers of a request, from a 2xx or from the 400 Mailjet gives when one +// message of the request failed. +const answersOf = (error) => + error instanceof ProviderApiError && + error.statusCode === 400 && + Array.isArray(error.providerDetails?.Messages) + ? error.providerDetails.Messages + : undefined; + +/** + * A transport over the Send API v3.1 of Mailjet. + */ +class MailjetTransport { + id = 'mailjet'; + + constructor(config) { + this.config = createMailjetConfig(config); + this.httpClient = createMailjetClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + receiptOf(entry) { + if (entry?.Status !== 'success') { + const errors = Array.isArray(entry?.Errors) ? entry.Errors : []; + const text = describeErrors(errors) || `message status "${entry?.Status ?? 'unknown'}"`; + return createFailedReceipt(`Mailjet: ${text}`, { + provider: 'mailjet', + statusCode: errors[0]?.StatusCode, + code: errors[0]?.ErrorCode == null ? undefined : `mailjet.${errors[0].ErrorCode}`, + deliveryUnknown: entry == null, + providerDetails: entry ?? undefined, + }); + } + const recipient = entry.To?.[0] ?? entry.Cc?.[0] ?? entry.Bcc?.[0]; + let messageId = + recipient?.MessageUUID || (recipient?.MessageID ? String(recipient.MessageID) : ''); + // The sandbox checks and drops a message: it has no id of its own. + if (messageId === '' && this.config.sandbox) { + messageId = `sandbox-${generateId('receipt-id', this.config.generateId)}`; + } + if (messageId === '') { + return createFailedReceipt('Mailjet answered without a MessageUUID.', { + provider: 'mailjet', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'mailjet' }; + } + + // Sends entries in one request: the receipt of each, or the failure of all. + async sendEntries(entries, options) { + let answers; + try { + const answer = await this.httpClient.send(entries, options); + answers = Array.isArray(answer.json?.Messages) ? answer.json.Messages : []; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + answers = answersOf(error); + if (answers === undefined) { + const failure = providerFailure('mailjet', error); + const receipts = []; + for (let i = 0; i < entries.length; i++) receipts.push(failure); + return receipts; + } + } + const receipts = []; + for (let i = 0; i < entries.length; i++) receipts.push(this.receiptOf(answers[i])); + return receipts; + } + + /** + * Sends one message. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the MessageUUID of the first recipient. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const entry = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + return (await this.sendEntries([entry], options))[0]; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('mailjet', error); + } + } + + /** + * Sends messages in requests of up to 50 recipients, each message with the + * receipt of its own answer. + * + * @param messages The messages. + * @param options Optional abort signal and retry callback. + * @returns The receipts, in the order of the messages. + */ + async *sendMany(messages, options) { + const signal = options?.signal; + for await (const batch of chunks(messages, MAX_RECIPIENTS)) { + signal?.throwIfAborted(); + const converted = await convertBatch( + batch, + (message) => convertMessage(message, this.config, signal), + 'mailjet', + signal, + ); + const receipts = []; + let run = []; + let recipients = 0; + for (let i = 0; i < batch.length; i++) { + if (typeof converted.slots[i] !== 'number') continue; + const count = recipientCount(batch[i]); + if (run.length > 0 && recipients + count > MAX_RECIPIENTS) { + for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); + run = []; + recipients = 0; + } + run.push(converted.payloads[converted.slots[i]]); + recipients += count; + } + if (run.length > 0) { + for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); + } + yield* mergeBatch(converted.slots, (index) => receipts[index]); + } + } + + /** + * Verifies the key pair by reading its profile (`GET /v3/REST/myprofile`). + * + * @param options Optional abort signal. + * @throws {Error} If Mailjet refuses the keys or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'Mailjet', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/v3/REST/myprofile`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { MailjetTransport }; diff --git a/src/mailjet/message-converter.js b/src/mailjet/message-converter.js new file mode 100644 index 0000000..3865fbe --- /dev/null +++ b/src/mailjet/message-converter.js @@ -0,0 +1,167 @@ +// Ported from unemail v0.7.0 (src/drivers/mailjet.ts): a message as an entry +// of the `Messages` of Mailjet's Send API v3.1, `toMessage` and `checkLimits`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the fields Mailjet sets itself are refused by name, +// those of `list` and the threading fields among them; a second tag is +// refused, as `CustomCampaign` is one name; `metadata` goes in `EventPayload`; +// and a scheduled message is refused. + +const { toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +// "MAX RECIPIENTS: 50", across To, Cc and Bcc of the whole request. +const MAX_RECIPIENTS = 50; +// "Maximum length is 255 chars". +const MAX_SUBJECT = 255; + +// The fields Mailjet sets itself, or from a property of its own; it fails a +// message that gives one with send-0011. +const FORBIDDEN_HEADERS = new Set([ + 'from', + 'sender', + 'subject', + 'to', + 'cc', + 'bcc', + 'reply-to', + 'return-path', + 'delivered-to', + 'dkim-signature', + 'domainkey-status', + 'received-spf', + 'authentication-results', + 'received', + 'date', + 'message-id', + 'user-agent', + 'x-mailer', + 'list-id', + 'x-csa-complaints', + 'x-feedback-id', + 'x-mailjet-prio', + 'x-mailjet-debug', + 'x-mailjet-campaign', + 'x-mailjet-segmentation', + 'x-mailjet-trackopen', + 'x-mailjet-trackclick', + 'x-mj-customid', + 'x-mj-eventpayload', + 'x-mj-vars', + 'x-mj-templateid', + 'x-mj-templatelanguage', + 'x-mj-templateerrordeliver', + 'x-mj-templateerrorreporting', + 'x-mj-workflowid', + 'x-mj-mid', + 'x-mj-errormessage', + 'x-mj-statisticscontactslistid', +]); + +const addressOf = (address) => { + const email = toAsciiAddress(address.address); + return address.name ? { Email: email, Name: address.name } : { Email: email }; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +/** + * The recipients of a message, an address counted once, as Mailjet counts + * them against its limit. + * + * @internal + */ +const recipientCount = (message) => { + const seen = new Set(); + for (const list of [message.recipients, message.ccRecipients, message.bccRecipients]) { + for (let i = 0; i < list.length; i++) seen.add(list[i].address.toLowerCase()); + } + return seen.size; +}; + +const checkLimits = (message) => { + refuseScheduledAt(message, 'mailjet'); + if (message.subject.length > MAX_SUBJECT) { + throw new RangeError(`Mailjet takes a subject of at most ${MAX_SUBJECT} characters.`); + } + if (recipientCount(message) > MAX_RECIPIENTS) { + throw new RangeError(`Mailjet takes at most ${MAX_RECIPIENTS} recipients a message.`); + } + if (message.replyRecipients.length > 1) { + throw new RangeError('Mailjet takes one reply-to address.'); + } + if (message.tags.length > 1) throw new RangeError('Mailjet takes one tag per message.'); +}; + +const trackingOf = (message, config, hasHtml) => { + const tracking = {}; + // Mailjet refuses both without an HTML part to track. + if (!hasHtml) return tracking; + const opens = message.tracking?.opens ?? config.openTracking; + const clicks = message.tracking?.clicks ?? config.clickTracking; + if (opens !== undefined) tracking.TrackOpens = opens ? 'enabled' : 'disabled'; + if (clicks !== undefined) tracking.TrackClicks = clicks ? 'enabled' : 'disabled'; + return tracking; +}; + +/** + * Converts a message to an entry of Mailjet's `Messages`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The entry. + * @throws {RangeError} If the message is over a limit of Mailjet, or gives a + * field Mailjet sets itself. + * @throws {TypeError} If it is scheduled. + */ +const convertMessage = async (message, config, signal) => { + checkLimits(message); + const entry = { + From: addressOf(message.sender), + To: addressesOf(message.recipients), + Subject: message.subject, + }; + if (message.ccRecipients.length > 0) entry.Cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) entry.Bcc = addressesOf(message.bccRecipients); + if (message.replyRecipients.length === 1) entry.ReplyTo = addressOf(message.replyRecipients[0]); + const hasHtml = 'html' in message.content; + if (message.content.text) entry.TextPart = message.content.text; + if (hasHtml) entry.HTMLPart = message.content.html; + const headers = {}; + let hasHeaders = false; + for (const field of headerFieldsOf(message)) { + if (FORBIDDEN_HEADERS.has(field.name.toLowerCase())) { + throw new RangeError(`Mailjet sets ${field.name} itself; the message cannot give it.`); + } + headers[field.name] = field.value; + hasHeaders = true; + } + if (hasHeaders) entry.Headers = headers; + if (message.tags.length === 1) entry.CustomCampaign = message.tags[0]; + if (message.metadata !== undefined) entry.EventPayload = JSON.stringify(message.metadata); + Object.assign(entry, trackingOf(message, config, hasHtml)); + const attachments = await base64Attachments(message, signal); + for (const item of attachments) { + const part = { + ContentType: item.attachment.contentType, + Filename: item.attachment.filename, + Base64Content: item.content, + }; + if (item.attachment.inline) { + part.ContentID = item.attachment.contentId; + entry.InlinedAttachments ??= []; + entry.InlinedAttachments.push(part); + } else { + entry.Attachments ??= []; + entry.Attachments.push(part); + } + } + return entry; +}; + +module.exports = { MAX_RECIPIENTS, convertMessage, recipientCount }; diff --git a/tests/types/mailjet.test-d.ts b/tests/types/mailjet.test-d.ts new file mode 100644 index 0000000..404e639 --- /dev/null +++ b/tests/types/mailjet.test-d.ts @@ -0,0 +1,22 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { MailjetTransport } from '../../src/mailjet/index.js'; + +const transport = new MailjetTransport({ + apiKey: 'public', + apiSecret: 'private', + sandbox: true, + openTracking: true, +}); +expectType<'mailjet'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new MailjetTransport({ apiKey: 'public' })); diff --git a/tests/unit/mailjet.test.js b/tests/unit/mailjet.test.js new file mode 100644 index 0000000..6fa6430 --- /dev/null +++ b/tests/unit/mailjet.test.js @@ -0,0 +1,224 @@ +// Not ported: tests of ./mailjet, ported from unemail's Mailjet driver, +// against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { MailjetTransport } = require('../../src/mailjet/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const transport = (config = {}) => + new MailjetTransport({ apiKey: 'public', apiSecret: 'secret-key', retries: 0, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const success = (uuid) => ({ + Status: 'success', + To: [{ Email: 'x', MessageUUID: uuid, MessageID: 1 }], +}); +const failed = (code, text, related = ['To']) => ({ + Status: 'error', + Errors: [{ ErrorCode: code, StatusCode: 400, ErrorMessage: text, ErrorRelatedTo: related }], +}); + +test('Mailjet: a message is an entry of Messages, with Basic authentication', async (t) => { + const requests = stubFetch(t, () => json({ Messages: [success('uuid-1')] })); + const receipt = await transport({ openTracking: false }).send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: 'support@example.com', + tags: ['autumn'], + metadata: { orderId: '42' }, + tracking: { clicks: true }, + list: { unsubscribe: 'https://example.com/u' }, + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'uuid-1', provider: 'mailjet' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.mailjet.com/v3.1/send'); + assert.strictEqual( + request.headers.get('authorization'), + `Basic ${Buffer.from('public:secret-key').toString('base64')}`, + ); + assert.strictEqual(request.json.SandboxMode, undefined); + const [entry] = request.json.Messages; + assert.deepStrictEqual(entry.From, { Email: 'shop@example.com', Name: 'Shop' }); + assert.deepStrictEqual(entry.To, [ + { Email: 'bob@example.com' }, + { Email: 'carol@xn--bcher-kva.de', Name: 'Carol' }, + ]); + assert.deepStrictEqual(entry.Cc, [{ Email: 'dave@example.com' }]); + assert.deepStrictEqual(entry.Bcc, [{ Email: 'erin@example.com' }]); + assert.deepStrictEqual(entry.ReplyTo, { Email: 'support@example.com' }); + assert.strictEqual(entry.TextPart, 'Shipped'); + assert.strictEqual(entry.HTMLPart, '

Shipped

'); + assert.strictEqual(entry.CustomCampaign, 'autumn'); + assert.strictEqual(entry.EventPayload, '{"orderId":"42"}'); + assert.strictEqual(entry.TrackOpens, 'disabled'); + assert.strictEqual(entry.TrackClicks, 'enabled'); + assert.deepStrictEqual(entry.Headers, { 'List-Unsubscribe': '' }); + assert.deepStrictEqual(entry.Attachments, [ + { ContentType: 'text/plain', Filename: 'r.txt', Base64Content: 'aGk=' }, + ]); + assert.deepStrictEqual(entry.InlinedAttachments, [ + { + ContentType: 'image/png', + Filename: 'logo.png', + Base64Content: 'AQ==', + ContentID: 'logo@example.com', + }, + ]); +}); + +test('Mailjet: no tracking without HTML, and the sandbox gives an id of its own', async (t) => { + const requests = stubFetch(t, () => + json({ + Messages: [{ Status: 'success', To: [{ Email: 'x', MessageUUID: '', MessageID: 0 }] }], + }), + ); + const receipt = await transport({ + sandbox: true, + openTracking: true, + generateId: () => 'g1', + }).send(message({ content: { text: 'plain' } })); + assert.strictEqual(receipt.messageId, 'sandbox-g1'); + assert.strictEqual(requests[0].json.SandboxMode, true); + assert.strictEqual(requests[0].json.Messages[0].TrackOpens, undefined); + stubFetch(t, () => + json({ + Messages: [{ Status: 'success', To: [{ Email: 'x', MessageUUID: '', MessageID: 0 }] }], + }), + ); + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); +}); + +test('Mailjet: a 400 carries the answer of each message, those that went too', async (t) => { + const requests = stubFetch(t, () => + json( + { + Messages: [ + success('u-0'), + failed('send-0003', 'At least "HTMLPart" is required'), + success('u-2'), + ], + }, + 400, + ), + ); + const receipts = []; + for await (const receipt of transport().sendMany([ + message(), + message(), + message({ tags: ['a', 'b'] }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, false, true], + ); + assert.strictEqual(receipts[0].messageId, 'u-0'); + assert.strictEqual(receipts[1].errorMessages[0], 'Mailjet: To: At least "HTMLPart" is required'); + assert.strictEqual(receipts[1].errors[0].code, 'mailjet.send-0003'); + assert.strictEqual(receipts[1].retryable, false); + assert.match(receipts[2].errorMessages[0], /one tag per message/); + assert.strictEqual(receipts[3].messageId, 'u-2'); + assert.strictEqual(requests.length, 1); + assert.strictEqual(requests[0].json.Messages.length, 3); +}); + +test('Mailjet: requests hold at most 50 recipients', async (t) => { + const requests = stubFetch(t, (request) => + json({ Messages: request.json.Messages.map((_, index) => success(`u-${index}`)) }), + ); + const many = []; + for (let i = 0; i < 30; i++) many.push(`r${i}@example.com`); + const receipts = []; + for await (const receipt of transport().sendMany([ + message({ to: many }), + message({ to: many.slice(0, 25) }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + requests.map((request) => request.json.Messages.length), + [1, 2], + ); + assert.ok(receipts.every((receipt) => receipt.successful)); + const tooMany = []; + for (let i = 0; i < 51; i++) tooMany.push(`r${i}@example.com`); + assert.match( + (await transport().send(message({ to: tooMany }))).errorMessages[0], + /at most 50 recipients/, + ); +}); + +test('Mailjet: what Mailjet sets itself, and its limits, are refused before a request', async (t) => { + const requests = stubFetch(t, () => json({ Messages: [success('u')] })); + const refusals = [ + [message({ list: { id: 'news.example.com' } }), /sets List-ID itself/], + [message({ headers: { 'X-Mailer': 'Mine 1.0' } }), /sets X-Mailer itself/], + [message({ subject: 'x'.repeat(256) }), /subject of at most 255/], + [message({ replyTo: ['a@example.com', 'b@example.com'] }), /one reply-to/], + [message({ scheduledAt: new Date() }), /Invalid scheduledAt: mailjet/], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('Mailjet: a failed request fails its messages, an empty answer is unknown', async (t) => { + const answers = [ + json({ ErrorMessage: 'API key authentication/authorization failure' }, 401), + json({}), + ]; + stubFetch(t, () => answers.shift()); + const receipts = []; + for await (const receipt of transport().sendMany([message(), message()])) receipts.push(receipt); + assert.strictEqual(receipts.length, 2); + for (const receipt of receipts) { + assert.strictEqual(receipt.errors[0].category, 'auth'); + assert.match(receipt.errorMessages[0], /authentication\/authorization failure/); + } + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); +}); + +test('Mailjet: verify reads the profile, the options are checked, the secret never shows', async (t) => { + const requests = stubFetch(t, () => json({ Data: [] })); + await transport().verify(); + assert.strictEqual(requests[0].url, 'https://api.mailjet.com/v3/REST/myprofile'); + assert.throws(() => new MailjetTransport({ apiKey: 'public' }), /needs apiSecret/); + assert.throws(() => transport({ clickTracking: 'on' }), /clickTracking must be a boolean/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('secret-key')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('secret-key')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); From 52226e7d95ed6f45f7ccb22cde0acfa22ccafc41 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:39:40 +0300 Subject: [PATCH 020/143] feat(mailersend): a transport over the MailerSend API A new subpath, ported from the MailerSend drivers of unemail and sently: /v1/email with a Bearer token, the id of a message the X-Message-Id of the 202 that answers it. scheduledAt is send_at (at most 72 hours ahead), tags (up to 5) its tags, tracking its settings, metadata X-Metadata- headers; In-Reply-To, References and List-Unsubscribe go in the fields MailerSend has for them, which writes the one-click Post field itself. Its limits (50 to, 10 cc, 10 bcc, one reply-to) are refused before the request, and a 422 names the fields it refuses. Fixed in the port (U6): unemail's batch went through the bulk endpoint, one id for every message and failures unknown to the receipts; sendMany sends single requests, `concurrency` at a time. sently dropped custom headers; they are sent. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 5 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/mailersend.md | 77 +++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/mailersend/config.js | 43 +++++++ src/mailersend/http-client.js | 46 +++++++ src/mailersend/index.d.ts | 72 +++++++++++ src/mailersend/index.js | 7 + src/mailersend/mailersend-transport.js | 103 +++++++++++++++ src/mailersend/message-converter.js | 150 ++++++++++++++++++++++ tests/types/mailersend.test-d.ts | 23 ++++ tests/unit/mailersend.test.js | 171 +++++++++++++++++++++++++ 14 files changed, 704 insertions(+) create mode 100644 docs/transports/mailersend.md create mode 100644 src/mailersend/config.js create mode 100644 src/mailersend/http-client.js create mode 100644 src/mailersend/index.d.ts create mode 100644 src/mailersend/index.js create mode 100644 src/mailersend/mailersend-transport.js create mode 100644 src/mailersend/message-converter.js create mode 100644 tests/types/mailersend.test-d.ts create mode 100644 tests/unit/mailersend.test.js diff --git a/CLAUDE.md b/CLAUDE.md index ee8f675..5c46c73 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -511,6 +511,11 @@ fails on upstream's behavior): two batches could share one. `./brevo` sends single requests, each with its own key, generated when the message has none, so a retry after an unknown delivery cannot send twice; Brevo's `duplicate_parameter` is reported as `deliveryUnknown`. +- U6. unemail (`src/drivers/mailersend.ts`): `sendBatch` went through the bulk endpoint, which + answers with one `bulk_email_id` for every message and accepts what may fail later, so each + receipt said "sent" for a message whose fate was unknown. `./mailersend` sends single requests, + the id of each the `X-Message-Id` of its answer. sently's MailerSend driver dropped custom + headers; they are sent, with the threading fields and List-Unsubscribe in MailerSend's own. - U1. unemail (`src/core/message.ts`): `List-Unsubscribe` was built from the `unsubscribe` URL after the check for line breaks in header values, so a CR or LF in the URL added header fields. `listFieldsOf` (`src/core/list.js`) refuses whitespace, controls and angle brackets in a URL and diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index f6088ba..c716185 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -87,6 +87,7 @@ const guide = [ { text: 'Postmark', link: '/transports/postmark' }, { text: 'Brevo', link: '/transports/brevo' }, { text: 'Mailjet', link: '/transports/mailjet' }, + { text: 'MailerSend', link: '/transports/mailersend' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 2360f04..204285b 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -56,6 +56,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/postmark` | `PostmarkTransport` | [Postmark](/transports/postmark) | | `@alexify/mailer/brevo` | `BrevoTransport` | [Brevo](/transports/brevo) | | `@alexify/mailer/mailjet` | `MailjetTransport` | [Mailjet](/transports/mailjet) | +| `@alexify/mailer/mailersend` | `MailerSendTransport` | [MailerSend](/transports/mailersend) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/mailersend.md b/docs/transports/mailersend.md new file mode 100644 index 0000000..f7b412e --- /dev/null +++ b/docs/transports/mailersend.md @@ -0,0 +1,77 @@ +# MailerSend transport + +The MailerSend transport sends through the email API of [MailerSend], from the +MailerSend drivers of unemail and sently. + +[MailerSend]: https://www.mailersend.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { MailerSendTransport } = require('@alexify/mailer/mailersend'); + +const transport = new MailerSendTransport({ apiKey: process.env.MAILERSEND_API_KEY }); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + }), +); +if (receipt.successful) console.log(receipt.messageId); // the X-Message-Id +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `apiKey` | (required) | An API token. | +| `openTracking`, `clickTracking` | the domain's | Tracking, unless a message sets its own `tracking`. | +| `precedenceBulk` | | Marks every message as bulk. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `baseUrl` | `https://api.mailersend.com` | HTTPS, or HTTP to this machine. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before MailerSend took it. | +| `headers` | | Header fields added to every request. | +| `now` | `Date.now` | The clock for the 72-hour window of `scheduledAt`. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- `scheduledAt` is `send_at`, at most 72 hours ahead; `tags` (up to 5) its tags; + `tracking` its settings. +- `metadata` goes in `X-Metadata-` headers, as MailerSend has no metadata of + its own and returns custom headers in its events. +- In-Reply-To and References go in `in_reply_to` and `references`, and + List-Unsubscribe in `list_unsubscribe`; MailerSend writes the + List-Unsubscribe-Post of one-click itself. +- An inline attachment goes with its Content-ID as `id`. +- MailerSend takes 50 `to`, 10 `cc` and 10 `bcc` recipients and one reply-to + address; more are refused before the request. + +MailerSend answers 202 with an empty body; the receipt's id is the `X-Message-Id` +header of the answer, and an answer without one is reported as `deliveryUnknown`. +A 422 names the fields it refuses and why, which the receipt's message carries. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. The bulk endpoint is not used: it answers with one id for the whole batch +and takes messages that may fail later, which no receipt would tell. + +## Verifying the configuration + +`verify()` reads the domains of the account (`GET /v1/domains`); a token without +that scope is verified too. diff --git a/package.json b/package.json index 09d0a1f..4a0f4c0 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,10 @@ "types": "./src/mailjet/index.d.ts", "default": "./src/mailjet/index.js" }, + "./mailersend": { + "types": "./src/mailersend/index.d.ts", + "default": "./src/mailersend/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 5c66b35..0577396 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -16,6 +16,7 @@ "./postmark": 7, "./brevo": 7, "./mailjet": 8, + "./mailersend": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/mailersend/config.js b/src/mailersend/config.js new file mode 100644 index 0000000..864103b --- /dev/null +++ b/src/mailersend/config.js @@ -0,0 +1,43 @@ +// Ported from unemail v0.7.0 (src/drivers/mailersend.ts): the options of the +// MailerSend driver. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkConcurrency, + checkOptionalBoolean, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +/** + * Resolves the options of the MailerSend transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createMailerSendConfig = (config = {}) => { + const resolved = { + apiKey: checkCredential(config.apiKey, 'apiKey', 'MailerSendTransport'), + baseUrl: config.baseUrl ?? 'https://api.mailersend.com', + openTracking: checkOptionalBoolean(config.openTracking, 'openTracking'), + clickTracking: checkOptionalBoolean(config.clickTracking, 'clickTracking'), + precedenceBulk: checkOptionalBoolean(config.precedenceBulk, 'precedenceBulk'), + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + now: config.now ?? Date.now, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createMailerSendConfig }; diff --git a/src/mailersend/http-client.js b/src/mailersend/http-client.js new file mode 100644 index 0000000..16cb640 --- /dev/null +++ b/src/mailersend/http-client.js @@ -0,0 +1,46 @@ +// Ported from unemail v0.7.0 (src/drivers/mailersend.ts): the requests of the +// MailerSend API, and the reasons of a validation error. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers. + +const { createApiClient } = require('../internal/api.js'); + +// MailerSend gives `{ message, errors: { field: [reason] } }`, the message the +// same sentence for any of them: the reasons of the fields say what failed. +const messageOf = (json) => { + const errors = json?.errors; + if (errors == null || typeof errors !== 'object') return json?.message; + let text = ''; + for (const field in errors) { + if (!Object.hasOwn(errors, field)) continue; + const reasons = Array.isArray(errors[field]) ? errors[field].join(', ') : String(errors[field]); + text += text === '' ? `${field}: ${reasons}` : `; ${field}: ${reasons}`; + } + return text === '' ? json?.message : text; +}; + +/** + * The client of the MailerSend API. + * + * @internal + */ +const createMailerSendClient = (config) => { + const client = createApiClient(config, { + id: 'mailersend', + label: 'MailerSend', + authorize: (headers) => headers.set('Authorization', `Bearer ${config.apiKey}`), + messageOf, + }); + return { + ...client, + send: (payload, options) => + client.request({ + url: `${config.baseUrl}/v1/email`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createMailerSendClient }; diff --git a/src/mailersend/index.d.ts b/src/mailersend/index.d.ts new file mode 100644 index 0000000..0b92bb7 --- /dev/null +++ b/src/mailersend/index.d.ts @@ -0,0 +1,72 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link MailerSendTransport}. + */ +export interface MailerSendConfig extends TransportDependencies { + /** An API token. */ + readonly apiKey: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.mailersend.com" + */ + readonly baseUrl?: string; + /** Open tracking, unless a message sets its own `tracking`. */ + readonly openTracking?: boolean; + /** Click tracking, unless a message sets its own. */ + readonly clickTracking?: boolean; + /** Marks every message as bulk (`precedence_bulk`). */ + readonly precedenceBulk?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before MailerSend took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; + /** The clock, in milliseconds, for the 72 hours of `scheduledAt`. */ + readonly now?: () => number; +} + +/** + * A transport over the email API of MailerSend. `scheduledAt` (at most 72 + * hours ahead) is `send_at`, `tags` (up to 5) its tags, `metadata` goes in + * `X-Metadata-` headers, `tracking` in its settings; the threading + * fields and List-Unsubscribe go in the fields MailerSend has for them. + * + * @example + * ```js + * const { MailerSendTransport } = require('@alexify/mailer/mailersend'); + * const transport = new MailerSendTransport({ apiKey: process.env.MAILERSEND_API_KEY }); + * ``` + */ +export declare class MailerSendTransport implements VerifiableTransport<'mailersend'> { + readonly id = 'mailersend'; + constructor(config: MailerSendConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the API token by reading the domains of the account. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/mailersend/index.js b/src/mailersend/index.js new file mode 100644 index 0000000..04cf1fe --- /dev/null +++ b/src/mailersend/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/mailersend.ts): the entry of the +// MailerSend transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { MailerSendTransport } = require('./mailersend-transport.js'); + +module.exports = { MailerSendTransport }; diff --git a/src/mailersend/mailersend-transport.js b/src/mailersend/mailersend-transport.js new file mode 100644 index 0000000..9ca38b3 --- /dev/null +++ b/src/mailersend/mailersend-transport.js @@ -0,0 +1,103 @@ +// Ported from unemail v0.7.0 (src/drivers/mailersend.ts): a transport over the +// MailerSend API, whose message id is a header of its answer. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/mailersend.ts): `verify`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: a batch goes as concurrent single requests, not +// through the bulk API, which answers with one id for the whole batch and +// accepts what may fail later, unknown to the receipts. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createMailerSendConfig } = require('./config.js'); +const { createMailerSendClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +/** + * A transport over the email API of MailerSend. + */ +class MailerSendTransport { + id = 'mailersend'; + + constructor(config) { + this.config = createMailerSendConfig(config); + this.httpClient = createMailerSendClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/v1/email`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the `X-Message-Id` of the answer. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const answer = await this.httpClient.send(payload, options); + const messageId = answer.headers.get('x-message-id') ?? answer.json?.message_id; + if (typeof messageId !== 'string' || messageId === '') { + return createFailedReceipt('MailerSend answered without an X-Message-Id.', { + provider: 'mailersend', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'mailersend' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('mailersend', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the API token by reading the domains of the account + * (`GET /v1/domains`); a token without that scope is verified too. + * + * @param options Optional abort signal. + * @throws {Error} If MailerSend refuses the token or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'MailerSend', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/v1/domains`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { MailerSendTransport }; diff --git a/src/mailersend/message-converter.js b/src/mailersend/message-converter.js new file mode 100644 index 0000000..9ef9965 --- /dev/null +++ b/src/mailersend/message-converter.js @@ -0,0 +1,150 @@ +// Ported from unemail v0.7.0 (src/drivers/mailersend.ts): a message as the +// request of MailerSend's email API, `toPayload` and `validate`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/transports/mailersend.ts): the reply-to +// address and the inline attachments. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the threading fields and List-Unsubscribe go in the +// fields MailerSend has for them, the name of a metadata header is a checked +// key, a second reply-to address is refused, and sently's dropped custom +// headers are sent. + +const { toAsciiAddress } = require('../core/address.js'); +const { parseMessageId } = require('../core/message-id.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); + +const LIMITS = [ + ['recipients', 'to', 50], + ['ccRecipients', 'cc', 10], + ['bccRecipients', 'bcc', 10], +]; +const MAX_TAGS = 5; +// `send_at` may be at most 72 hours ahead. +const SCHEDULE_WINDOW_MS = 72 * 60 * 60 * 1000; + +// The fields MailerSend reads from a property of its own. +const OWN_FIELDS = new Set([ + 'in-reply-to', + 'references', + 'list-unsubscribe', + 'list-unsubscribe-post', +]); + +const addressOf = (address) => { + const email = toAsciiAddress(address.address); + return address.name ? { email, name: address.name } : { email }; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +const checkLimits = (message, config) => { + for (const [field, name, limit] of LIMITS) { + if (message[field].length > limit) { + throw new RangeError(`MailerSend takes at most ${limit} ${name} recipients.`); + } + } + if (message.tags.length > MAX_TAGS) { + throw new RangeError(`MailerSend takes at most ${MAX_TAGS} tags.`); + } + if (message.replyRecipients.length > 1) { + throw new RangeError('MailerSend takes one reply-to address.'); + } + if ( + message.scheduledAt !== undefined && + message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS + ) { + throw new RangeError('MailerSend schedules a message at most 72 hours ahead.'); + } +}; + +// The identifiers of In-Reply-To or References, without their brackets. +const messageIdsOf = (value) => { + const ids = []; + for (const part of value.split(/\s+/)) { + const id = parseMessageId(part); + if (id !== undefined) ids.push(id); + } + return ids; +}; + +const placeHeaders = (payload, message) => { + const headers = []; + for (const field of headerFieldsOf(message)) { + const name = field.name.toLowerCase(); + if (!OWN_FIELDS.has(name)) { + headers.push({ name: field.name, value: field.value }); + } else if (name === 'in-reply-to') { + payload.in_reply_to = messageIdsOf(field.value)[0]; + } else if (name === 'references') { + payload.references = messageIdsOf(field.value); + } else if (name === 'list-unsubscribe') { + // MailerSend writes List-Unsubscribe-Post itself, as RFC 8058 asks. + payload.list_unsubscribe = field.value; + } + } + // MailerSend has no metadata of its own: a header reaches its events. + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers.push({ name: `X-Metadata-${key}`, value: message.metadata[key] }); + } + } + if (headers.length > 0) payload.headers = headers; +}; + +/** + * Converts a message to a request of MailerSend's `/v1/email`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message is over a limit of MailerSend. + */ +const convertMessage = async (message, config, signal) => { + checkLimits(message, config); + const payload = { + from: addressOf(message.sender), + to: addressesOf(message.recipients), + subject: message.subject, + }; + if (message.ccRecipients.length > 0) payload.cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.bcc = addressesOf(message.bccRecipients); + if (message.replyRecipients.length === 1) { + payload.reply_to = addressOf(message.replyRecipients[0]); + } + if (message.content.text) payload.text = message.content.text; + if ('html' in message.content) payload.html = message.content.html; + placeHeaders(payload, message); + if (message.tags.length > 0) payload.tags = [...message.tags]; + const settings = {}; + const opens = message.tracking?.opens ?? config.openTracking; + const clicks = message.tracking?.clicks ?? config.clickTracking; + if (opens !== undefined) settings.track_opens = opens; + if (clicks !== undefined) settings.track_clicks = clicks; + if (opens !== undefined || clicks !== undefined) payload.settings = settings; + if (config.precedenceBulk !== undefined) payload.precedence_bulk = config.precedenceBulk; + if (message.scheduledAt !== undefined) { + payload.send_at = Math.floor(message.scheduledAt.getTime() / 1000); + } + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.attachments = []; + for (const item of attachments) { + const part = { filename: item.attachment.filename, content: item.content }; + if (item.attachment.inline) { + part.disposition = 'inline'; + part.id = item.attachment.contentId; + } else { + part.disposition = 'attachment'; + } + payload.attachments.push(part); + } + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/tests/types/mailersend.test-d.ts b/tests/types/mailersend.test-d.ts new file mode 100644 index 0000000..ea1f8e0 --- /dev/null +++ b/tests/types/mailersend.test-d.ts @@ -0,0 +1,23 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { MailerSendTransport } from '../../src/mailersend/index.js'; + +const transport = new MailerSendTransport({ + apiKey: 'mlsn.key', + openTracking: true, + precedenceBulk: false, + concurrency: 2, + now: () => Date.now(), +}); +expectType<'mailersend'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new MailerSendTransport({})); diff --git a/tests/unit/mailersend.test.js b/tests/unit/mailersend.test.js new file mode 100644 index 0000000..865957b --- /dev/null +++ b/tests/unit/mailersend.test.js @@ -0,0 +1,171 @@ +// Not ported: tests of ./mailersend, ported from unemail's and sently's +// MailerSend drivers, against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { MailerSendTransport } = require('../../src/mailersend/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const NOW = Date.parse('2026-10-17T12:00:00Z'); +const transport = (config = {}) => + new MailerSendTransport({ apiKey: 'mlsn.secret', retries: 0, now: () => NOW, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const accepted = (id = 'ms-1') => + new Response(null, { status: 202, headers: { 'x-message-id': id } }); + +test('MailerSend: a message is the request of /v1/email, its id a header of the answer', async (t) => { + const requests = stubFetch(t, () => accepted()); + const receipt = await transport({ precedenceBulk: true, clickTracking: false }).send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: 'support@example.com', + priority: 'high', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + tracking: { opens: true }, + inReplyTo: ['a@example.com'], + references: ['r1@example.com', 'a@example.com'], + list: { unsubscribe: 'https://example.com/u', oneClick: true }, + scheduledAt: new Date(NOW + 3600_000), + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'ms-1', provider: 'mailersend' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.mailersend.com/v1/email'); + assert.strictEqual(request.headers.get('authorization'), 'Bearer mlsn.secret'); + const body = request.json; + assert.deepStrictEqual(body.from, { email: 'shop@example.com', name: 'Shop' }); + assert.deepStrictEqual(body.to, [ + { email: 'bob@example.com' }, + { email: 'carol@xn--bcher-kva.de', name: 'Carol' }, + ]); + assert.deepStrictEqual(body.reply_to, { email: 'support@example.com' }); + assert.strictEqual(body.in_reply_to, 'a@example.com'); + assert.deepStrictEqual(body.references, ['r1@example.com', 'a@example.com']); + assert.strictEqual(body.list_unsubscribe, ''); + assert.deepStrictEqual(body.tags, ['orders', 'eu']); + assert.deepStrictEqual(body.settings, { track_opens: true, track_clicks: false }); + assert.strictEqual(body.precedence_bulk, true); + assert.strictEqual(body.send_at, Math.floor((NOW + 3600_000) / 1000)); + assert.deepStrictEqual(body.headers, [ + { name: 'X-Priority', value: '1' }, + { name: 'X-MSMail-Priority', value: 'High' }, + { name: 'Importance', value: 'High' }, + { name: 'X-Metadata-orderId', value: '42' }, + ]); + assert.deepStrictEqual(body.attachments, [ + { filename: 'r.txt', content: 'aGk=', disposition: 'attachment' }, + { filename: 'logo.png', content: 'AQ==', disposition: 'inline', id: 'logo@example.com' }, + ]); +}); + +test('MailerSend: a plain message has no settings or headers, an answer without id is unknown', async (t) => { + const answers = [accepted('ms-2'), new Response(null, { status: 202 })]; + const requests = stubFetch(t, () => answers.shift()); + const receipt = await transport().send(message({ content: { text: 'plain' } })); + assert.strictEqual(receipt.messageId, 'ms-2'); + const body = requests[0].json; + assert.strictEqual(body.settings, undefined); + assert.strictEqual(body.headers, undefined); + assert.strictEqual(body.html, undefined); + assert.strictEqual(body.text, 'plain'); + const unknown = await transport().send(message()); + assert.strictEqual(unknown.deliveryUnknown, true); +}); + +test('MailerSend: a 422 names the fields and their reasons', async (t) => { + stubFetch(t, () => + json( + { + message: 'The given data was invalid.', + errors: { + 'from.email': ['The from.email domain must be verified.'], + subject: ['Required.'], + }, + }, + 422, + ), + ); + const receipt = await transport().send(message()); + assert.strictEqual( + receipt.errorMessages[0], + 'MailerSend: from.email: The from.email domain must be verified.; subject: Required.', + ); + assert.strictEqual(receipt.errors[0].category, 'validation'); + assert.strictEqual(receipt.retryable, false); +}); + +test('MailerSend: its limits are refused before a request', async (t) => { + const requests = stubFetch(t, () => accepted()); + const many = (count) => { + const list = []; + for (let i = 0; i < count; i++) list.push(`r${i}@example.com`); + return list; + }; + const refusals = [ + [message({ to: many(51) }), /at most 50 to recipients/], + [message({ cc: many(11) }), /at most 10 cc recipients/], + [message({ bcc: many(11) }), /at most 10 bcc recipients/], + [message({ tags: ['a', 'b', 'c', 'd', 'e', 'f'] }), /at most 5 tags/], + [message({ replyTo: ['a@example.com', 'b@example.com'] }), /one reply-to/], + [message({ scheduledAt: new Date(NOW + 73 * 3600_000) }), /72 hours ahead/], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('MailerSend: sendMany sends concurrently, verify reads the domains', async (t) => { + let index = 0; + const requests = stubFetch(t, (request) => + request.method === 'GET' ? json({ data: [] }) : accepted(`ms-${index++}`), + ); + const receipts = []; + for await (const receipt of transport().sendMany([ + message(), + message({ tags: ['a', 'b', 'c', 'd', 'e', 'f'] }), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false], + ); + await transport().verify(); + assert.strictEqual(requests.at(-1).url, 'https://api.mailersend.com/v1/domains'); +}); + +test('MailerSend: the options are checked, and the token never shows', () => { + assert.throws(() => new MailerSendTransport({}), /needs apiKey/); + assert.throws(() => transport({ precedenceBulk: 'yes' }), /precedenceBulk must be a boolean/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('mlsn.secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('mlsn.secret')); +}); From e22fde613498dce80419327a086b8fc8f8a27e74 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:47:11 +0300 Subject: [PATCH 021/143] feat(sparkpost): a transport over the SparkPost Transmissions API Ported from sently's SparkPost transport onto the shared client of the HTTP providers. The US or EU host comes from `region`; cc and bcc recipients get the shared To field in `header_to` and the Cc field in the headers of the content (S3); tags go on each recipient, metadata on the transmission, tracking and `scheduledAt` in its options; inline images in `inline_images`; and `sendRaw` sends a UTF-8 MIME message as `email_rfc822`. Rejected recipients are counted on the receipt, which fails as `rejected` when none was accepted. The budget is 8 KB: the raw-message reader of `sendRaw` adds half a kilobyte. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/sparkpost.md | 97 ++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/sparkpost/config.js | 51 ++++++ src/sparkpost/http-client.js | 54 ++++++ src/sparkpost/index.d.ts | 98 ++++++++++ src/sparkpost/index.js | 7 + src/sparkpost/message-converter.js | 147 +++++++++++++++ src/sparkpost/sparkpost-transport.js | 143 +++++++++++++++ tests/types/sparkpost.test-d.ts | 28 +++ tests/unit/sparkpost.test.js | 259 +++++++++++++++++++++++++++ 14 files changed, 898 insertions(+) create mode 100644 docs/transports/sparkpost.md create mode 100644 src/sparkpost/config.js create mode 100644 src/sparkpost/http-client.js create mode 100644 src/sparkpost/index.d.ts create mode 100644 src/sparkpost/index.js create mode 100644 src/sparkpost/message-converter.js create mode 100644 src/sparkpost/sparkpost-transport.js create mode 100644 tests/types/sparkpost.test-d.ts create mode 100644 tests/unit/sparkpost.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 5c46c73..689c5a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -525,6 +525,13 @@ fails on upstream's behavior): - U3. unemail: the keys and values of `metadata` became `X-Metadata-` header fields in eight drivers with no check, so a key or value with a line break added header fields. `createMessage` holds a key to 1 to 128 of `[A-Za-z0-9_.-]` and a value to a string without control characters. +- S3. sently (`src/transports/sparkpost.ts`): cc and bcc recipients went into `recipients` with no + `header_to` and no Cc field, so the message lost its Cc field and each copy named only its own + recipient in To; every address was reported accepted whatever `total_rejected_recipients` said, + and an answer without an id was a success with an empty one. `./sparkpost` gives each recipient + the shared To in `header_to` and the Cc field in `content.headers`, counts the rejected ones on + the receipt (`rejectedRecipientCount`), fails it as `rejected` when none was accepted, and + reports an answer without an id as `deliveryUnknown`. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c716185..20e865c 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -88,6 +88,7 @@ const guide = [ { text: 'Brevo', link: '/transports/brevo' }, { text: 'Mailjet', link: '/transports/mailjet' }, { text: 'MailerSend', link: '/transports/mailersend' }, + { text: 'SparkPost', link: '/transports/sparkpost' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 204285b..9d9044f 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -57,6 +57,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/brevo` | `BrevoTransport` | [Brevo](/transports/brevo) | | `@alexify/mailer/mailjet` | `MailjetTransport` | [Mailjet](/transports/mailjet) | | `@alexify/mailer/mailersend` | `MailerSendTransport` | [MailerSend](/transports/mailersend) | +| `@alexify/mailer/sparkpost` | `SparkPostTransport` | [SparkPost](/transports/sparkpost) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/sparkpost.md b/docs/transports/sparkpost.md new file mode 100644 index 0000000..2e192de --- /dev/null +++ b/docs/transports/sparkpost.md @@ -0,0 +1,97 @@ +# SparkPost transport + +The SparkPost transport sends through the Transmissions API of [SparkPost], from +the SparkPost transport of sently. + +[SparkPost]: https://www.sparkpost.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { SparkPostTransport } = require('@alexify/mailer/sparkpost'); + +const transport = new SparkPostTransport({ + apiKey: process.env.SPARKPOST_API_KEY, + region: 'eu', +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + cc: 'orders@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + tags: ['orders'], + }), +); +if (receipt.successful) console.log(receipt.messageId); // the transmission id +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `apiKey` | (required) | An API key with the Transmissions permission. | +| `region` | `'us'` | `'us'` for `api.sparkpost.com`, `'eu'` for `api.eu.sparkpost.com`. | +| `baseUrl` | the region's | HTTPS, or HTTP to this machine, in place of the region's host. | +| `campaignId` | | The `campaign_id` of every transmission. | +| `transactional` | | Marks every transmission as transactional, or not. | +| `openTracking`, `clickTracking` | the account's | Tracking, unless a message sets its own `tracking`. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before SparkPost took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- SparkPost sends a copy to each recipient. Every copy carries the To field of the + message (`header_to`), and the Cc field goes in its headers, so Cc and Bcc + recipients see what the sender wrote; a Bcc address is in no field. +- `scheduledAt` is `start_time`; `tags` (up to 10) the tags of each recipient, + which reach the events; `metadata` the metadata of the transmission; + `tracking` its `open_tracking` and `click_tracking`. +- Custom header fields, List-* and the threading fields go in the headers of the + content. +- An inline attachment goes in `inline_images`, named by its Content-ID, the + others in `attachments`. + +SparkPost answers with the counts of the recipients it accepted and turned away +(such as those on its suppression list). A receipt counts the latter in +`rejectedRecipientCount`; when none was accepted it fails as `rejected`, not +retryable. A suppressed single recipient is the error 1902, also `rejected`; an +unverified sending domain (7001) is a `configuration` failure, and the 420 of a +spent sending limit a `rate-limit` one. + +## Raw messages + +`sendRaw()` sends a ready MIME message as `email_rfc822`, to the recipients of its +envelope. The API takes it as a JSON string, so its bytes must be UTF-8; others +are refused as `validation`, before any request. + +```js +const receipt = await transport.sendRaw({ + envelope: { from: 'shop@example.com', to: ['user@example.com'] }, + content: mimeBytes, +}); +``` + +## Batches + +`sendMany()` sends `concurrency` transmissions at a time, each with its own +receipt. + +## Verifying the configuration + +`verify()` reads the account (`GET /api/v1/account`); a key without that +permission is verified too. diff --git a/package.json b/package.json index 4a0f4c0..83a18ad 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,10 @@ "types": "./src/mailersend/index.d.ts", "default": "./src/mailersend/index.js" }, + "./sparkpost": { + "types": "./src/sparkpost/index.d.ts", + "default": "./src/sparkpost/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 0577396..a749d2d 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -17,6 +17,7 @@ "./brevo": 7, "./mailjet": 8, "./mailersend": 7, + "./sparkpost": 8, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/sparkpost/config.js b/src/sparkpost/config.js new file mode 100644 index 0000000..5178305 --- /dev/null +++ b/src/sparkpost/config.js @@ -0,0 +1,51 @@ +// Ported from sently v1.2.2 (src/transports/sparkpost.ts): the options of the +// SparkPost transport, the key and the region. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors + +const { checkCredential } = require('../internal/api.js'); +const { + checkConcurrency, + checkOptionalBoolean, + checkOptionalString, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +const BASE_URLS = { us: 'https://api.sparkpost.com', eu: 'https://api.eu.sparkpost.com' }; + +/** + * Resolves the options of the SparkPost transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createSparkPostConfig = (config = {}) => { + const region = config.region ?? 'us'; + if (!Object.hasOwn(BASE_URLS, region)) { + throw new TypeError(`region must be 'us' or 'eu': ${String(region)}`); + } + const resolved = { + apiKey: checkCredential(config.apiKey, 'apiKey', 'SparkPostTransport'), + region, + baseUrl: config.baseUrl ?? BASE_URLS[region], + campaignId: checkOptionalString(config.campaignId, 'campaignId'), + transactional: checkOptionalBoolean(config.transactional, 'transactional'), + openTracking: checkOptionalBoolean(config.openTracking, 'openTracking'), + clickTracking: checkOptionalBoolean(config.clickTracking, 'clickTracking'), + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createSparkPostConfig }; diff --git a/src/sparkpost/http-client.js b/src/sparkpost/http-client.js new file mode 100644 index 0000000..b4790df --- /dev/null +++ b/src/sparkpost/http-client.js @@ -0,0 +1,54 @@ +// Ported from sently v1.2.2 (src/transports/sparkpost.ts): the requests of the +// SparkPost Transmissions API. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: through the shared client of the HTTP providers, which +// reads the status before the body, as sently did not. + +const { createApiClient } = require('../internal/api.js'); + +// The codes of SparkPost whose category the status does not tell: 1902, a +// recipient on the suppression list; 7001, a sending domain not verified. +const CODES = new Map([ + ['1902', { category: 'rejected', retryable: false }], + ['7001', { category: 'configuration', retryable: false }], +]); +// 420: the sending limit of the account is spent. +const STATUSES = new Map([[420, { category: 'rate-limit', retryable: true }]]); + +// SparkPost gives `{ errors: [{ message, description, code }] }`. +const messageOf = (json) => { + const error = Array.isArray(json?.errors) ? json.errors[0] : undefined; + if (error == null) return json?.message; + return error.description ? `${error.message}: ${error.description}` : error.message; +}; + +/** + * The client of the SparkPost API. + * + * @internal + */ +const createSparkPostClient = (config) => { + const client = createApiClient(config, { + id: 'sparkpost', + label: 'SparkPost', + authorize: (headers) => headers.set('Authorization', config.apiKey), + messageOf, + classify: (status, json) => { + const code = Array.isArray(json?.errors) ? json.errors[0]?.code : undefined; + const classified = CODES.get(String(code)) ?? STATUSES.get(status) ?? {}; + return code == null ? classified : { ...classified, code: `sparkpost.${code}` }; + }, + }); + return { + ...client, + send: (transmission, options) => + client.request({ + url: `${config.baseUrl}/api/v1/transmissions`, + body: transmission, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createSparkPostClient }; diff --git a/src/sparkpost/index.d.ts b/src/sparkpost/index.d.ts new file mode 100644 index 0000000..b28b49a --- /dev/null +++ b/src/sparkpost/index.d.ts @@ -0,0 +1,98 @@ +// Types ported from sently v1.2.2 (src/transports) +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link SparkPostTransport}. + */ +export interface SparkPostConfig extends TransportDependencies { + /** An API key with the Transmissions permission. */ + readonly apiKey: string; + /** + * The region of the account, which chooses the host of the API. + * @default "us" + */ + readonly region?: 'us' | 'eu'; + /** + * The base URL of the API, https (or http to this machine), in place of the + * host of the region. + */ + readonly baseUrl?: string; + /** The campaign of every transmission (`campaign_id`). */ + readonly campaignId?: string; + /** Marks every transmission as transactional, or not. */ + readonly transactional?: boolean; + /** Open tracking, unless a message sets its own `tracking`. */ + readonly openTracking?: boolean; + /** Click tracking, unless a message sets its own. */ + readonly clickTracking?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before SparkPost took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * The receipt of {@link SparkPostTransport}: a successful one counts the + * recipients SparkPost turned away, such as those on its suppression list. + */ +export type SparkPostReceipt = + | (Extract, { readonly successful: true }> & { + /** The recipients SparkPost turned away, when there are any. */ + readonly rejectedRecipientCount?: number; + }) + | Extract, { readonly successful: false }>; + +/** + * A transport over the Transmissions API of SparkPost. Cc and Bcc + * recipients see the To field of the message, and the Cc ones its Cc field; + * `scheduledAt` is `start_time`, `tags` the tags of each recipient (up to + * 10), `metadata` that of the transmission, `tracking` its options; a ready + * MIME message goes as `email_rfc822`. + * + * @example + * ```js + * const { SparkPostTransport } = require('@alexify/mailer/sparkpost'); + * const transport = new SparkPostTransport({ + * apiKey: process.env.SPARKPOST_API_KEY, + * region: 'eu', + * }); + * ``` + */ +export declare class SparkPostTransport + implements RawTransport<'sparkpost'>, VerifiableTransport<'sparkpost'> +{ + readonly id = 'sparkpost'; + constructor(config: SparkPostConfig); + send(message: Message, options?: TransportOptions): Promise; + /** Sends a ready MIME message, whose bytes must be UTF-8. */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable; + /** Verifies the API key by reading the account. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/sparkpost/index.js b/src/sparkpost/index.js new file mode 100644 index 0000000..6e4caf3 --- /dev/null +++ b/src/sparkpost/index.js @@ -0,0 +1,7 @@ +// Ported from sently v1.2.2 (src/transports/sparkpost.ts): the entry of the +// SparkPost transport. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors + +const { SparkPostTransport } = require('./sparkpost-transport.js'); + +module.exports = { SparkPostTransport }; diff --git a/src/sparkpost/message-converter.js b/src/sparkpost/message-converter.js new file mode 100644 index 0000000..cbcc6d3 --- /dev/null +++ b/src/sparkpost/message-converter.js @@ -0,0 +1,147 @@ +// Ported from sently v1.2.2 (src/transports/sparkpost.ts): a message as a +// transmission of the SparkPost API. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: cc and bcc recipients get `header_to`, the To field +// every recipient sees, and the cc ones a CC field, where sently put them all +// in `recipients` alone, so the Cc field was lost and each recipient saw only +// itself in To; inline images go in `inline_images`; custom fields, tags, +// metadata, tracking and `scheduledAt` (`start_time`) are sent; and a raw +// message goes as `email_rfc822`. + +const { formatMailbox, toAsciiAddress } = require('../core/address.js'); +const { RawMessageValidationError } = require('../core/raw-message.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { formatAddresses } = require('../internal/provider.js'); + +// "Maximum number of tags - 10 per recipient". +const MAX_TAGS = 10; +const UTF8 = new TextDecoder('utf-8', { fatal: true }); + +// `start_time` is "YYYY-MM-DDTHH:MM:SS+-HH:MM", without milliseconds. +const startTimeOf = (date) => `${date.toISOString().slice(0, 19)}+00:00`; + +const addressOf = (address, headerTo) => { + const result = { email: toAsciiAddress(address.address) }; + if (address.name) result.name = address.name; + if (headerTo !== undefined) result.header_to = headerTo; + return result; +}; + +const optionsOf = (message, config) => { + const options = {}; + const opens = message?.tracking?.opens ?? config.openTracking; + const clicks = message?.tracking?.clicks ?? config.clickTracking; + if (opens !== undefined) options.open_tracking = opens; + if (clicks !== undefined) options.click_tracking = clicks; + if (config.transactional !== undefined) options.transactional = config.transactional; + if (message?.scheduledAt !== undefined) options.start_time = startTimeOf(message.scheduledAt); + return options; +}; + +const recipientsOf = (message) => { + // SparkPost sends a copy to each recipient, whose To field names that + // recipient alone unless `header_to` gives the one every copy shares. + const total = + message.recipients.length + message.ccRecipients.length + message.bccRecipients.length; + const headerTo = + total > 1 ? formatAddresses(message.recipients, formatMailbox).join(', ') : undefined; + const tags = message.tags.length > 0 ? [...message.tags] : undefined; + const recipients = []; + for (const list of [message.recipients, message.ccRecipients, message.bccRecipients]) { + for (let i = 0; i < list.length; i++) { + const recipient = { address: addressOf(list[i], headerTo) }; + if (tags !== undefined) recipient.tags = tags; + recipients.push(recipient); + } + } + return recipients; +}; + +const contentOf = async (message, signal) => { + const content = { from: addressOf(message.sender), subject: message.subject }; + if (message.content.text) content.text = message.content.text; + if ('html' in message.content) content.html = message.content.html; + if (message.replyRecipients.length > 0) { + content.reply_to = formatAddresses(message.replyRecipients, formatMailbox).join(', '); + } + const headers = {}; + let hasHeaders = false; + if (message.ccRecipients.length > 0) { + headers.CC = formatAddresses(message.ccRecipients, formatMailbox).join(', '); + hasHeaders = true; + } + for (const field of headerFieldsOf(message)) { + headers[field.name] = field.value; + hasHeaders = true; + } + if (hasHeaders) content.headers = headers; + for (const item of await base64Attachments(message, signal)) { + const part = { type: item.attachment.contentType, data: item.content }; + if (item.attachment.inline) { + part.name = item.attachment.contentId; + content.inline_images ??= []; + content.inline_images.push(part); + } else { + part.name = item.attachment.filename; + content.attachments ??= []; + content.attachments.push(part); + } + } + return content; +}; + +/** + * Converts a message to a transmission. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The transmission. + * @throws {RangeError} If the message has more than ten tags. + */ +const convertMessage = async (message, config, signal) => { + if (message.tags.length > MAX_TAGS) { + throw new RangeError(`SparkPost takes at most ${MAX_TAGS} tags.`); + } + const transmission = { + recipients: recipientsOf(message), + content: await contentOf(message, signal), + }; + const options = optionsOf(message, config); + if (Object.keys(options).length > 0) transmission.options = options; + if (config.campaignId !== undefined) transmission.campaign_id = config.campaignId; + if (message.metadata !== undefined) transmission.metadata = { ...message.metadata }; + return transmission; +}; + +// The API takes the message as a JSON string, so its bytes must be UTF-8. +const textOf = (bytes) => { + try { + return UTF8.decode(bytes); + } catch { + throw new RawMessageValidationError('SparkPost takes a raw message only as UTF-8.', 'content'); + } +}; + +/** + * Converts a raw message, read with `readRawMessage`, to a transmission of + * its RFC 822 text to the recipients of its envelope. + * + * @param raw The envelope and bytes. + * @param config The configuration. + * @returns The transmission. + */ +const convertRawMessage = (raw, config) => { + const recipients = []; + for (const email of raw.envelope.to) recipients.push({ address: { email } }); + const transmission = { + recipients, + content: { email_rfc822: textOf(raw.bytes) }, + }; + const options = optionsOf(undefined, config); + if (Object.keys(options).length > 0) transmission.options = options; + if (config.campaignId !== undefined) transmission.campaign_id = config.campaignId; + return transmission; +}; + +module.exports = { convertMessage, convertRawMessage }; diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js new file mode 100644 index 0000000..68faa32 --- /dev/null +++ b/src/sparkpost/sparkpost-transport.js @@ -0,0 +1,143 @@ +// Ported from sently v1.2.2 (src/transports/sparkpost.ts): a transport over +// the SparkPost Transmissions API, and `verify`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: `sendRaw` sends a ready message as `email_rfc822`; a +// batch goes as concurrent requests; and the recipients SparkPost turns away +// are counted on the receipt, or fail it when it took none. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createSparkPostConfig } = require('./config.js'); +const { createSparkPostClient } = require('./http-client.js'); +const { convertMessage, convertRawMessage } = require('./message-converter.js'); + +// The receipt of `{ results: { id, total_accepted_recipients, +// total_rejected_recipients } }`. +const receiptOf = (json) => { + const results = json?.results; + const accepted = results?.total_accepted_recipients; + const rejected = results?.total_rejected_recipients; + if (accepted === 0 && rejected > 0) { + return createFailedReceipt('SparkPost: every recipient was rejected.', { + provider: 'sparkpost', + category: 'rejected', + retryable: false, + code: 'sparkpost.recipients_rejected', + providerDetails: results, + }); + } + const messageId = results?.id; + if (typeof messageId !== 'string' || messageId === '') { + return createFailedReceipt('SparkPost answered without a transmission id.', { + provider: 'sparkpost', + deliveryUnknown: true, + }); + } + const receipt = { successful: true, messageId, provider: 'sparkpost' }; + if (Number.isInteger(rejected) && rejected > 0) receipt.rejectedRecipientCount = rejected; + return receipt; +}; + +/** + * A transport over the Transmissions API of SparkPost. + */ +class SparkPostTransport { + id = 'sparkpost'; + + constructor(config) { + this.config = createSparkPostConfig(config); + this.httpClient = createSparkPostClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message as a transmission. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the id of the transmission. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const transmission = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + return receiptOf((await this.httpClient.send(transmission, options)).json); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('sparkpost', error); + } + } + + /** + * Sends a ready MIME message as `email_rfc822`, to the recipients of its + * envelope. + * + * @param message The raw message; its bytes must be UTF-8. + * @param options Optional abort signal and retry callback. + * @returns The receipt. + * @throws The reason of the signal if it aborts. + */ + async sendRaw(message, options) { + if (message?.content instanceof Promise) message.content.catch(() => {}); + options?.signal?.throwIfAborted(); + try { + const raw = await readRawMessage(message, options?.signal); + const transmission = convertRawMessage(raw, this.config); + return receiptOf((await this.httpClient.send(transmission, options)).json); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return rawMessageFailure('sparkpost', error) ?? providerFailure('sparkpost', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the API key by reading the account (`GET /api/v1/account`); a key + * without that permission is verified too. + * + * @param options Optional abort signal. + * @throws {Error} If SparkPost refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'SparkPost', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/api/v1/account`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { SparkPostTransport }; diff --git a/tests/types/sparkpost.test-d.ts b/tests/types/sparkpost.test-d.ts new file mode 100644 index 0000000..a089805 --- /dev/null +++ b/tests/types/sparkpost.test-d.ts @@ -0,0 +1,28 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type RawTransport, type VerifiableTransport } from '../../index.js'; +import { SparkPostTransport, type SparkPostReceipt } from '../../src/sparkpost/index.js'; + +const transport = new SparkPostTransport({ + apiKey: 'sp-key', + region: 'eu', + campaignId: 'autumn', + transactional: true, + openTracking: false, + concurrency: 2, +}); +expectType<'sparkpost'>(transport.id); +expectAssignable>(transport); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>(transport.send(message)); +expectType>(transport.sendMany([message])); +expectType>(transport.verify()); +const receipt = await transport.send(message); +if (receipt.successful) expectType(receipt.rejectedRecipientCount); +expectError(new SparkPostTransport({})); +expectError(new SparkPostTransport({ apiKey: 'k', region: 'asia' })); diff --git a/tests/unit/sparkpost.test.js b/tests/unit/sparkpost.test.js new file mode 100644 index 0000000..cc1c5fb --- /dev/null +++ b/tests/unit/sparkpost.test.js @@ -0,0 +1,259 @@ +// Not ported: tests of ./sparkpost, ported from sently's SparkPost transport, +// against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { SparkPostTransport } = require('../../src/sparkpost/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const transport = (config = {}) => + new SparkPostTransport({ apiKey: 'sp-secret', retries: 0, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const results = (id = 'sp-1', rejected = 0, accepted = 1) => + json({ + results: { total_rejected_recipients: rejected, total_accepted_recipients: accepted, id }, + }); + +test('SparkPost: a message is a transmission, with the raw key', async (t) => { + const requests = stubFetch(t, () => results()); + const receipt = await transport({ campaignId: 'autumn', transactional: true }).send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: 'support@example.com', + priority: 'high', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + tracking: { opens: false, clicks: true }, + list: { unsubscribe: 'https://example.com/u', oneClick: true }, + scheduledAt: new Date('2026-10-18T09:00:00.250Z'), + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'sp-1', provider: 'sparkpost' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.sparkpost.com/api/v1/transmissions'); + assert.strictEqual(request.headers.get('authorization'), 'sp-secret'); + assert.strictEqual(request.redirect, 'manual'); + const body = request.json; + const headerTo = 'bob@example.com, Carol '; + const tags = ['orders', 'eu']; + assert.deepStrictEqual(body.recipients, [ + { address: { email: 'bob@example.com', header_to: headerTo }, tags }, + { address: { email: 'carol@xn--bcher-kva.de', name: 'Carol', header_to: headerTo }, tags }, + { address: { email: 'dave@example.com', header_to: headerTo }, tags }, + { address: { email: 'erin@example.com', header_to: headerTo }, tags }, + ]); + assert.deepStrictEqual(body.options, { + open_tracking: false, + click_tracking: true, + transactional: true, + start_time: '2026-10-18T09:00:00+00:00', + }); + assert.strictEqual(body.campaign_id, 'autumn'); + assert.deepStrictEqual(body.metadata, { orderId: '42' }); + const { content } = body; + assert.deepStrictEqual(content.from, { email: 'shop@example.com', name: 'Shop' }); + assert.strictEqual(content.subject, 'Your order'); + assert.strictEqual(content.text, 'Shipped'); + assert.strictEqual(content.html, '

Shipped

'); + assert.strictEqual(content.reply_to, 'support@example.com'); + assert.deepStrictEqual(content.headers, { + CC: 'dave@example.com', + 'X-Priority': '1', + 'X-MSMail-Priority': 'High', + Importance: 'High', + 'List-Unsubscribe': '', + 'List-Unsubscribe-Post': 'List-Unsubscribe=One-Click', + }); + assert.deepStrictEqual(content.attachments, [ + { type: 'text/plain', data: 'aGk=', name: 'r.txt' }, + ]); + assert.deepStrictEqual(content.inline_images, [ + { type: 'image/png', data: 'AQ==', name: 'logo@example.com' }, + ]); +}); + +test('SparkPost: one recipient needs no header_to, a plain message no options', async (t) => { + const requests = stubFetch(t, () => results()); + await transport().send(message({ content: { text: 'plain' } })); + const body = requests[0].json; + assert.deepStrictEqual(body.recipients, [{ address: { email: 'bob@example.com' } }]); + assert.strictEqual(body.options, undefined); + assert.strictEqual(body.content.headers, undefined); + assert.strictEqual(body.content.html, undefined); + assert.strictEqual(body.campaign_id, undefined); +}); + +test('SparkPost: the EU region and a base URL choose the host', async (t) => { + const requests = stubFetch(t, () => results()); + await transport({ region: 'eu' }).send(message()); + await transport({ region: 'eu', baseUrl: 'http://127.0.0.1:9000' }).send(message()); + assert.strictEqual(requests[0].url, 'https://api.eu.sparkpost.com/api/v1/transmissions'); + assert.strictEqual(requests[1].url, 'http://127.0.0.1:9000/api/v1/transmissions'); +}); + +test('SparkPost: rejected recipients are counted, or fail the receipt when none went', async (t) => { + const answers = [results('sp-2', 1, 2), results('sp-3', 2, 0), json({ results: {} })]; + stubFetch(t, () => answers.shift()); + const partial = await transport().send(message()); + assert.strictEqual(partial.successful, true); + assert.strictEqual(partial.rejectedRecipientCount, 1); + const none = await transport().send(message()); + assert.strictEqual(none.successful, false); + assert.strictEqual(none.retryable, false); + assert.strictEqual(none.errors[0].category, 'rejected'); + assert.strictEqual(none.errors[0].code, 'sparkpost.recipients_rejected'); + const unknown = await transport().send(message()); + assert.strictEqual(unknown.deliveryUnknown, true); +}); + +test('SparkPost: an error code says what failed', async (t) => { + const answers = [ + json( + { + errors: [ + { + message: 'Message generation rejected', + description: 'recipient address suppressed due to customer policy', + code: '1902', + }, + ], + }, + 400, + ), + json( + { errors: [{ message: 'Unconfigured or unverified sending domain.', code: '7001' }] }, + 400, + ), + json({ errors: [{ message: 'Exceed Sending Limit (daily)', code: '2102' }] }, 420), + json({ errors: [{ message: 'Unauthorized.' }] }, 401), + ]; + stubFetch(t, () => answers.shift()); + const suppressed = await transport().send(message()); + assert.strictEqual( + suppressed.errorMessages[0], + 'SparkPost: Message generation rejected: recipient address suppressed due to customer policy', + ); + assert.strictEqual(suppressed.errors[0].category, 'rejected'); + assert.strictEqual(suppressed.errors[0].code, 'sparkpost.1902'); + assert.strictEqual(suppressed.retryable, false); + assert.strictEqual((await transport().send(message())).errors[0].category, 'configuration'); + const limited = await transport().send(message()); + assert.strictEqual(limited.errors[0].category, 'rate-limit'); + assert.strictEqual(limited.retryable, true); + const auth = await transport().send(message()); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errorMessages[0], 'SparkPost: Unauthorized.'); +}); + +test('SparkPost: more than ten tags are refused before a request', async (t) => { + const requests = stubFetch(t, () => results()); + const tags = []; + for (let i = 0; i < 11; i++) tags.push(`t${i}`); + const receipt = await transport().send(message({ tags })); + assert.match(receipt.errorMessages[0], /at most 10 tags/); + assert.strictEqual(requests.length, 0); +}); + +test('SparkPost: sendRaw sends the message as email_rfc822 to its envelope', async (t) => { + const requests = stubFetch(t, () => results('sp-raw')); + const text = 'From: shop@example.com\r\nTo: bob@example.com\r\nSubject: Grüße\r\n\r\nHi\r\n'; + const receipt = await transport({ campaignId: 'raw' }).sendRaw({ + envelope: { from: 'shop@example.com', to: ['bob@example.com', 'erin@bücher.de'] }, + content: new TextEncoder().encode(text), + }); + assert.strictEqual(receipt.messageId, 'sp-raw'); + assert.deepStrictEqual(requests[0].json, { + recipients: [ + { address: { email: 'bob@example.com' } }, + { address: { email: 'erin@xn--bcher-kva.de' } }, + ], + content: { email_rfc822: text }, + campaign_id: 'raw', + }); + const latin1 = await transport().sendRaw({ + envelope: { from: 'shop@example.com', to: ['bob@example.com'] }, + content: new Uint8Array([...new TextEncoder().encode('Subject: x\r\n\r\n'), 0xe9, 13, 10]), + }); + assert.strictEqual(latin1.successful, false); + assert.strictEqual(latin1.errors[0].category, 'validation'); + assert.match(latin1.errorMessages[0], /only as UTF-8/); + const invalid = await transport().sendRaw({ + envelope: { from: 'shop@example.com', to: [] }, + content: new TextEncoder().encode(text), + }); + assert.strictEqual(invalid.errors[0].category, 'validation'); + assert.strictEqual(requests.length, 1); +}); + +test('SparkPost: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => results(`sp-${index++}`)); + const tags = []; + for (let i = 0; i < 11; i++) tags.push(`t${i}`); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ tags }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('SparkPost: verify reads the account, the options are checked, the key never shows', async (t) => { + const answers = [json({ results: {} }), json({ errors: [{ message: 'Unauthorized.' }] }, 401)]; + const requests = stubFetch(t, () => answers.shift()); + await transport({ region: 'eu' }).verify(); + assert.strictEqual(requests[0].url, 'https://api.eu.sparkpost.com/api/v1/account'); + assert.strictEqual(requests[0].headers.get('authorization'), 'sp-secret'); + await assert.rejects(transport().verify(), /SparkPost refused the credentials/); + assert.throws(() => new SparkPostTransport({}), /needs apiKey/); + assert.throws(() => transport({ region: 'asia' }), /region must be 'us' or 'eu'/); + assert.throws(() => transport({ campaignId: 42 }), /campaignId must be a non-empty string/); + assert.throws(() => transport({ transactional: 'yes' }), /transactional must be a boolean/); + assert.throws(() => transport({ baseUrl: 'http://api.example.com' }), /baseUrl/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('sp-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('sp-secret')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); + await assert.rejects( + made.sendRaw( + { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }, + { signal: controller.signal }, + ), + /stop/, + ); +}); From c2b28408c2bccc48032d12cacfac4399aebde7b5 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:51:05 +0300 Subject: [PATCH 022/143] feat(zeptomail): a transport over the ZeptoMail API Ported from unemail's ZeptoMail driver onto the shared client of the HTTP providers. The Send Mail token gets its `Zoho-enczapikey` scheme unless it has it; a message's one tag is its `client_reference`, as ZeptoMail has no tags; metadata goes in `X-Metadata-` headers and tracking in `track_opens` and `track_clicks`; inline images in `inline_images`. The codes in the details of an error choose the category (auth, rate limit, spent credits, malformed request). Its limits and a scheduled message are refused before a request. A batch goes as concurrent single requests rather than the batch endpoint, which sends one message to many recipients with one answer. There is no `verify()`: a Send Mail token can only send. Co-Authored-By: Claude Opus 5.5 --- docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/zeptomail.md | 82 ++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/zeptomail/config.js | 48 ++++++ src/zeptomail/http-client.js | 73 +++++++++ src/zeptomail/index.d.ts | 78 ++++++++++ src/zeptomail/index.js | 7 + src/zeptomail/message-converter.js | 116 +++++++++++++++ src/zeptomail/zeptomail-transport.js | 76 ++++++++++ tests/types/zeptomail.test-d.ts | 24 +++ tests/unit/zeptomail.test.js | 214 +++++++++++++++++++++++++++ 13 files changed, 725 insertions(+) create mode 100644 docs/transports/zeptomail.md create mode 100644 src/zeptomail/config.js create mode 100644 src/zeptomail/http-client.js create mode 100644 src/zeptomail/index.d.ts create mode 100644 src/zeptomail/index.js create mode 100644 src/zeptomail/message-converter.js create mode 100644 src/zeptomail/zeptomail-transport.js create mode 100644 tests/types/zeptomail.test-d.ts create mode 100644 tests/unit/zeptomail.test.js diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 20e865c..77deeff 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -89,6 +89,7 @@ const guide = [ { text: 'Mailjet', link: '/transports/mailjet' }, { text: 'MailerSend', link: '/transports/mailersend' }, { text: 'SparkPost', link: '/transports/sparkpost' }, + { text: 'ZeptoMail', link: '/transports/zeptomail' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 9d9044f..ae30eda 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -58,6 +58,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/mailjet` | `MailjetTransport` | [Mailjet](/transports/mailjet) | | `@alexify/mailer/mailersend` | `MailerSendTransport` | [MailerSend](/transports/mailersend) | | `@alexify/mailer/sparkpost` | `SparkPostTransport` | [SparkPost](/transports/sparkpost) | +| `@alexify/mailer/zeptomail` | `ZeptoMailTransport` | [ZeptoMail](/transports/zeptomail) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/zeptomail.md b/docs/transports/zeptomail.md new file mode 100644 index 0000000..499230e --- /dev/null +++ b/docs/transports/zeptomail.md @@ -0,0 +1,82 @@ +# ZeptoMail transport + +The ZeptoMail transport sends through the email API of [ZeptoMail], the +transactional service of Zoho, from the ZeptoMail driver of unemail. + +[ZeptoMail]: https://www.zoho.com/zeptomail/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { ZeptoMailTransport } = require('@alexify/mailer/zeptomail'); + +const transport = new ZeptoMailTransport({ + token: process.env.ZEPTOMAIL_TOKEN, + bounceAddress: 'bounce@bounces.example.com', +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + tags: ['order-42'], + }), +); +if (receipt.successful) console.log(receipt.messageId); // the request_id +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `token` | (required) | The Send Mail token of a Mail Agent; `Zoho-enczapikey ` is added unless it is there. | +| `baseUrl` | `https://api.zeptomail.com` | HTTPS, or HTTP to this machine; the host of the account's data center, such as `https://api.zeptomail.eu`. | +| `bounceAddress` | | The bounce address, on a bounce domain verified with ZeptoMail. | +| `clientReference` | | The `client_reference` of a message without a tag. | +| `openTracking`, `clickTracking` | the agent's | Tracking, unless a message sets its own `tracking`. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before ZeptoMail took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- ZeptoMail has no tags: a message's one tag is its `client_reference`, which its + reports and events show; a second tag is refused. +- `metadata` goes in `X-Metadata-` headers (`mime_headers`), with custom + header fields, List-* and the threading fields; `tracking` is `track_opens` + and `track_clicks`. +- An inline attachment goes in `inline_images` with its Content-ID as `cid`. +- ZeptoMail takes 500 addresses in each of `to`, `cc` and `bcc`, 60 + attachments and a subject of 500 characters; more are refused before the + request. It cannot schedule a message, so `scheduledAt` is refused as + `validation`. + +ZeptoMail answers 201 with a `request_id`, the receipt's id; an answer without +one is reported as `deliveryUnknown`. The code in the details of an error says +more than its status: an invalid token (`SERR_157`) is `auth`, a rate limit +(`SM_133`) `rate-limit`, spent credits (`LE_101`, `LE_102`) `configuration`, +and a malformed request (`TM_3201`, `TM_3301`, `TM_8001`) `validation`. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. The batch endpoint is not used: it sends one message to many +recipients, with one answer for all of them. + +## Verifying the configuration + +This transport has no `verify()`: a Send Mail token can only send, and +ZeptoMail offers no request that checks it. diff --git a/package.json b/package.json index 83a18ad..b22b625 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,10 @@ "types": "./src/sparkpost/index.d.ts", "default": "./src/sparkpost/index.js" }, + "./zeptomail": { + "types": "./src/zeptomail/index.d.ts", + "default": "./src/zeptomail/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index a749d2d..07761b1 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -18,6 +18,7 @@ "./mailjet": 8, "./mailersend": 7, "./sparkpost": 8, + "./zeptomail": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/zeptomail/config.js b/src/zeptomail/config.js new file mode 100644 index 0000000..7a603a4 --- /dev/null +++ b/src/zeptomail/config.js @@ -0,0 +1,48 @@ +// Ported from unemail v0.7.0 (src/drivers/zeptomail.ts): the options of the +// ZeptoMail driver. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkConcurrency, + checkOptionalBoolean, + checkOptionalString, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +const AUTH_PREFIX = 'zoho-enczapikey '; + +/** + * Resolves the options of the ZeptoMail transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createZeptoMailConfig = (config = {}) => { + const token = checkCredential(config.token, 'token', 'ZeptoMailTransport'); + const resolved = { + // The token is shown with its scheme, which may be pasted with it. + token: token.toLowerCase().startsWith(AUTH_PREFIX) ? token : `Zoho-enczapikey ${token}`, + baseUrl: config.baseUrl ?? 'https://api.zeptomail.com', + bounceAddress: checkOptionalString(config.bounceAddress, 'bounceAddress'), + clientReference: checkOptionalString(config.clientReference, 'clientReference'), + openTracking: checkOptionalBoolean(config.openTracking, 'openTracking'), + clickTracking: checkOptionalBoolean(config.clickTracking, 'clickTracking'), + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createZeptoMailConfig }; diff --git a/src/zeptomail/http-client.js b/src/zeptomail/http-client.js new file mode 100644 index 0000000..edec2de --- /dev/null +++ b/src/zeptomail/http-client.js @@ -0,0 +1,73 @@ +// Ported from unemail v0.7.0 (src/drivers/zeptomail.ts): the requests of the +// ZeptoMail API and the category of its error codes, `classifyZeptoError`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers. + +const { createApiClient } = require('../internal/api.js'); + +// The codes of `error.details`, which say more than the status: one 400 +// covers a bad token, an unverified sender and a long subject. +const DETAIL_CODES = new Map([ + ['SERR_156', { category: 'auth', retryable: false }], + ['SERR_157', { category: 'auth', retryable: false }], + ['SM_111', { category: 'auth', retryable: false }], + ['SM_128', { category: 'auth', retryable: false }], + ['AE_101', { category: 'auth', retryable: false }], + ['SM_133', { category: 'rate-limit', retryable: true }], + ['SMI_115', { category: 'rate-limit', retryable: true }], + // Credits spent come back only when bought. + ['LE_101', { category: 'configuration', retryable: false }], + ['LE_102', { category: 'configuration', retryable: false }], +]); +// The codes of a request that is malformed or too large. +const INVALID_CODES = new Set(['TM_3201', 'TM_3301', 'TM_8001']); +const INVALID = { category: 'validation', retryable: false }; + +// `{ error: { code, message, details: [{ code, message, target }] } }`. +const messageOf = (json) => { + const error = json?.error; + if (error == null || typeof error !== 'object') return undefined; + const detail = Array.isArray(error.details) ? error.details[0] : undefined; + let message = ''; + for (const part of [error.message, detail?.message, detail?.target]) { + if (typeof part === 'string' && part !== '') message += message === '' ? part : `: ${part}`; + } + return message === '' ? undefined : message; +}; + +const classify = (_status, json) => { + const error = json?.error; + if (error == null || typeof error !== 'object') return undefined; + const detail = Array.isArray(error.details) ? error.details[0]?.code : undefined; + const classified = + DETAIL_CODES.get(detail) ?? (INVALID_CODES.has(error.code) ? INVALID : undefined) ?? {}; + const code = detail ?? error.code; + return typeof code === 'string' ? { ...classified, code: `zeptomail.${code}` } : classified; +}; + +/** + * The client of the ZeptoMail API. + * + * @internal + */ +const createZeptoMailClient = (config) => { + const client = createApiClient(config, { + id: 'zeptomail', + label: 'ZeptoMail', + authorize: (headers) => headers.set('Authorization', config.token), + messageOf, + classify, + }); + return { + ...client, + send: (payload, options) => + client.request({ + url: `${config.baseUrl}/v1.1/email`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createZeptoMailClient }; diff --git a/src/zeptomail/index.d.ts b/src/zeptomail/index.d.ts new file mode 100644 index 0000000..f4b4ddf --- /dev/null +++ b/src/zeptomail/index.d.ts @@ -0,0 +1,78 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + Transport, + TransportDependencies, + TransportOptions, +} from '../../index.js'; + +/** + * The options of {@link ZeptoMailTransport}. + */ +export interface ZeptoMailConfig extends TransportDependencies { + /** + * The Send Mail token of a Mail Agent; `Zoho-enczapikey ` is added before + * it unless it is there already. + */ + readonly token: string; + /** + * The base URL of the API, https (or http to this machine); an account in + * another data center uses that center's host, such as + * `https://api.zeptomail.eu`. + * @default "https://api.zeptomail.com" + */ + readonly baseUrl?: string; + /** The bounce address, on a bounce domain verified with ZeptoMail. */ + readonly bounceAddress?: string; + /** The `client_reference` of a message without a tag. */ + readonly clientReference?: string; + /** Open tracking, unless a message sets its own `tracking`. */ + readonly openTracking?: boolean; + /** Click tracking, unless a message sets its own. */ + readonly clickTracking?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before ZeptoMail took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * A transport over the email API of ZeptoMail (Zoho). A message's one tag is + * its `client_reference`, `metadata` goes in `X-Metadata-` headers, + * `tracking` in `track_opens` and `track_clicks`; ZeptoMail cannot schedule a + * message. It has no `verify()`: a Send Mail token can only send. + * + * @example + * ```js + * const { ZeptoMailTransport } = require('@alexify/mailer/zeptomail'); + * const transport = new ZeptoMailTransport({ + * token: process.env.ZEPTOMAIL_TOKEN, + * bounceAddress: 'bounce@bounces.example.com', + * }); + * ``` + */ +export declare class ZeptoMailTransport implements Transport<'zeptomail'> { + readonly id = 'zeptomail'; + constructor(config: ZeptoMailConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; +} diff --git a/src/zeptomail/index.js b/src/zeptomail/index.js new file mode 100644 index 0000000..2e6f432 --- /dev/null +++ b/src/zeptomail/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/zeptomail.ts): the entry of the +// ZeptoMail transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { ZeptoMailTransport } = require('./zeptomail-transport.js'); + +module.exports = { ZeptoMailTransport }; diff --git a/src/zeptomail/message-converter.js b/src/zeptomail/message-converter.js new file mode 100644 index 0000000..c91d534 --- /dev/null +++ b/src/zeptomail/message-converter.js @@ -0,0 +1,116 @@ +// Ported from unemail v0.7.0 (src/drivers/zeptomail.ts): a message as the +// request of ZeptoMail's `/v1.1/email`, `sharedPayload` and `validate`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a tag is the `client_reference`, a second one +// refused, as ZeptoMail has no tags; and a scheduled message is refused. + +const { toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +// At most 500 addresses in each of to, cc and bcc (TM_8001, SM_127). +const MAX_RECIPIENTS = 500; +const MAX_ATTACHMENTS = 60; +const MAX_SUBJECT = 500; + +const addressOf = (address) => { + const result = { address: toAsciiAddress(address.address) }; + if (address.name) result.name = address.name; + return result; +}; + +const recipientsOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push({ email_address: addressOf(addresses[i]) }); + return list; +}; + +const checkLimits = (message) => { + refuseScheduledAt(message, 'zeptomail'); + for (const [field, name] of [ + ['recipients', 'to'], + ['ccRecipients', 'cc'], + ['bccRecipients', 'bcc'], + ]) { + if (message[field].length > MAX_RECIPIENTS) { + throw new RangeError(`ZeptoMail takes at most ${MAX_RECIPIENTS} ${name} recipients.`); + } + } + if (message.attachments.length > MAX_ATTACHMENTS) { + throw new RangeError(`ZeptoMail takes at most ${MAX_ATTACHMENTS} attachments.`); + } + if (message.subject.length > MAX_SUBJECT) { + throw new RangeError(`ZeptoMail takes a subject of at most ${MAX_SUBJECT} characters.`); + } + if (message.tags.length > 1) { + throw new RangeError('ZeptoMail takes one tag per message, as its client_reference.'); + } +}; + +const headersOf = (message) => { + const headers = {}; + let hasHeaders = false; + for (const field of headerFieldsOf(message)) { + headers[field.name] = field.value; + hasHeaders = true; + } + // ZeptoMail has no metadata of its own: its events carry the headers. + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers[`X-Metadata-${key}`] = message.metadata[key]; + hasHeaders = true; + } + } + return hasHeaders ? headers : undefined; +}; + +/** + * Converts a message to a request of `/v1.1/email`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message is over a limit of ZeptoMail. + * @throws {TypeError} If it is scheduled. + */ +const convertMessage = async (message, config, signal) => { + checkLimits(message); + const payload = { + from: addressOf(message.sender), + to: recipientsOf(message.recipients), + subject: message.subject, + }; + if (message.ccRecipients.length > 0) payload.cc = recipientsOf(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.bcc = recipientsOf(message.bccRecipients); + if (message.replyRecipients.length > 0) { + payload.reply_to = []; + for (const address of message.replyRecipients) payload.reply_to.push(addressOf(address)); + } + if (message.content.text) payload.textbody = message.content.text; + if ('html' in message.content) payload.htmlbody = message.content.html; + const headers = headersOf(message); + if (headers !== undefined) payload.mime_headers = headers; + const reference = message.tags[0] ?? config.clientReference; + if (reference !== undefined) payload.client_reference = reference; + if (config.bounceAddress !== undefined) payload.bounce_address = config.bounceAddress; + const opens = message.tracking?.opens ?? config.openTracking; + const clicks = message.tracking?.clicks ?? config.clickTracking; + if (opens !== undefined) payload.track_opens = opens; + if (clicks !== undefined) payload.track_clicks = clicks; + for (const item of await base64Attachments(message, signal)) { + const part = { content: item.content, mime_type: item.attachment.contentType }; + if (item.attachment.inline) { + part.cid = item.attachment.contentId; + payload.inline_images ??= []; + payload.inline_images.push(part); + } else { + part.name = item.attachment.filename; + payload.attachments ??= []; + payload.attachments.push(part); + } + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/zeptomail/zeptomail-transport.js b/src/zeptomail/zeptomail-transport.js new file mode 100644 index 0000000..b8529d0 --- /dev/null +++ b/src/zeptomail/zeptomail-transport.js @@ -0,0 +1,76 @@ +// Ported from unemail v0.7.0 (src/drivers/zeptomail.ts): a transport over the +// ZeptoMail API. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a batch goes as concurrent single requests, not as the +// `/email/batch` of one message to many recipients, whose one answer every +// message shared. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { createZeptoMailConfig } = require('./config.js'); +const { createZeptoMailClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +/** + * A transport over the email API of ZeptoMail. + */ +class ZeptoMailTransport { + id = 'zeptomail'; + + constructor(config) { + this.config = createZeptoMailConfig(config); + this.httpClient = createZeptoMailClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/v1.1/email`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the `request_id` of the answer. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const answer = await this.httpClient.send(payload, options); + const messageId = answer.json?.request_id; + if (typeof messageId !== 'string' || messageId === '') { + return createFailedReceipt('ZeptoMail answered without a request_id.', { + provider: 'zeptomail', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'zeptomail' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('zeptomail', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } +} + +module.exports = { ZeptoMailTransport }; diff --git a/tests/types/zeptomail.test-d.ts b/tests/types/zeptomail.test-d.ts new file mode 100644 index 0000000..2f3a67b --- /dev/null +++ b/tests/types/zeptomail.test-d.ts @@ -0,0 +1,24 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type Transport } from '../../index.js'; +import { ZeptoMailTransport } from '../../src/zeptomail/index.js'; + +const transport = new ZeptoMailTransport({ + token: 'zm-token', + baseUrl: 'https://api.zeptomail.eu', + bounceAddress: 'bounce@bounces.example.com', + clientReference: 'ref', + clickTracking: true, + concurrency: 2, +}); +expectType<'zeptomail'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectError(transport.verify()); +expectError(new ZeptoMailTransport({})); diff --git a/tests/unit/zeptomail.test.js b/tests/unit/zeptomail.test.js new file mode 100644 index 0000000..56b7a35 --- /dev/null +++ b/tests/unit/zeptomail.test.js @@ -0,0 +1,214 @@ +// Not ported: tests of ./zeptomail, ported from unemail's ZeptoMail driver, +// against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { ZeptoMailTransport } = require('../../src/zeptomail/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const transport = (config = {}) => + new ZeptoMailTransport({ token: 'zm-secret', retries: 0, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const accepted = (id = 'zm-1') => + json({ data: [{ code: 'EM_104', message: 'Email request received' }], request_id: id }, 201); +const failure = (code, text, detail, status = 400) => + json({ error: { code, message: text, details: detail === undefined ? [] : [detail] } }, status); + +test('ZeptoMail: a message is the request of /v1.1/email, with the token', async (t) => { + const requests = stubFetch(t, () => accepted()); + const receipt = await transport({ + bounceAddress: 'bounce@b.example.com', + clickTracking: true, + }).send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: ['support@example.com', { name: 'Desk', address: 'desk@example.com' }], + priority: 'low', + tags: ['order-42'], + metadata: { orderId: '42' }, + tracking: { opens: false }, + list: { unsubscribe: 'https://example.com/u' }, + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'zm-1', provider: 'zeptomail' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://api.zeptomail.com/v1.1/email'); + assert.strictEqual(request.headers.get('authorization'), 'Zoho-enczapikey zm-secret'); + assert.strictEqual(request.redirect, 'manual'); + const body = request.json; + assert.deepStrictEqual(body.from, { address: 'shop@example.com', name: 'Shop' }); + assert.deepStrictEqual(body.to, [ + { email_address: { address: 'bob@example.com' } }, + { email_address: { address: 'carol@xn--bcher-kva.de', name: 'Carol' } }, + ]); + assert.deepStrictEqual(body.cc, [{ email_address: { address: 'dave@example.com' } }]); + assert.deepStrictEqual(body.bcc, [{ email_address: { address: 'erin@example.com' } }]); + assert.deepStrictEqual(body.reply_to, [ + { address: 'support@example.com' }, + { address: 'desk@example.com', name: 'Desk' }, + ]); + assert.strictEqual(body.subject, 'Your order'); + assert.strictEqual(body.textbody, 'Shipped'); + assert.strictEqual(body.htmlbody, '

Shipped

'); + assert.strictEqual(body.client_reference, 'order-42'); + assert.strictEqual(body.bounce_address, 'bounce@b.example.com'); + assert.strictEqual(body.track_opens, false); + assert.strictEqual(body.track_clicks, true); + assert.deepStrictEqual(body.mime_headers, { + 'X-Priority': '5', + 'X-MSMail-Priority': 'Low', + Importance: 'Low', + 'List-Unsubscribe': '', + 'X-Metadata-orderId': '42', + }); + assert.deepStrictEqual(body.attachments, [ + { content: 'aGk=', mime_type: 'text/plain', name: 'r.txt' }, + ]); + assert.deepStrictEqual(body.inline_images, [ + { content: 'AQ==', mime_type: 'image/png', cid: 'logo@example.com' }, + ]); +}); + +test('ZeptoMail: a plain message sends no options, a token keeps a scheme given', async (t) => { + const requests = stubFetch(t, () => accepted()); + await transport({ token: 'Zoho-enczapikey zm-secret', clientReference: 'ref' }).send( + message({ content: { text: 'plain' } }), + ); + const [request] = requests; + assert.strictEqual(request.headers.get('authorization'), 'Zoho-enczapikey zm-secret'); + const body = request.json; + assert.strictEqual(body.client_reference, 'ref'); + assert.strictEqual(body.htmlbody, undefined); + for (const key of ['mime_headers', 'track_opens', 'track_clicks', 'bounce_address', 'cc']) { + assert.strictEqual(body[key], undefined, key); + } +}); + +test('ZeptoMail: the codes of the details say what failed', async (t) => { + const answers = [ + failure('TM_4001', 'Access Denied', { + code: 'SERR_157', + message: 'Invalid API Token found', + }), + failure('TM_4001', 'Access Denied', { code: 'SM_133', message: 'Rate limit exceeded' }), + failure('TM_4001', 'Access Denied', { code: 'LE_102', message: 'Credits exhausted' }), + failure('TM_3201', 'Mandatory Field Missing', { + code: 'SERR_101', + message: 'Mandatory field missing', + target: 'subject', + }), + failure('TM_5001', 'Internal error', undefined, 400), + json({ data: [] }, 201), + ]; + stubFetch(t, () => answers.shift()); + const auth = await transport().send(message()); + assert.strictEqual(auth.errorMessages[0], 'ZeptoMail: Access Denied: Invalid API Token found'); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errors[0].code, 'zeptomail.SERR_157'); + const limited = await transport().send(message()); + assert.strictEqual(limited.errors[0].category, 'rate-limit'); + assert.strictEqual(limited.retryable, true); + const exhausted = await transport().send(message()); + assert.strictEqual(exhausted.errors[0].category, 'configuration'); + assert.strictEqual(exhausted.retryable, false); + const invalid = await transport().send(message()); + assert.strictEqual( + invalid.errorMessages[0], + 'ZeptoMail: Mandatory Field Missing: Mandatory field missing: subject', + ); + assert.strictEqual(invalid.errors[0].category, 'validation'); + const other = await transport().send(message()); + assert.strictEqual(other.errors[0].code, 'zeptomail.TM_5001'); + assert.strictEqual(other.errors[0].category, 'validation'); + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); +}); + +test('ZeptoMail: its limits, a second tag and a schedule are refused before a request', async (t) => { + const requests = stubFetch(t, () => accepted()); + const many = (count) => { + const list = []; + for (let i = 0; i < count; i++) list.push(`r${i}@example.com`); + return list; + }; + const files = []; + for (let i = 0; i < 61; i++) { + files.push({ + filename: `f${i}.txt`, + contentType: 'text/plain', + inline: false, + contentId: '', + content: new Uint8Array(1), + }); + } + const refusals = [ + [message({ to: many(501) }), /at most 500 to recipients/], + [message({ cc: many(501) }), /at most 500 cc recipients/], + [message({ bcc: many(501) }), /at most 500 bcc recipients/], + [message({ attachments: files }), /at most 60 attachments/], + [message({ subject: 'x'.repeat(501) }), /subject of at most 500/], + [message({ tags: ['a', 'b'] }), /one tag per message/], + [message({ scheduledAt: new Date() }), /Invalid scheduledAt: zeptomail/], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('ZeptoMail: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => accepted(`zm-${index++}`)); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ tags: ['a', 'b'] }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('ZeptoMail: the options are checked, and the token never shows', async () => { + assert.throws(() => new ZeptoMailTransport({}), /needs token/); + assert.throws(() => transport({ bounceAddress: '' }), /bounceAddress/); + assert.throws(() => transport({ openTracking: 'yes' }), /openTracking must be a boolean/); + assert.throws(() => transport({ baseUrl: 'http://api.zeptomail.eu' }), /baseUrl/); + const made = transport({ baseUrl: 'https://api.zeptomail.eu' }); + assert.ok(!JSON.stringify(made).includes('zm-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('zm-secret')); + assert.strictEqual('verify' in made, false); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); From 05d47f230edb9e6349c33fb774bc64a53164ebd9 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:53:47 +0300 Subject: [PATCH 023/143] feat(smtp2go): a transport over the SMTP2GO API Ported from unemail's SMTP2GO driver onto the shared client of the HTTP providers. The region chooses the US, EU or AU host, the global one without it; the reply-to addresses and metadata go in header fields, as SMTP2GO has no field for them, and tags and tracking, which it has none for either, are not sent, as over SMTP. `scheduledAt` is `schedule`, written in the form SMTP2GO documents and held to its three days by an injected clock. A 200 whose `failed` count or `failures` say the send failed is a `rejected` receipt; the id is the `email_id`, which the events name. `verify` reads the summary of the account, a key without that permission verified too. Co-Authored-By: Claude Opus 5.5 --- docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/smtp2go.md | 86 +++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/smtp2go/config.js | 60 ++++++++ src/smtp2go/http-client.js | 46 ++++++ src/smtp2go/index.d.ts | 79 ++++++++++ src/smtp2go/index.js | 7 + src/smtp2go/message-converter.js | 106 ++++++++++++++ src/smtp2go/smtp2go-transport.js | 134 +++++++++++++++++ tests/types/smtp2go.test-d.ts | 24 +++ tests/unit/smtp2go.test.js | 242 +++++++++++++++++++++++++++++++ 13 files changed, 791 insertions(+) create mode 100644 docs/transports/smtp2go.md create mode 100644 src/smtp2go/config.js create mode 100644 src/smtp2go/http-client.js create mode 100644 src/smtp2go/index.d.ts create mode 100644 src/smtp2go/index.js create mode 100644 src/smtp2go/message-converter.js create mode 100644 src/smtp2go/smtp2go-transport.js create mode 100644 tests/types/smtp2go.test-d.ts create mode 100644 tests/unit/smtp2go.test.js diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 77deeff..b314139 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -90,6 +90,7 @@ const guide = [ { text: 'MailerSend', link: '/transports/mailersend' }, { text: 'SparkPost', link: '/transports/sparkpost' }, { text: 'ZeptoMail', link: '/transports/zeptomail' }, + { text: 'SMTP2GO', link: '/transports/smtp2go' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index ae30eda..38a8b4b 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -59,6 +59,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/mailersend` | `MailerSendTransport` | [MailerSend](/transports/mailersend) | | `@alexify/mailer/sparkpost` | `SparkPostTransport` | [SparkPost](/transports/sparkpost) | | `@alexify/mailer/zeptomail` | `ZeptoMailTransport` | [ZeptoMail](/transports/zeptomail) | +| `@alexify/mailer/smtp2go` | `Smtp2goTransport` | [SMTP2GO](/transports/smtp2go) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/smtp2go.md b/docs/transports/smtp2go.md new file mode 100644 index 0000000..2472f3d --- /dev/null +++ b/docs/transports/smtp2go.md @@ -0,0 +1,86 @@ +# SMTP2GO transport + +The SMTP2GO transport sends through the email API of [SMTP2GO], from the +SMTP2GO driver of unemail. + +[SMTP2GO]: https://www.smtp2go.com/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { Smtp2goTransport } = require('@alexify/mailer/smtp2go'); + +const transport = new Smtp2goTransport({ + apiKey: process.env.SMTP2GO_API_KEY, + region: 'eu', +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

', text: 'On its way.' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); // the email_id +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `apiKey` | (required) | An API key with the permission to send. | +| `region` | | `'us'`, `'eu'` or `'au'`; without one, `api.smtp2go.com` routes to the region nearest the resolver. | +| `baseUrl` | the region's | HTTPS, or HTTP to this machine, in place of the region's host. | +| `fastAccept` | | SMTP2GO accepts the message at once and sends it in the background. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before SMTP2GO took it. | +| `headers` | | Header fields added to every request. | +| `now` | `Date.now` | The clock for the 3-day window of `scheduledAt`. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- `scheduledAt` is `schedule`, at most 3 days ahead. +- SMTP2GO has no field for the reply-to addresses or for metadata: Reply-To + goes in the header fields, as SMTP2GO documents, and `metadata` in + `X-Metadata-` fields, with custom fields, List-* and the threading + fields. +- SMTP2GO has no tags, and tracking is set on the API key, so `tags` and + `tracking` are not sent, as over SMTP. +- An inline attachment goes in `inlines`, named by its Content-ID, which the + HTML refers to as `cid:`. +- SMTP2GO takes 100 addresses in each of `to`, `cc` and `bcc`; more are refused + before the request. + +SMTP2GO answers 200 even when the send failed: the outcome is in the `failed` +count and the `failures` of its answer, which the receipt reads. A failed send +is `rejected`, not retryable, with SMTP2GO's messages. With `fastAccept` the +answer says nothing of a failure, which only the events tell. An error of the +request comes with a 400 whose code names it; one about the key (such as +`API_KEY_NOT_FOUND` or `ENDPOINT_PERMISSION_DENIED`) is `auth`. + +The receipt's id is the `email_id` of the answer, which the events name, or the +`schedule_id` of a scheduled message without one. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. + +## Verifying the configuration + +`verify()` reads the summary of the account (`POST /v3/stats/email_summary`); a +key without that permission, which SMTP2GO answers with +`ENDPOINT_PERMISSION_DENIED`, is verified too. diff --git a/package.json b/package.json index b22b625..8c7bc8c 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,10 @@ "types": "./src/zeptomail/index.d.ts", "default": "./src/zeptomail/index.js" }, + "./smtp2go": { + "types": "./src/smtp2go/index.d.ts", + "default": "./src/smtp2go/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 07761b1..d44e7c2 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -19,6 +19,7 @@ "./mailersend": 7, "./sparkpost": 8, "./zeptomail": 7, + "./smtp2go": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/smtp2go/config.js b/src/smtp2go/config.js new file mode 100644 index 0000000..1828564 --- /dev/null +++ b/src/smtp2go/config.js @@ -0,0 +1,60 @@ +// Ported from unemail v0.7.0 (src/drivers/smtp2go.ts): the options of the +// SMTP2GO driver, the key and the region. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { + checkConcurrency, + checkOptionalBoolean, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); + +// Without a region, the global host routes to the region nearest the +// resolver, which need not hold the account's data. +const HOSTS = { + us: 'https://us-api.smtp2go.com', + eu: 'https://eu-api.smtp2go.com', + au: 'https://au-api.smtp2go.com', +}; +const GLOBAL_HOST = 'https://api.smtp2go.com'; + +const hostOf = (region) => { + if (region === undefined) return GLOBAL_HOST; + if (!Object.hasOwn(HOSTS, region)) { + throw new TypeError(`region must be 'us', 'eu' or 'au': ${String(region)}`); + } + return HOSTS[region]; +}; + +/** + * Resolves the options of the SMTP2GO transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createSmtp2goConfig = (config = {}) => { + const host = hostOf(config.region); + const resolved = { + apiKey: checkCredential(config.apiKey, 'apiKey', 'Smtp2goTransport'), + region: config.region, + baseUrl: config.baseUrl ?? host, + fastAccept: checkOptionalBoolean(config.fastAccept, 'fastAccept'), + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + now: config.now ?? Date.now, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createSmtp2goConfig }; diff --git a/src/smtp2go/http-client.js b/src/smtp2go/http-client.js new file mode 100644 index 0000000..93b7a5d --- /dev/null +++ b/src/smtp2go/http-client.js @@ -0,0 +1,46 @@ +// Ported from unemail v0.7.0 (src/drivers/smtp2go.ts): the requests of the +// SMTP2GO API, whose error is under `data` and a refused key a 400. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers. + +const { createApiClient } = require('../internal/api.js'); + +// The codes that say the key, not the message, is the problem. +const AUTH_ERROR_CODE = /PERMISSION|AUTH|API_KEY|UNAUTHORIZED|FORBIDDEN/i; + +const messageOf = (json) => (typeof json?.data?.error === 'string' ? json.data.error : undefined); + +const classify = (_status, json) => { + const code = json?.data?.error_code; + if (typeof code !== 'string' || code === '') return undefined; + const name = code.startsWith('E_ApiResponseCodes.') ? code.slice(19) : code; + const classified = AUTH_ERROR_CODE.test(name) ? { category: 'auth', retryable: false } : {}; + return { ...classified, code: `smtp2go.${name}` }; +}; + +/** + * The client of the SMTP2GO API. + * + * @internal + */ +const createSmtp2goClient = (config) => { + const client = createApiClient(config, { + id: 'smtp2go', + label: 'SMTP2GO', + authorize: (headers) => headers.set('X-Smtp2go-Api-Key', config.apiKey), + messageOf, + classify, + }); + return { + ...client, + send: (payload, options) => + client.request({ + url: `${config.baseUrl}/v3/email/send`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createSmtp2goClient }; diff --git a/src/smtp2go/index.d.ts b/src/smtp2go/index.d.ts new file mode 100644 index 0000000..bd0159a --- /dev/null +++ b/src/smtp2go/index.d.ts @@ -0,0 +1,79 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link Smtp2goTransport}. + */ +export interface Smtp2goConfig extends TransportDependencies { + /** An API key with the permission to send. */ + readonly apiKey: string; + /** + * The region of the account, whose host the requests go to; without one, + * `api.smtp2go.com` routes them to the region nearest the resolver. + */ + readonly region?: 'us' | 'eu' | 'au'; + /** + * The base URL of the API, https (or http to this machine), in place of the + * host of the region. + */ + readonly baseUrl?: string; + /** + * Has SMTP2GO accept a message at once and send it in the background; the + * answer then says nothing of a failure, which only the events tell. + */ + readonly fastAccept?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before SMTP2GO took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; + /** The clock, in milliseconds, for the 3 days of `scheduledAt`. */ + readonly now?: () => number; +} + +/** + * A transport over the email API of SMTP2GO. `scheduledAt` (at most 3 days + * ahead) is `schedule`; the reply-to addresses and `metadata` + * (`X-Metadata-`) go in header fields; SMTP2GO has no tags, and tracking + * is set on the API key, so neither is sent. + * + * @example + * ```js + * const { Smtp2goTransport } = require('@alexify/mailer/smtp2go'); + * const transport = new Smtp2goTransport({ + * apiKey: process.env.SMTP2GO_API_KEY, + * region: 'eu', + * }); + * ``` + */ +export declare class Smtp2goTransport implements VerifiableTransport<'smtp2go'> { + readonly id = 'smtp2go'; + constructor(config: Smtp2goConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the API key by reading the summary of the account. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/smtp2go/index.js b/src/smtp2go/index.js new file mode 100644 index 0000000..85faecf --- /dev/null +++ b/src/smtp2go/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/smtp2go.ts): the entry of the +// SMTP2GO transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { Smtp2goTransport } = require('./smtp2go-transport.js'); + +module.exports = { Smtp2goTransport }; diff --git a/src/smtp2go/message-converter.js b/src/smtp2go/message-converter.js new file mode 100644 index 0000000..b417dbb --- /dev/null +++ b/src/smtp2go/message-converter.js @@ -0,0 +1,106 @@ +// Ported from unemail v0.7.0 (src/drivers/smtp2go.ts): a message as the +// request of SMTP2GO's `/v3/email/send`, `toPayload` and `checkLimits`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: `schedule` is written as SMTP2GO documents it, with +// the window of three days read from an injected clock; and tags, which +// SMTP2GO has no field for, are not sent, as over SMTP. + +const { formatMailbox } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { formatAddresses } = require('../internal/provider.js'); + +// "An array of names and email addresses (up to 100)", in each field. +const MAX_RECIPIENTS = 100; +// `schedule` "must be in the future and within the next 3 days". +const SCHEDULE_WINDOW_MS = 3 * 24 * 60 * 60 * 1000; + +const checkLimits = (message, config) => { + for (const [field, name] of [ + ['recipients', 'to'], + ['ccRecipients', 'cc'], + ['bccRecipients', 'bcc'], + ]) { + if (message[field].length > MAX_RECIPIENTS) { + throw new RangeError(`SMTP2GO takes at most ${MAX_RECIPIENTS} ${name} recipients.`); + } + } + if ( + message.scheduledAt !== undefined && + message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS + ) { + throw new RangeError('SMTP2GO schedules a message at most 3 days ahead.'); + } +}; + +// "2025-09-10 13:15:00 +1200", the form of SMTP2GO's example. +const scheduleOf = (date) => { + const iso = date.toISOString(); + return `${iso.slice(0, 10)} ${iso.slice(11, 19)} +0000`; +}; + +const headersOf = (message) => { + const headers = []; + // The fields SMTP2GO forbids, Content-Type, Content-Transfer-Encoding and + // MIME-Version, are those of the MIME structure, never among these. + for (const field of headerFieldsOf(message)) { + headers.push({ header: field.name, value: field.value }); + } + // SMTP2GO has no reply-to or metadata field: a header field is the + // documented way for the first, and reaches the events for the second. + if (message.replyRecipients.length > 0) { + const value = formatAddresses(message.replyRecipients, formatMailbox).join(', '); + headers.push({ header: 'Reply-To', value }); + } + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers.push({ header: `X-Metadata-${key}`, value: message.metadata[key] }); + } + } + return headers; +}; + +/** + * Converts a message to a request of `/v3/email/send`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message is over a limit of SMTP2GO. + */ +const convertMessage = async (message, config, signal) => { + checkLimits(message, config); + const payload = { + sender: formatMailbox(message.sender), + to: formatAddresses(message.recipients, formatMailbox), + subject: message.subject, + }; + if (message.ccRecipients.length > 0) { + payload.cc = formatAddresses(message.ccRecipients, formatMailbox); + } + if (message.bccRecipients.length > 0) { + payload.bcc = formatAddresses(message.bccRecipients, formatMailbox); + } + if (message.content.text) payload.text_body = message.content.text; + if ('html' in message.content) payload.html_body = message.content.html; + const headers = headersOf(message); + if (headers.length > 0) payload.custom_headers = headers; + for (const item of await base64Attachments(message, signal)) { + const file = { fileblob: item.content, mimetype: item.attachment.contentType }; + if (item.attachment.inline) { + // An inline image is named in the HTML as `cid:`. + file.filename = item.attachment.contentId; + payload.inlines ??= []; + payload.inlines.push(file); + } else { + file.filename = item.attachment.filename; + payload.attachments ??= []; + payload.attachments.push(file); + } + } + if (message.scheduledAt !== undefined) payload.schedule = scheduleOf(message.scheduledAt); + if (config.fastAccept === true) payload.fastaccept = true; + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/smtp2go/smtp2go-transport.js b/src/smtp2go/smtp2go-transport.js new file mode 100644 index 0000000..db4fb4a --- /dev/null +++ b/src/smtp2go/smtp2go-transport.js @@ -0,0 +1,134 @@ +// Ported from unemail v0.7.0 (src/drivers/smtp2go.ts): a transport over the +// SMTP2GO API, which answers 200 to a send that failed. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the id of a receipt is the `email_id`, which the +// events name, before the `schedule_id`; and `verify` reads the summary of +// the account. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createSmtp2goConfig } = require('./config.js'); +const { createSmtp2goClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +const describeFailure = (failure) => { + if (typeof failure === 'string') return failure; + return JSON.stringify(failure); +}; + +// "this endpoint returns 200 OK even if there are errors in the response": +// the outcome is in `failed` and `failures`, which count messages, not +// recipients. +const receiptOf = (json) => { + const data = json?.data ?? {}; + const failures = Array.isArray(data.failures) ? data.failures : []; + if (data.failed > 0 || failures.length > 0) { + let reason = ''; + for (const failure of failures) { + reason += reason === '' ? describeFailure(failure) : `; ${describeFailure(failure)}`; + } + return createFailedReceipt(`SMTP2GO: ${reason === '' ? 'the message failed.' : reason}`, { + provider: 'smtp2go', + category: 'rejected', + retryable: false, + code: 'smtp2go.failed', + providerDetails: data, + }); + } + const messageId = data.email_id || data.schedule_id; + if (typeof messageId !== 'string' || messageId === '') { + return createFailedReceipt('SMTP2GO answered without an email_id.', { + provider: 'smtp2go', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'smtp2go' }; +}; + +/** + * A transport over the email API of SMTP2GO. + */ +class Smtp2goTransport { + id = 'smtp2go'; + + constructor(config) { + this.config = createSmtp2goConfig(config); + this.httpClient = createSmtp2goClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/v3/email/send`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the `email_id` of the answer. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + return receiptOf((await this.httpClient.send(payload, options)).json); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('smtp2go', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the API key by reading the summary of the account + * (`POST /v3/stats/email_summary`); a key without that permission is + * verified too. + * + * @param options Optional abort signal. + * @throws {Error} If SMTP2GO refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + await verifyByRequest({ + provider: 'SMTP2GO', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.config.baseUrl}/v3/stats/email_summary`, + { + method: 'POST', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: '{}', + }, + signal, + ), + // SMTP2GO answers a key that may not read the summary with a 400. + check: (status, text) => status === 400 && text.includes('ENDPOINT_PERMISSION_DENIED'), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { Smtp2goTransport }; diff --git a/tests/types/smtp2go.test-d.ts b/tests/types/smtp2go.test-d.ts new file mode 100644 index 0000000..acc93dc --- /dev/null +++ b/tests/types/smtp2go.test-d.ts @@ -0,0 +1,24 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { Smtp2goTransport } from '../../src/smtp2go/index.js'; + +const transport = new Smtp2goTransport({ + apiKey: 'api-key', + region: 'au', + fastAccept: true, + concurrency: 2, + now: () => Date.now(), +}); +expectType<'smtp2go'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new Smtp2goTransport({})); +expectError(new Smtp2goTransport({ apiKey: 'k', region: 'asia' })); diff --git a/tests/unit/smtp2go.test.js b/tests/unit/smtp2go.test.js new file mode 100644 index 0000000..df5a81a --- /dev/null +++ b/tests/unit/smtp2go.test.js @@ -0,0 +1,242 @@ +// Not ported: tests of ./smtp2go, ported from unemail's SMTP2GO driver, +// against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { Smtp2goTransport } = require('../../src/smtp2go/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const NOW = Date.parse('2026-10-17T12:00:00Z'); +const transport = (config = {}) => + new Smtp2goTransport({ apiKey: 'api-secret', retries: 0, now: () => NOW, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const sent = (data = { email_id: 'em-1' }) => + json({ request_id: 'req-1', data: { succeeded: 1, failed: 0, failures: [], ...data } }); + +test('SMTP2GO: a message is the request of /v3/email/send, with the key', async (t) => { + const requests = stubFetch(t, () => sent({ email_id: 'em-1', schedule_id: 'sc-1' })); + const receipt = await transport({ region: 'eu', fastAccept: true }).send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: ['support@example.com', { name: 'Desk', address: 'desk@example.com' }], + priority: 'high', + tags: ['orders'], + metadata: { orderId: '42' }, + tracking: { opens: true }, + list: { unsubscribe: 'https://example.com/u' }, + scheduledAt: new Date(NOW + 3600_000), + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'em-1', provider: 'smtp2go' }); + const [request] = requests; + assert.strictEqual(request.url, 'https://eu-api.smtp2go.com/v3/email/send'); + assert.strictEqual(request.headers.get('x-smtp2go-api-key'), 'api-secret'); + assert.strictEqual(request.redirect, 'manual'); + assert.deepStrictEqual(request.json, { + sender: 'Shop ', + to: ['bob@example.com', 'Carol '], + cc: ['dave@example.com'], + bcc: ['erin@example.com'], + subject: 'Your order', + text_body: 'Shipped', + html_body: '

Shipped

', + custom_headers: [ + { header: 'X-Priority', value: '1' }, + { header: 'X-MSMail-Priority', value: 'High' }, + { header: 'Importance', value: 'High' }, + { header: 'List-Unsubscribe', value: '' }, + { header: 'Reply-To', value: 'support@example.com, Desk ' }, + { header: 'X-Metadata-orderId', value: '42' }, + ], + attachments: [{ fileblob: 'aGk=', mimetype: 'text/plain', filename: 'r.txt' }], + inlines: [{ fileblob: 'AQ==', mimetype: 'image/png', filename: 'logo@example.com' }], + schedule: '2026-10-17 13:00:00 +0000', + fastaccept: true, + }); +}); + +test('SMTP2GO: the region chooses the host, the global one without it', async (t) => { + const requests = stubFetch(t, () => sent()); + await transport().send(message({ content: { text: 'plain' } })); + await transport({ region: 'au' }).send(message()); + await transport({ region: 'us', baseUrl: 'http://localhost:9000' }).send(message()); + assert.deepStrictEqual( + requests.map((request) => request.url), + [ + 'https://api.smtp2go.com/v3/email/send', + 'https://au-api.smtp2go.com/v3/email/send', + 'http://localhost:9000/v3/email/send', + ], + ); + const body = requests[0].json; + for (const key of ['cc', 'bcc', 'html_body', 'custom_headers', 'schedule', 'fastaccept']) { + assert.strictEqual(body[key], undefined, key); + } +}); + +test('SMTP2GO: a 200 with failures fails the receipt, one without an id is unknown', async (t) => { + const answers = [ + sent({ succeeded: 0, failed: 1, failures: ['Sender domain not verified'] }), + sent({ succeeded: 0, failed: 1, failures: [{ reason: 'blocked' }] }), + sent({ succeeded: 0, failed: 1, failures: [] }), + sent({ schedule_id: 'sc-2', email_id: undefined }), + json({ request_id: 'req-2', data: {} }), + ]; + stubFetch(t, () => answers.shift()); + const failed = await transport().send(message()); + assert.strictEqual(failed.successful, false); + assert.strictEqual(failed.errorMessages[0], 'SMTP2GO: Sender domain not verified'); + assert.strictEqual(failed.errors[0].category, 'rejected'); + assert.strictEqual(failed.errors[0].code, 'smtp2go.failed'); + assert.strictEqual(failed.retryable, false); + assert.strictEqual( + (await transport().send(message())).errorMessages[0], + 'SMTP2GO: {"reason":"blocked"}', + ); + assert.strictEqual( + (await transport().send(message())).errorMessages[0], + 'SMTP2GO: the message failed.', + ); + assert.strictEqual((await transport().send(message())).messageId, 'sc-2'); + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); +}); + +test('SMTP2GO: an error under data says what failed, a refused key is auth', async (t) => { + const answers = [ + json( + { + request_id: 'r', + data: { + error_code: 'E_ApiResponseCodes.API_KEY_NOT_FOUND', + error: 'The API key was not found', + }, + }, + 400, + ), + json( + { + request_id: 'r', + data: { + error_code: 'E_ApiResponseCodes.NON_VALIDATING_IN_PAYLOAD', + error: 'An email address is invalid', + }, + }, + 400, + ), + json({ request_id: 'r', data: {} }, 500), + ]; + stubFetch(t, () => answers.shift()); + const auth = await transport().send(message()); + assert.strictEqual(auth.errorMessages[0], 'SMTP2GO: The API key was not found'); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errors[0].code, 'smtp2go.API_KEY_NOT_FOUND'); + const invalid = await transport().send(message()); + assert.strictEqual(invalid.errors[0].category, 'validation'); + assert.strictEqual(invalid.errors[0].code, 'smtp2go.NON_VALIDATING_IN_PAYLOAD'); + const server = await transport().send(message()); + assert.strictEqual(server.errors[0].category, 'server-error'); + assert.strictEqual(server.errors[0].code, 'http.500'); +}); + +test('SMTP2GO: its limits are refused before a request', async (t) => { + const requests = stubFetch(t, () => sent()); + const many = (count) => { + const list = []; + for (let i = 0; i < count; i++) list.push(`r${i}@example.com`); + return list; + }; + const refusals = [ + [message({ to: many(101) }), /at most 100 to recipients/], + [message({ cc: many(101) }), /at most 100 cc recipients/], + [message({ bcc: many(101) }), /at most 100 bcc recipients/], + [message({ scheduledAt: new Date(NOW + 73 * 3600_000) }), /3 days ahead/], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('SMTP2GO: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => sent({ email_id: `em-${index++}` })); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ scheduledAt: new Date(NOW + 73 * 3600_000) }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('SMTP2GO: verify reads the summary, a key that may not is verified too', async (t) => { + const answers = [ + json({ request_id: 'r', data: { cycle_used: 1 } }), + json( + { + request_id: 'r', + data: { + error_code: 'E_ApiResponseCodes.ENDPOINT_PERMISSION_DENIED', + error: 'You do not have permission to access this API endpoint', + }, + }, + 400, + ), + json({ request_id: 'r', data: { error_code: 'E_ApiResponseCodes.API_KEY_NOT_FOUND' } }, 400), + json({}, 401), + ]; + const requests = stubFetch(t, () => answers.shift()); + await transport({ region: 'eu' }).verify(); + assert.strictEqual(requests[0].url, 'https://eu-api.smtp2go.com/v3/stats/email_summary'); + assert.strictEqual(requests[0].method, 'POST'); + assert.deepStrictEqual(requests[0].json, {}); + assert.strictEqual(requests[0].headers.get('x-smtp2go-api-key'), 'api-secret'); + await transport().verify(); + await assert.rejects(transport().verify(), /SMTP2GO answered the check with HTTP 400/); + await assert.rejects(transport().verify(), /SMTP2GO refused the credentials/); +}); + +test('SMTP2GO: the options are checked, and the key never shows', async () => { + assert.throws(() => new Smtp2goTransport({}), /needs apiKey/); + assert.throws(() => transport({ region: 'asia' }), /region must be 'us', 'eu' or 'au'/); + assert.throws(() => transport({ fastAccept: 1 }), /fastAccept must be a boolean/); + assert.throws(() => transport({ baseUrl: 'http://api.example.com' }), /baseUrl/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('api-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('api-secret')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); From fe23fb1bf0e79544ec085b123bfd9e7c99c68249 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 21:59:46 +0300 Subject: [PATCH 024/143] feat(azure): a transport over Azure Communication Services Email Ported from unemail's Azure Communication Services driver onto the shared client of the HTTP providers. Each attempt is signed with the access key (HMAC-SHA256 on node:crypto, checked in the tests against Web Crypto) at the time it is sent. The credentials come only from the options, the endpoint is HTTPS or loopback HTTP, and every send carries its key as the Operation-Id and the Repeatability-Request-Id, with the time of the first attempt as Repeatability-First-Sent, so Azure answers a retry with the outcome of the first request; a repeat answered 409 or 412 is `deliveryUnknown` (U7). Metadata goes in `X-Metadata-` headers; tracking it cannot express, a schedule and a request over 10 MB are refused before a request. `verify` reads an operation that does not exist. The budget is 8 KB: the signature and the connection string add almost a kilobyte. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/azure.md | 97 ++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/azure/azure-transport.js | 141 ++++++++++++++ src/azure/config.js | 78 ++++++++ src/azure/http-client.js | 64 +++++++ src/azure/index.d.ts | 97 ++++++++++ src/azure/index.js | 7 + src/azure/message-converter.js | 106 +++++++++++ src/azure/sign.js | 82 ++++++++ tests/types/azure.test-d.ts | 32 ++++ tests/unit/azure.test.js | 329 +++++++++++++++++++++++++++++++++ 15 files changed, 1048 insertions(+) create mode 100644 docs/transports/azure.md create mode 100644 src/azure/azure-transport.js create mode 100644 src/azure/config.js create mode 100644 src/azure/http-client.js create mode 100644 src/azure/index.d.ts create mode 100644 src/azure/index.js create mode 100644 src/azure/message-converter.js create mode 100644 src/azure/sign.js create mode 100644 tests/types/azure.test-d.ts create mode 100644 tests/unit/azure.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 689c5a0..99f2e4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -532,6 +532,14 @@ fails on upstream's behavior): the shared To in `header_to` and the Cc field in `content.headers`, counts the rejected ones on the receipt (`rejectedRecipientCount`), fails it as `rejected` when none was accepted, and reports an answer without an id as `deliveryUnknown`. +- U7. unemail (`src/drivers/azure-communication.ts`): with no option the connection string came + from `COMMUNICATION_SERVICES_CONNECTION_STRING` of `process.env`, an endpoint could be `http:` + to any host, and its idempotency was the `Operation-Id`, while the API takes a repeat as one + only by the `Repeatability-Request-Id` and `Repeatability-First-Sent` of OASIS Repeatable + Requests, which it never sent; tracking with opens on and clicks off turned both on. `./azure` + reads only its options, holds the endpoint to HTTPS or loopback HTTP, sends both headers with + the key as `Operation-Id` too, reports a repeat Azure answers 409 or 412 as `deliveryUnknown`, + and refuses tracking it cannot express. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index b314139..9d6a6f9 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -91,6 +91,7 @@ const guide = [ { text: 'SparkPost', link: '/transports/sparkpost' }, { text: 'ZeptoMail', link: '/transports/zeptomail' }, { text: 'SMTP2GO', link: '/transports/smtp2go' }, + { text: 'Azure Communication Services', link: '/transports/azure' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 38a8b4b..5c1efda 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -60,6 +60,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/sparkpost` | `SparkPostTransport` | [SparkPost](/transports/sparkpost) | | `@alexify/mailer/zeptomail` | `ZeptoMailTransport` | [ZeptoMail](/transports/zeptomail) | | `@alexify/mailer/smtp2go` | `Smtp2goTransport` | [SMTP2GO](/transports/smtp2go) | +| `@alexify/mailer/azure` | `AzureTransport` | [Azure Communication Services](/transports/azure) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/azure.md b/docs/transports/azure.md new file mode 100644 index 0000000..cf568a1 --- /dev/null +++ b/docs/transports/azure.md @@ -0,0 +1,97 @@ +# Azure Communication Services transport + +The Azure transport sends through the Email API of [Azure Communication +Services], signed with the access key of the resource (HMAC-SHA256) on +`node:crypto`, with no Azure SDK. It comes from the Azure Communication +Services driver of unemail. + +[Azure Communication Services]: https://learn.microsoft.com/en-us/azure/communication-services/concepts/email/email-overview + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { AzureTransport } = require('@alexify/mailer/azure'); + +const transport = new AzureTransport({ + connectionString: process.env.ACS_CONNECTION_STRING, +}); + +const receipt = await transport.send( + createMessage({ + from: 'DoNotReply@example.com', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + idempotencyKey: 'order-42-shipped', + }), +); +if (receipt.successful) console.log(receipt.messageId); // the operation id +``` + +The connection string is the one the Keys of the resource show, +`endpoint=https://….communication.azure.com/;accesskey=…`. It is never read from +the environment: pass it, or `endpoint` and `accessKey`, as options. + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `connectionString` | | The connection string of the resource. | +| `endpoint`, `accessKey` | | Its endpoint (HTTPS, or HTTP to this machine) and base64 access key, in place of the connection string. | +| `apiVersion` | `'2025-09-01'` | The version of the Email API. | +| `userEngagementTrackingDisabled` | | Turns tracking off for every message, unless a message sets its own `tracking`. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request; see below. | +| `headers` | | Header fields added to every request. | +| `now` | `Date.now` | The clock of the signatures and of the repeats. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- The sender goes as its address alone: the API has no field for a display + name, which is that of the sender username in the resource. +- `metadata` goes in `X-Metadata-` headers, with custom header fields, + List-* and the threading fields. +- Azure has one switch for open and click tracking, which can turn off what the + resource has on but not the reverse. `tracking` sets it; a message that asks + for opens without clicks, or the reverse, is refused. +- Azure has no tags, which are not sent, as over SMTP; it cannot schedule a + message, so `scheduledAt` is refused as `validation`. +- An inline attachment goes with its Content-ID as `contentId`. +- A request may be at most 10 MB, its attachments in base64 included; a larger + one is refused before it is sent. + +## Retries and repeats + +A send is a long-running operation, which Azure accepts with a 202 and the id of +the operation, the receipt's id. Every send carries a key: the message's +`idempotencyKey` as a UUID, or one made for that send. It is the +`Operation-Id` and the `Repeatability-Request-Id` of the request, with the time +of the first attempt as `Repeatability-First-Sent`. Azure answers a repeat it +has tracked, within five minutes, with the outcome of the first request rather +than sending again, so a retry after a timeout or a reset is safe; a repeat +answered 409 (the first still running) or 412 (after the five minutes) is +reported as `deliveryUnknown`, never sent again. + +The same `idempotencyKey` sent again within five minutes is a repeat too, and +gets the outcome of the first. Past that window Azure no longer knows it. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. + +## Verifying the configuration + +`verify()` reads an operation that does not exist, signed with the key: Azure +answers 404 when it accepts the signature and 401 when it does not. diff --git a/package.json b/package.json index 8c7bc8c..e978305 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,10 @@ "types": "./src/smtp2go/index.d.ts", "default": "./src/smtp2go/index.js" }, + "./azure": { + "types": "./src/azure/index.d.ts", + "default": "./src/azure/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d44e7c2..3109f10 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -20,6 +20,7 @@ "./sparkpost": 8, "./zeptomail": 7, "./smtp2go": 7, + "./azure": 8, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/azure/azure-transport.js b/src/azure/azure-transport.js new file mode 100644 index 0000000..e261e3d --- /dev/null +++ b/src/azure/azure-transport.js @@ -0,0 +1,141 @@ +// Ported from unemail v0.7.0 (src/drivers/azure-communication.ts): a transport +// over the Email API of Azure Communication Services, whose send is a +// long-running operation it accepts with a 202. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a repeat Azure answers with 409 (the first request +// still running) or 412 (after the first, outside the five minutes it +// tracks) is a delivery of unknown outcome, not a failure to send again; and +// `verify` reads an operation that does not exist. + +const { generateId } = require('../core/id.js'); +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { toUuid } = require('../internal/uuid.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createAzureConfig } = require('./config.js'); +const { createAzureClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +// The id of no operation, which `verify` reads. +const NO_OPERATION = '00000000-0000-0000-0000-000000000000'; +const FAILED_STATES = new Set(['Failed', 'Canceled']); + +const failureOf = (error) => { + const repeated = error?.statusCode === 409 || (error?.statusCode === 412 && error.attempts > 1); + if (repeated) { + // Azure took this request before: it is running, or went then. + return createFailedReceipt(`${error.message} (a repeat of a request Azure took before)`, { + provider: 'azure', + deliveryUnknown: true, + statusCode: error.statusCode, + attempts: error.attempts, + code: error.receiptCode, + }); + } + return providerFailure('azure', error); +}; + +const receiptOf = (answer, requestId) => { + const json = answer.json ?? {}; + if (FAILED_STATES.has(json.status)) { + const detail = json.error?.message ?? json.status; + return createFailedReceipt(`Azure Communication Services: ${detail}`, { + provider: 'azure', + category: 'rejected', + retryable: false, + code: typeof json.error?.code === 'string' ? `azure.${json.error.code}` : undefined, + providerDetails: json, + }); + } + const messageId = typeof json.id === 'string' && json.id !== '' ? json.id : requestId; + return { successful: true, messageId, provider: 'azure' }; +}; + +/** + * A transport over the Email API of Azure Communication Services. + */ +class AzureTransport { + id = 'azure'; + + constructor(config) { + this.config = createAzureConfig(config); + this.httpClient = createAzureClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `emails:send`. Its `idempotencyKey`, or one made + * for this send, is the id of the operation and of its repeats, so a + * retry within five minutes is answered with the outcome of the first. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the id of the operation. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const requestId = toUuid( + message.idempotencyKey ?? generateId('idempotency-key', this.config.generateId), + ); + const firstSent = new Date(this.config.now()); + const answer = await this.httpClient.send(payload, requestId, firstSent, options); + return receiptOf(answer, requestId); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return failureOf(error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the access key by reading an operation that does not exist: a + * request Azure authenticates is answered 404, one it does not 401. + * + * @param options Optional abort signal. + * @throws {Error} If Azure refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const { endpoint, apiVersion } = this.config; + await verifyByRequest({ + provider: 'Azure Communication Services', + send: (signal) => + this.httpClient.fetchWithAuth( + `${endpoint}/emails/operations/${NO_OPERATION}?api-version=${apiVersion}`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + check: (status) => status === 404, + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { AzureTransport }; diff --git a/src/azure/config.js b/src/azure/config.js new file mode 100644 index 0000000..c1262c1 --- /dev/null +++ b/src/azure/config.js @@ -0,0 +1,78 @@ +// Ported from unemail v0.7.0 (src/drivers/azure-communication.ts): the options +// of the Azure Communication Services driver, `resolveCredentials`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the connection string is never read from +// `process.env`, and the endpoint must be HTTPS (or HTTP to this machine), +// where unemail took any `http:` one. + +const { + checkConcurrency, + checkOptionalBoolean, + checkRequestOptions, + checkSecureUrl, +} = require('../internal/config.js'); +const { isAccessKey, parseConnectionString } = require('./sign.js'); + +const API_VERSION = /^\d{4}-\d{2}-\d{2}(-preview)?$/; + +const credentialsOf = (config) => { + if (config.connectionString !== undefined) { + const parsed = + typeof config.connectionString === 'string' + ? parseConnectionString(config.connectionString) + : undefined; + if (parsed === undefined) { + throw new TypeError( + 'connectionString must be `endpoint=https://…;accesskey=…`, as the Keys of the ' + + 'resource show it.', + ); + } + return parsed; + } + if (typeof config.endpoint !== 'string' || config.endpoint === '') { + throw new TypeError('AzureTransport needs connectionString, or endpoint and accessKey.'); + } + return { endpoint: config.endpoint, accessKey: config.accessKey }; +}; + +/** + * Resolves the options of the Azure transport. + * + * @param config The options. + * @returns The options with their defaults, the credentials split. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createAzureConfig = (config = {}) => { + const credentials = credentialsOf(config); + if (!isAccessKey(credentials.accessKey)) { + throw new TypeError('accessKey must be the base64 key of the resource, as its Keys show it.'); + } + checkSecureUrl(credentials.endpoint, 'endpoint'); + const apiVersion = config.apiVersion ?? '2025-09-01'; + if (typeof apiVersion !== 'string' || !API_VERSION.test(apiVersion)) { + throw new TypeError(`apiVersion must be a date such as 2025-09-01: ${String(apiVersion)}`); + } + const resolved = { + endpoint: credentials.endpoint.replace(/\/+$/, ''), + accessKey: credentials.accessKey, + apiVersion, + userEngagementTrackingDisabled: checkOptionalBoolean( + config.userEngagementTrackingDisabled, + 'userEngagementTrackingDisabled', + ), + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + now: config.now ?? Date.now, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + return resolved; +}; + +module.exports = { createAzureConfig }; diff --git a/src/azure/http-client.js b/src/azure/http-client.js new file mode 100644 index 0000000..a03356e --- /dev/null +++ b/src/azure/http-client.js @@ -0,0 +1,64 @@ +// Ported from unemail v0.7.0 (src/drivers/azure-communication.ts): the signed +// requests of the Email API of Azure Communication Services, and the category +// of its error codes. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers; and a +// send carries the Repeatability-Request-Id and Repeatability-First-Sent of +// OASIS Repeatable Requests, by which Azure answers a repeat with the outcome +// of the first request rather than sending again. + +const { createApiClient } = require('../internal/api.js'); +const { signRequest } = require('./sign.js'); + +// Azure's codes that mean the key may not do this, whatever the status. +const AUTH_CODES = /Unauthorized|Unauthenticated|Forbidden|DenyAction|AuthenticationFailed/i; +const THROTTLE_CODES = /TooManyRequests|Throttl|QuotaExceeded/i; + +const classify = (_status, json) => { + const code = json?.error?.code; + if (typeof code !== 'string' || code === '') return undefined; + let classified = {}; + if (AUTH_CODES.test(code)) classified = { category: 'auth', retryable: false }; + else if (THROTTLE_CODES.test(code)) classified = { category: 'rate-limit', retryable: true }; + return { ...classified, code: `azure.${code}` }; +}; + +/** + * The client of the Email API of Azure Communication Services. + * + * @internal + */ +const createAzureClient = (config) => { + const client = createApiClient(config, { + id: 'azure', + label: 'Azure Communication Services', + // Each attempt is signed anew, with the time it is sent. + authorize: (headers, request) => { + const signed = signRequest({ ...request, date: new Date(config.now()) }, config.accessKey); + for (const name in signed) headers.set(name, signed[name]); + }, + classify, + }); + return { + ...client, + /** + * Sends a message. `requestId` is a UUID, the operation's id and the key + * of its repeats; `firstSent`, the time of the first attempt. + */ + send: (payload, requestId, firstSent, options) => + client.request({ + url: `${config.endpoint}/emails:send?api-version=${config.apiVersion}`, + body: payload, + headers: { + 'Operation-Id': requestId, + 'Repeatability-Request-Id': requestId, + 'Repeatability-First-Sent': firstSent.toUTCString(), + }, + hasIdempotencyKey: true, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createAzureClient }; diff --git a/src/azure/index.d.ts b/src/azure/index.d.ts new file mode 100644 index 0000000..8896595 --- /dev/null +++ b/src/azure/index.d.ts @@ -0,0 +1,97 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +interface AzureOptions extends TransportDependencies { + /** + * The version of the Email API. + * @default "2025-09-01" + */ + readonly apiVersion?: string; + /** + * Turns open and click tracking off for every message, unless a message + * sets its own `tracking`; it cannot turn on what the resource has off. + */ + readonly userEngagementTrackingDisabled?: boolean; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request; a repeat within five minutes is answered with the + * outcome of the first. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; + /** The clock, in milliseconds, of the signatures and of the repeats. */ + readonly now?: () => number; +} + +/** + * The options of {@link AzureTransport}: the connection string of the + * resource, or its endpoint and access key. + */ +export type AzureConfig = AzureOptions & + ( + | { + /** + * `endpoint=https://….communication.azure.com/;accesskey=…`, as the + * Keys of the resource show it. + */ + readonly connectionString: string; + readonly endpoint?: never; + readonly accessKey?: never; + } + | { + readonly connectionString?: never; + /** + * The endpoint of the resource, HTTPS (or HTTP to this machine), such + * as `https://my-resource.communication.azure.com`. + */ + readonly endpoint: string; + /** The base64 access key of the resource. */ + readonly accessKey: string; + } + ); + +/** + * A transport over the Email API of Azure Communication Services, signed + * with the access key (HMAC-SHA256). The sender goes as its address alone, + * its display name that of the sender in the resource; `metadata` goes in + * `X-Metadata-` headers; `tracking` turns both kinds on or off; Azure + * has no tags and cannot schedule a message. + * + * @example + * ```js + * const { AzureTransport } = require('@alexify/mailer/azure'); + * const transport = new AzureTransport({ + * connectionString: process.env.ACS_CONNECTION_STRING, + * }); + * ``` + */ +export declare class AzureTransport implements VerifiableTransport<'azure'> { + readonly id = 'azure'; + constructor(config: AzureConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the access key with a signed read of the API. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/azure/index.js b/src/azure/index.js new file mode 100644 index 0000000..1d9585f --- /dev/null +++ b/src/azure/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/azure-communication.ts): the entry +// of the Azure Communication Services transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { AzureTransport } = require('./azure-transport.js'); + +module.exports = { AzureTransport }; diff --git a/src/azure/message-converter.js b/src/azure/message-converter.js new file mode 100644 index 0000000..cd51a2b --- /dev/null +++ b/src/azure/message-converter.js @@ -0,0 +1,106 @@ +// Ported from unemail v0.7.0 (src/drivers/azure-communication.ts): a message +// as the request of `emails:send`, `toPayload` and `trackingDisabled`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a message that asks for opens without clicks, or the +// reverse, is refused, as Azure has one switch for both and unemail turned +// both on; tags, which Azure has no field for, are not sent, as over SMTP; +// and a scheduled message is refused. + +const { toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +/** Azure takes a request of at most 10 MB, its attachments included. */ +const MAX_REQUEST_BYTES = 10 * 1024 * 1024; + +const addressOf = (address) => { + const result = { address: toAsciiAddress(address.address) }; + if (address.name) result.displayName = address.name; + return result; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +// `userEngagementTrackingDisabled` turns opens and clicks off together, and +// can only turn off what the resource has on. +const trackingDisabledOf = (message, config) => { + const tracking = message.tracking; + if (tracking === undefined) return config.userEngagementTrackingDisabled; + const { opens, clicks } = tracking; + if (opens !== undefined && clicks !== undefined && opens !== clicks) { + throw new RangeError('Azure turns open and click tracking on or off together.'); + } + const wanted = opens ?? clicks; + return wanted === undefined ? config.userEngagementTrackingDisabled : !wanted; +}; + +const headersOf = (message) => { + const headers = {}; + let hasHeaders = false; + for (const field of headerFieldsOf(message)) { + headers[field.name] = field.value; + hasHeaders = true; + } + // Azure has no metadata of its own: a header field carries it. + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers[`X-Metadata-${key}`] = message.metadata[key]; + hasHeaders = true; + } + } + return hasHeaders ? headers : undefined; +}; + +/** + * Converts a message to a request of `emails:send`. The sender goes as its + * address alone: its display name is that of the sender in the resource. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message asks for one kind of tracking without + * the other, or the request would be over 10 MB. + * @throws {TypeError} If it is scheduled. + */ +const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'azure'); + const recipients = { to: addressesOf(message.recipients) }; + if (message.ccRecipients.length > 0) recipients.cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) recipients.bcc = addressesOf(message.bccRecipients); + const content = { subject: message.subject }; + if (message.content.text) content.plainText = message.content.text; + if ('html' in message.content) content.html = message.content.html; + const payload = { senderAddress: toAsciiAddress(message.sender.address), recipients, content }; + if (message.replyRecipients.length > 0) payload.replyTo = addressesOf(message.replyRecipients); + const headers = headersOf(message); + if (headers !== undefined) payload.headers = headers; + const disabled = trackingDisabledOf(message, config); + if (disabled !== undefined) payload.userEngagementTrackingDisabled = disabled; + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.attachments = []; + for (const item of attachments) { + const part = { + name: item.attachment.filename, + contentType: item.attachment.contentType, + contentInBase64: item.content, + }; + if (item.attachment.inline) part.contentId = item.attachment.contentId; + payload.attachments.push(part); + } + } + const bytes = Buffer.byteLength(JSON.stringify(payload), 'utf8'); + if (bytes > MAX_REQUEST_BYTES) { + throw new RangeError( + `The request is ${bytes} bytes; Azure takes at most ${MAX_REQUEST_BYTES}, attachments included.`, + ); + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/azure/sign.js b/src/azure/sign.js new file mode 100644 index 0000000..57756af --- /dev/null +++ b/src/azure/sign.js @@ -0,0 +1,82 @@ +// Ported from unemail v0.7.0 (src/drivers/_azure/sign.ts): the HMAC-SHA256 +// signature of a request to Azure Communication Services, and the +// connection string of a resource. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the digests are the synchronous ones of node:crypto, +// the key decoded with `Buffer` rather than `atob`. + +const { createHash, createHmac } = require('node:crypto'); + +// The header fields the signature covers, in its order. +const SIGNED_HEADERS = 'x-ms-date;host;x-ms-content-sha256'; +const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; + +/** + * Whether a string can be the base64 access key Azure issues. A key that + * does not decode signs every request wrong, which Azure answers with a bare + * 401, so it is checked when the transport is made. + * + * @internal + */ +const isAccessKey = (value) => + typeof value === 'string' && value.length > 0 && value.length % 4 === 0 && BASE64.test(value); + +/** + * Signs a request: HMAC-SHA256 with the decoded key over its method, its + * path and query, and the date, host and SHA-256 of its body, as + * `HMAC-SHA256 SignedHeaders=…&Signature=…`. + * + * @param request `method`, `url`, `body` (a string, or none for a GET) and + * `date`, the time of the request. + * @param accessKey The base64 access key. + * @returns The header fields to add: `x-ms-date`, `x-ms-content-sha256` and + * `Authorization`. Host is written by `fetch` from the URL. + * @internal + */ +const signRequest = (request, accessKey) => { + const url = new URL(request.url); + // RFC 1123, which `toUTCString` writes. + const date = request.date.toUTCString(); + const contentHash = createHash('sha256') + .update(request.body ?? '', 'utf8') + .digest('base64'); + const stringToSign = + `${request.method.toUpperCase()}\n${url.pathname}${url.search}\n` + + `${date};${url.host};${contentHash}`; + const signature = createHmac('sha256', Buffer.from(accessKey, 'base64')) + .update(stringToSign, 'utf8') + .digest('base64'); + return { + 'x-ms-date': date, + 'x-ms-content-sha256': contentHash, + Authorization: `HMAC-SHA256 SignedHeaders=${SIGNED_HEADERS}&Signature=${signature}`, + }; +}; + +/** + * Splits the connection string the Keys blade of a resource shows, + * `endpoint=https://….communication.azure.com/;accesskey=`. + * + * @param value The connection string. + * @returns `{ endpoint, accessKey }`, or `undefined` if it is malformed. + * @internal + */ +const parseConnectionString = (value) => { + let endpoint = ''; + let accessKey = ''; + for (const part of value.split(';')) { + const segment = part.trim(); + if (segment === '') continue; + const at = segment.indexOf('='); + if (at < 1) return undefined; + const name = segment.slice(0, at).trim().toLowerCase(); + // Only the first `=` separates: a base64 key ends with one or two. + const item = segment.slice(at + 1).trim(); + if (name === 'endpoint') endpoint = item; + else if (name === 'accesskey') accessKey = item; + } + if (endpoint === '' || accessKey === '') return undefined; + return { endpoint, accessKey }; +}; + +module.exports = { isAccessKey, parseConnectionString, signRequest }; diff --git a/tests/types/azure.test-d.ts b/tests/types/azure.test-d.ts new file mode 100644 index 0000000..fe55cdd --- /dev/null +++ b/tests/types/azure.test-d.ts @@ -0,0 +1,32 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { AzureTransport } from '../../src/azure/index.js'; + +const transport = new AzureTransport({ + connectionString: 'endpoint=https://r.communication.azure.com/;accesskey=a2V5', + apiVersion: '2025-09-01', + userEngagementTrackingDisabled: true, + concurrency: 2, + now: () => Date.now(), +}); +expectType<'azure'>(transport.id); +expectAssignable>(transport); +new AzureTransport({ endpoint: 'https://r.communication.azure.com', accessKey: 'a2V5' }); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new AzureTransport({})); +expectError(new AzureTransport({ endpoint: 'https://r.communication.azure.com' })); +expectError( + new AzureTransport({ + connectionString: 'endpoint=https://r;accesskey=a2V5', + endpoint: 'https://r', + accessKey: 'a2V5', + }), +); diff --git a/tests/unit/azure.test.js b/tests/unit/azure.test.js new file mode 100644 index 0000000..8f967cf --- /dev/null +++ b/tests/unit/azure.test.js @@ -0,0 +1,329 @@ +// Not ported: tests of ./azure, ported from unemail's Azure Communication +// Services driver, against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { AzureTransport } = require('../../src/azure/index.js'); +const { toUuid } = require('../../src/internal/uuid.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const NOW = Date.parse('2026-10-17T12:00:00Z'); +const KEY = Buffer.from('azure-access-key-secret').toString('base64'); +const ENDPOINT = 'https://contoso.communication.azure.com'; +const transport = (config = {}) => + new AzureTransport({ + connectionString: `endpoint=${ENDPOINT}/;accesskey=${KEY}`, + retries: 0, + now: () => NOW, + generateId: () => 'gen-1', + ...config, + }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const accepted = (id = 'op-1') => + json({ id, status: 'Running' }, 202, { + 'operation-location': `${ENDPOINT}/emails/operations/${id}?api-version=2025-09-01`, + 'retry-after': '20', + }); + +// The signature by Web Crypto, apart from the transport's node:crypto. +const expectedSignature = async (method, url, body, date) => { + const subtle = globalThis.crypto.subtle; + const encoder = new TextEncoder(); + const hash = Buffer.from(await subtle.digest('SHA-256', encoder.encode(body))).toString('base64'); + const parsed = new URL(url); + const text = `${method}\n${parsed.pathname}${parsed.search}\n${date};${parsed.host};${hash}`; + const key = await subtle.importKey( + 'raw', + Buffer.from(KEY, 'base64'), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + const signature = Buffer.from(await subtle.sign('HMAC', key, encoder.encode(text))); + return { + hash, + authorization: `HMAC-SHA256 SignedHeaders=x-ms-date;host;x-ms-content-sha256&Signature=${signature.toString('base64')}`, + }; +}; + +test('Azure: a message is the signed request of emails:send', async (t) => { + const requests = stubFetch(t, () => accepted()); + const receipt = await transport().send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: { name: 'Desk', address: 'desk@example.com' }, + priority: 'high', + tags: ['orders'], + metadata: { orderId: '42' }, + tracking: { opens: false, clicks: false }, + list: { unsubscribe: 'https://example.com/u' }, + idempotencyKey: 'order-42', + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'op-1', provider: 'azure' }); + const [request] = requests; + assert.strictEqual(request.url, `${ENDPOINT}/emails:send?api-version=2025-09-01`); + assert.strictEqual(request.redirect, 'manual'); + const date = new Date(NOW).toUTCString(); + const signed = await expectedSignature('POST', request.url, request.body, date); + assert.strictEqual(request.headers.get('x-ms-date'), date); + assert.strictEqual(request.headers.get('x-ms-content-sha256'), signed.hash); + assert.strictEqual(request.headers.get('authorization'), signed.authorization); + const key = toUuid('order-42'); + assert.strictEqual(request.headers.get('operation-id'), key); + assert.strictEqual(request.headers.get('repeatability-request-id'), key); + assert.strictEqual(request.headers.get('repeatability-first-sent'), date); + assert.deepStrictEqual(request.json, { + senderAddress: 'shop@example.com', + recipients: { + to: [ + { address: 'bob@example.com' }, + { address: 'carol@xn--bcher-kva.de', displayName: 'Carol' }, + ], + cc: [{ address: 'dave@example.com' }], + bcc: [{ address: 'erin@example.com' }], + }, + content: { subject: 'Your order', plainText: 'Shipped', html: '

Shipped

' }, + replyTo: [{ address: 'desk@example.com', displayName: 'Desk' }], + headers: { + 'X-Priority': '1', + 'X-MSMail-Priority': 'High', + Importance: 'High', + 'List-Unsubscribe': '', + 'X-Metadata-orderId': '42', + }, + userEngagementTrackingDisabled: true, + attachments: [ + { name: 'r.txt', contentType: 'text/plain', contentInBase64: 'aGk=' }, + { + name: 'logo.png', + contentType: 'image/png', + contentInBase64: 'AQ==', + contentId: 'logo@example.com', + }, + ], + }); +}); + +test('Azure: a retry keeps the key and the first time, and is signed anew', async (t) => { + t.mock.method(Math, 'random', () => 0); + let now = NOW; + const answers = [json({ error: { code: 'ServiceUnavailable' } }, 504), accepted('op-2')]; + const requests = stubFetch(t, () => { + const answer = answers.shift(); + now += 1000; + return answer; + }); + const receipt = await transport({ retries: 1, now: () => now }).send(message()); + assert.strictEqual(receipt.messageId, 'op-2'); + assert.strictEqual(requests.length, 2); + const key = toUuid('gen-1'); + for (const request of requests) { + assert.strictEqual(request.headers.get('repeatability-request-id'), key); + assert.strictEqual( + request.headers.get('repeatability-first-sent'), + new Date(NOW).toUTCString(), + ); + } + assert.notStrictEqual(requests[0].headers.get('x-ms-date'), requests[1].headers.get('x-ms-date')); + const signed = await expectedSignature( + 'POST', + requests[1].url, + requests[1].body, + requests[1].headers.get('x-ms-date'), + ); + assert.strictEqual(requests[1].headers.get('authorization'), signed.authorization); +}); + +test('Azure: a repeat Azure took before is of unknown delivery', async (t) => { + t.mock.method(Math, 'random', () => 0); + const answers = [ + json({ error: { code: 'Conflict', message: 'The request is still running.' } }, 409), + json({ error: { code: 'ServiceUnavailable' } }, 504), + json({ error: { code: 'PreconditionFailed', message: 'Outside the window.' } }, 412), + json({ error: { code: 'PreconditionFailed', message: 'Clock skew.' } }, 412), + ]; + stubFetch(t, () => answers.shift()); + const running = await transport().send(message()); + assert.strictEqual(running.successful, false); + assert.strictEqual(running.deliveryUnknown, true); + assert.strictEqual(running.retryable, false); + assert.match(running.errorMessages[0], /still running.*a repeat of a request Azure took before/); + const late = await transport({ retries: 1 }).send(message()); + assert.strictEqual(late.deliveryUnknown, true); + const skewed = await transport().send(message()); + assert.strictEqual(skewed.deliveryUnknown, undefined); + assert.strictEqual(skewed.errors[0].code, 'azure.PreconditionFailed'); +}); + +test('Azure: an error code says what failed, a failed operation fails the receipt', async (t) => { + const answers = [ + json({ error: { code: 'Unauthorized', message: 'Denied by the authentication policy.' } }, 401), + json({ error: { code: 'TooManyRequests', message: 'Slow down.' } }, 400), + json({ error: { code: 'InvalidSenderDomain', message: 'Domain not linked.' } }, 400), + json({ id: 'op-3', status: 'Failed', error: { code: 'Blocked', message: 'Blocked.' } }, 202), + json(null, 202, { 'operation-location': 'x' }), + json({ id: 'op-4', status: 'Canceled' }, 202), + json({ error: {} }, 500), + ]; + stubFetch(t, () => answers.shift()); + const auth = await transport().send(message()); + assert.strictEqual( + auth.errorMessages[0], + 'Azure Communication Services: Denied by the authentication policy.', + ); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errors[0].code, 'azure.Unauthorized'); + const throttled = await transport().send(message()); + assert.strictEqual(throttled.errors[0].category, 'rate-limit'); + assert.strictEqual(throttled.retryable, true); + assert.strictEqual((await transport().send(message())).errors[0].category, 'validation'); + const failed = await transport().send(message()); + assert.strictEqual(failed.errorMessages[0], 'Azure Communication Services: Blocked.'); + assert.strictEqual(failed.errors[0].code, 'azure.Blocked'); + assert.strictEqual(failed.retryable, false); + // Without an id in the body, the receipt has the one sent. + assert.strictEqual((await transport().send(message())).messageId, toUuid('gen-1')); + const canceled = await transport().send(message()); + assert.strictEqual(canceled.errorMessages[0], 'Azure Communication Services: Canceled'); + assert.strictEqual(canceled.errors[0].code, 'rejected'); + const server = await transport().send(message()); + assert.strictEqual(server.errors[0].category, 'server-error'); + assert.strictEqual(server.retryable, true); +}); + +test('Azure: tracking, size and a schedule are refused before a request', async (t) => { + const requests = stubFetch(t, () => accepted()); + const refusals = [ + [message({ tracking: { opens: true, clicks: false } }), /on or off together/], + [message({ scheduledAt: new Date(NOW + 1000) }), /Invalid scheduledAt: azure/], + [ + message({ + attachments: { + filename: 'big.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: new Uint8Array(8 * 1024 * 1024), + }, + }), + /Azure takes at most 10485760/, + ], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('Azure: tracking follows the message, then the configuration', async (t) => { + const requests = stubFetch(t, () => accepted()); + const disabled = transport({ userEngagementTrackingDisabled: true }); + await disabled.send(message()); + await disabled.send(message({ tracking: { clicks: true } })); + await disabled.send(message({ tracking: {} })); + await transport().send(message()); + assert.deepStrictEqual( + requests.map((request) => request.json.userEngagementTrackingDisabled), + [true, false, true, undefined], + ); +}); + +test('Azure: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => accepted(`op-${index++}`)); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ scheduledAt: new Date(NOW + 1000) }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('Azure: verify reads an operation that does not exist, signed', async (t) => { + const answers = [ + json({ error: { code: 'NotFound' } }, 404), + json({ id: 'x', status: 'Succeeded' }), + json({ error: { code: 'Unauthorized' } }, 401), + ]; + const requests = stubFetch(t, () => answers.shift()); + const made = transport({ connectionString: undefined, endpoint: `${ENDPOINT}/`, accessKey: KEY }); + await made.verify(); + const [request] = requests; + assert.strictEqual( + request.url, + `${ENDPOINT}/emails/operations/00000000-0000-0000-0000-000000000000?api-version=2025-09-01`, + ); + const date = new Date(NOW).toUTCString(); + const signed = await expectedSignature('GET', request.url, '', date); + assert.strictEqual(request.headers.get('authorization'), signed.authorization); + await made.verify(); + await assert.rejects(made.verify(), /Azure Communication Services refused the credentials/); +}); + +test('Azure: the credentials are checked, never read from the environment, never shown', async (t) => { + // unemail read the connection string from here when none was given. + process.env.COMMUNICATION_SERVICES_CONNECTION_STRING = `endpoint=${ENDPOINT};accesskey=${KEY}`; + t.after(() => delete process.env.COMMUNICATION_SERVICES_CONNECTION_STRING); + assert.throws(() => new AzureTransport({}), /needs connectionString, or endpoint and accessKey/); + assert.throws(() => transport({ connectionString: 'accesskey=abc' }), /connectionString must be/); + assert.throws(() => transport({ connectionString: 'endpoint' }), /connectionString must be/); + assert.throws(() => transport({ connectionString: 42 }), /connectionString must be/); + assert.throws( + () => transport({ connectionString: `endpoint=${ENDPOINT};accesskey=not base64!` }), + /accessKey must be the base64 key/, + ); + assert.throws( + () => transport({ connectionString: `endpoint=http://contoso.example.com;accesskey=${KEY}` }), + /`endpoint` must use HTTPS/, + ); + assert.throws(() => transport({ apiVersion: '2025-09-01&x=1' }), /apiVersion must be a date/); + assert.throws( + () => transport({ userEngagementTrackingDisabled: 'yes' }), + /userEngagementTrackingDisabled must be a boolean/, + ); + const local = transport({ + connectionString: ` endpoint = http://127.0.0.1:9000 ; ; accesskey=${KEY};`, + apiVersion: '2023-01-15-preview', + }); + assert.strictEqual(local.config.endpoint, 'http://127.0.0.1:9000'); + const made = transport(); + assert.ok(!JSON.stringify(made).includes(KEY)); + assert.ok(!util.inspect(made, { depth: 5 }).includes(KEY)); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); From fa4950607479713b44337349c3ea7080e84d22f1 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:02:11 +0300 Subject: [PATCH 025/143] feat(scaleway): a transport over Scaleway Transactional Email Ported from unemail's Scaleway driver onto the shared client of the HTTP providers. The region is checked and encoded as a segment of the path; the reply-to addresses and metadata go in `additional_headers`, as Scaleway has no field for them; an inline attachment, which it has no Content-ID for, is refused rather than sent as a file the HTML cannot refer to (U8), as are a schedule and a request over 2 MB. An error's type names the fields of `invalid_arguments`, and a spent quota is a `configuration` failure. `verify` lists the domains of the project. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/scaleway.md | 80 +++++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/scaleway/config.js | 44 ++++++ src/scaleway/http-client.js | 64 +++++++++ src/scaleway/index.d.ts | 74 +++++++++++ src/scaleway/index.js | 7 + src/scaleway/message-converter.js | 100 ++++++++++++++ src/scaleway/scaleway-transport.js | 102 ++++++++++++++ tests/types/scaleway.test-d.ts | 22 +++ tests/unit/scaleway.test.js | 207 +++++++++++++++++++++++++++++ 14 files changed, 710 insertions(+) create mode 100644 docs/transports/scaleway.md create mode 100644 src/scaleway/config.js create mode 100644 src/scaleway/http-client.js create mode 100644 src/scaleway/index.d.ts create mode 100644 src/scaleway/index.js create mode 100644 src/scaleway/message-converter.js create mode 100644 src/scaleway/scaleway-transport.js create mode 100644 tests/types/scaleway.test-d.ts create mode 100644 tests/unit/scaleway.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 99f2e4d..bb7e02f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -540,6 +540,9 @@ fails on upstream's behavior): reads only its options, holds the endpoint to HTTPS or loopback HTTP, sends both headers with the key as `Operation-Id` too, reports a repeat Azure answers 409 or 412 as `deliveryUnknown`, and refuses tracking it cannot express. +- U8. unemail (`src/drivers/scaleway.ts`): an inline attachment went as an ordinary file without + its Content-ID, which the API has no field for, so the `cid:` of the HTML matched nothing. + `./scaleway` refuses it before the request. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9d6a6f9..dcdfba9 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -92,6 +92,7 @@ const guide = [ { text: 'ZeptoMail', link: '/transports/zeptomail' }, { text: 'SMTP2GO', link: '/transports/smtp2go' }, { text: 'Azure Communication Services', link: '/transports/azure' }, + { text: 'Scaleway', link: '/transports/scaleway' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 5c1efda..2ae38b5 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -61,6 +61,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/zeptomail` | `ZeptoMailTransport` | [ZeptoMail](/transports/zeptomail) | | `@alexify/mailer/smtp2go` | `Smtp2goTransport` | [SMTP2GO](/transports/smtp2go) | | `@alexify/mailer/azure` | `AzureTransport` | [Azure Communication Services](/transports/azure) | +| `@alexify/mailer/scaleway` | `ScalewayTransport` | [Scaleway](/transports/scaleway) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/scaleway.md b/docs/transports/scaleway.md new file mode 100644 index 0000000..1f02d1d --- /dev/null +++ b/docs/transports/scaleway.md @@ -0,0 +1,80 @@ +# Scaleway transport + +The Scaleway transport sends through the [Transactional Email] API of Scaleway, +from the Scaleway driver of unemail. The API is `v1alpha1`, which Scaleway may +change without the guarantees of a stable version. + +[Transactional Email]: https://www.scaleway.com/en/transactional-email-tem/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { ScalewayTransport } = require('@alexify/mailer/scaleway'); + +const transport = new ScalewayTransport({ + secretKey: process.env.SCW_SECRET_KEY, + projectId: process.env.SCW_DEFAULT_PROJECT_ID, +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); // the id of the email +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `secretKey` | (required) | The secret key of an API key, sent as `X-Auth-Token`. | +| `projectId` | (required) | The project the emails are made in. | +| `region` | `'fr-par'` | The region of Transactional Email. | +| `baseUrl` | `https://api.scaleway.com` | HTTPS, or HTTP to this machine. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before Scaleway took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- Scaleway has no field for the reply-to addresses or for metadata: Reply-To + and `metadata` (`X-Metadata-`) go in `additional_headers`, with custom + header fields, List-* and the threading fields. +- Scaleway has no tags or per-message tracking, which are not sent, as over + SMTP. It cannot schedule a message, so `scheduledAt` is refused as + `validation`. +- It has no Content-ID for an attachment, so an inline one is refused rather + than sent as a file the HTML cannot refer to. +- A request may be at most 2 MB, its attachments in base64 included; a larger + one is refused before it is sent. Scaleway takes only the attachment types + it lists, and answers another with a 400 naming it. + +Scaleway makes an email of a message for each recipient; the receipt has the id +of the first. An error's type says more than its status: `invalid_arguments` +names the fields, and `quotas_exceeded`, the monthly quota, is a +`configuration` failure that no retry clears. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. + +## Verifying the configuration + +`verify()` lists the domains of the project; a key without that permission is +verified too. diff --git a/package.json b/package.json index e978305..a6d29af 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,10 @@ "types": "./src/azure/index.d.ts", "default": "./src/azure/index.js" }, + "./scaleway": { + "types": "./src/scaleway/index.d.ts", + "default": "./src/scaleway/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 3109f10..90efc4a 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -21,6 +21,7 @@ "./zeptomail": 7, "./smtp2go": 7, "./azure": 8, + "./scaleway": 7, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/scaleway/config.js b/src/scaleway/config.js new file mode 100644 index 0000000..ab5bb82 --- /dev/null +++ b/src/scaleway/config.js @@ -0,0 +1,44 @@ +// Ported from unemail v0.7.0 (src/drivers/scaleway.ts): the options of the +// Scaleway Transactional Email driver. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the region must look like one (`fr-par`), as it is a +// segment of the path. + +const { checkCredential } = require('../internal/api.js'); +const { checkConcurrency, checkRequestOptions, checkSecureUrl } = require('../internal/config.js'); + +const REGION = /^[a-z]{2}-[a-z]{3}$/; + +/** + * Resolves the options of the Scaleway transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createScalewayConfig = (config = {}) => { + const region = config.region ?? 'fr-par'; + if (typeof region !== 'string' || !REGION.test(region)) { + throw new TypeError(`region must be a Scaleway region such as fr-par: ${String(region)}`); + } + const resolved = { + secretKey: checkCredential(config.secretKey, 'secretKey', 'ScalewayTransport'), + projectId: checkCredential(config.projectId, 'projectId', 'ScalewayTransport'), + region, + baseUrl: config.baseUrl ?? 'https://api.scaleway.com', + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createScalewayConfig }; diff --git a/src/scaleway/http-client.js b/src/scaleway/http-client.js new file mode 100644 index 0000000..0499971 --- /dev/null +++ b/src/scaleway/http-client.js @@ -0,0 +1,64 @@ +// Ported from unemail v0.7.0 (src/drivers/scaleway.ts): the requests of the +// Transactional Email API of Scaleway, and what its error types say. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers; a +// spent quota is a `configuration` failure rather than a rate limit. + +const { createApiClient } = require('../internal/api.js'); + +// `{ type, message, details: [{ argument_name, reason }] }`. +const messageOf = (json) => { + if (json == null || typeof json !== 'object') return undefined; + let message = typeof json.message === 'string' ? json.message : ''; + if (json.type === 'invalid_arguments' && Array.isArray(json.details)) { + let fields = ''; + for (const detail of json.details) { + let field = typeof detail?.argument_name === 'string' ? detail.argument_name : ''; + if (typeof detail?.reason === 'string') { + field += field === '' ? detail.reason : ` ${detail.reason}`; + } + if (field !== '') fields += fields === '' ? field : `; ${field}`; + } + if (fields !== '') message = `${message === '' ? 'invalid arguments' : message}: ${fields}`; + } + return message === '' ? undefined : message; +}; + +const classify = (_status, json) => { + const type = json?.type; + if (typeof type !== 'string' || type === '') return undefined; + // A 403 for the monthly quota, which no retry clears. + const classified = + type === 'quotas_exceeded' ? { category: 'configuration', retryable: false } : {}; + return { ...classified, code: `scaleway.${type}` }; +}; + +/** + * The client of the Transactional Email API of Scaleway. + * + * @internal + */ +const createScalewayClient = (config) => { + const client = createApiClient(config, { + id: 'scaleway', + label: 'Scaleway', + authorize: (headers) => headers.set('X-Auth-Token', config.secretKey), + messageOf, + classify, + }); + const base = + `${config.baseUrl}/transactional-email/v1alpha1/regions/` + encodeURIComponent(config.region); + return { + ...client, + base, + send: (payload, options) => + client.request({ + url: `${base}/emails`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }), + }; +}; + +module.exports = { createScalewayClient }; diff --git a/src/scaleway/index.d.ts b/src/scaleway/index.d.ts new file mode 100644 index 0000000..2177ff2 --- /dev/null +++ b/src/scaleway/index.d.ts @@ -0,0 +1,74 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link ScalewayTransport}. + */ +export interface ScalewayConfig extends TransportDependencies { + /** The secret key of a Scaleway API key, sent as `X-Auth-Token`. */ + readonly secretKey: string; + /** The project the emails are made in. */ + readonly projectId: string; + /** + * The region of Transactional Email. + * @default "fr-par" + */ + readonly region?: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.scaleway.com" + */ + readonly baseUrl?: string; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before Scaleway took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * A transport over the Transactional Email API of Scaleway. The reply-to + * addresses and `metadata` (`X-Metadata-`) go in header fields; Scaleway + * has no tags, per-message tracking, scheduling or inline attachments, and + * takes a request of at most 2 MB. + * + * @example + * ```js + * const { ScalewayTransport } = require('@alexify/mailer/scaleway'); + * const transport = new ScalewayTransport({ + * secretKey: process.env.SCW_SECRET_KEY, + * projectId: process.env.SCW_DEFAULT_PROJECT_ID, + * }); + * ``` + */ +export declare class ScalewayTransport implements VerifiableTransport<'scaleway'> { + readonly id = 'scaleway'; + constructor(config: ScalewayConfig); + send(message: Message, options?: TransportOptions): Promise>; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the secret key by listing the domains of the project. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/scaleway/index.js b/src/scaleway/index.js new file mode 100644 index 0000000..9468524 --- /dev/null +++ b/src/scaleway/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/scaleway.ts): the entry of the +// Scaleway transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { ScalewayTransport } = require('./scaleway-transport.js'); + +module.exports = { ScalewayTransport }; diff --git a/src/scaleway/message-converter.js b/src/scaleway/message-converter.js new file mode 100644 index 0000000..e383c35 --- /dev/null +++ b/src/scaleway/message-converter.js @@ -0,0 +1,100 @@ +// Ported from unemail v0.7.0 (src/drivers/scaleway.ts): a message as the +// request of Scaleway's `/emails`, `toPayload` and `toHeaders`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: an inline attachment is refused, where unemail sent +// it as an ordinary file without its Content-ID, so the `cid:` of the HTML +// matched nothing; tags, which Scaleway has no field for, are not sent, as +// over SMTP; and a scheduled message is refused. + +const { formatMailbox, toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { formatAddresses } = require('../internal/provider.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +// "Maximum email size (API): 2 MB, including the email and all attachments". +const MAX_EMAIL_BYTES = 2 * 1024 * 1024; + +const addressOf = (address) => { + const result = { email: toAsciiAddress(address.address) }; + if (address.name) result.name = address.name; + return result; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +// Scaleway has no reply-to or metadata field: both are header fields, as its +// documentation shows for Reply-To. +const headersOf = (message) => { + const headers = []; + for (const field of headerFieldsOf(message)) { + headers.push({ key: field.name, value: field.value }); + } + if (message.replyRecipients.length > 0) { + const value = formatAddresses(message.replyRecipients, formatMailbox).join(', '); + headers.push({ key: 'Reply-To', value }); + } + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers.push({ key: `X-Metadata-${key}`, value: message.metadata[key] }); + } + } + return headers; +}; + +/** + * Converts a message to a request of `/emails`. + * + * @param message The message. + * @param config The configuration. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message has an inline attachment, or is over + * 2 MB. + * @throws {TypeError} If it is scheduled. + */ +const convertMessage = async (message, config, signal) => { + refuseScheduledAt(message, 'scaleway'); + for (const attachment of message.attachments) { + if (attachment.inline) { + throw new RangeError( + 'Scaleway cannot send an inline attachment, which it has no Content-ID for.', + ); + } + } + const payload = { + from: addressOf(message.sender), + to: addressesOf(message.recipients), + subject: message.subject, + project_id: config.projectId, + }; + if (message.ccRecipients.length > 0) payload.cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.bcc = addressesOf(message.bccRecipients); + if (message.content.text) payload.text = message.content.text; + if ('html' in message.content) payload.html = message.content.html; + const headers = headersOf(message); + if (headers.length > 0) payload.additional_headers = headers; + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.attachments = []; + for (const item of attachments) { + payload.attachments.push({ + name: item.attachment.filename, + type: item.attachment.contentType, + content: item.content, + }); + } + } + const bytes = Buffer.byteLength(JSON.stringify(payload), 'utf8'); + if (bytes > MAX_EMAIL_BYTES) { + throw new RangeError( + `The request is ${bytes} bytes; Scaleway takes at most ${MAX_EMAIL_BYTES}, attachments included.`, + ); + } + return payload; +}; + +module.exports = { convertMessage }; diff --git a/src/scaleway/scaleway-transport.js b/src/scaleway/scaleway-transport.js new file mode 100644 index 0000000..4aeffcc --- /dev/null +++ b/src/scaleway/scaleway-transport.js @@ -0,0 +1,102 @@ +// Ported from unemail v0.7.0 (src/drivers/scaleway.ts): a transport over the +// Transactional Email API of Scaleway. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: `verify` lists the domains of the project. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { verifyByRequest } = require('../internal/verify.js'); +const { createScalewayConfig } = require('./config.js'); +const { createScalewayClient } = require('./http-client.js'); +const { convertMessage } = require('./message-converter.js'); + +/** + * A transport over the Transactional Email API of Scaleway. + */ +class ScalewayTransport { + id = 'scaleway'; + + constructor(config) { + this.config = createScalewayConfig(config); + this.httpClient = createScalewayClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `/emails`. Scaleway makes an email of it for each + * recipient; the receipt has the id of the first. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, this.config, options?.signal); + options?.signal?.throwIfAborted(); + const answer = await this.httpClient.send(payload, options); + const emails = answer.json?.emails; + const messageId = Array.isArray(emails) ? emails[0]?.id : undefined; + if (typeof messageId !== 'string' || messageId === '') { + return createFailedReceipt('Scaleway answered without an email id.', { + provider: 'scaleway', + deliveryUnknown: true, + }); + } + return { successful: true, messageId, provider: 'scaleway' }; + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('scaleway', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the secret key by listing the domains of the project; a key + * without that permission is verified too. + * + * @param options Optional abort signal. + * @throws {Error} If Scaleway refuses the key or cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const query = `project_id=${encodeURIComponent(this.config.projectId)}&page_size=1`; + await verifyByRequest({ + provider: 'Scaleway', + send: (signal) => + this.httpClient.fetchWithAuth( + `${this.httpClient.base}/domains?${query}`, + { method: 'GET', headers: { Accept: 'application/json' } }, + signal, + ), + retries: this.config.retries, + timeoutMs: this.config.timeout, + signal: options?.signal, + log: this.httpClient.log, + }); + } +} + +module.exports = { ScalewayTransport }; diff --git a/tests/types/scaleway.test-d.ts b/tests/types/scaleway.test-d.ts new file mode 100644 index 0000000..45a9f5c --- /dev/null +++ b/tests/types/scaleway.test-d.ts @@ -0,0 +1,22 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type Receipt, type VerifiableTransport } from '../../index.js'; +import { ScalewayTransport } from '../../src/scaleway/index.js'; + +const transport = new ScalewayTransport({ + secretKey: 'scw-key', + projectId: 'project', + region: 'fr-par', + concurrency: 2, +}); +expectType<'scaleway'>(transport.id); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(transport.send(message)); +expectType>>(transport.sendMany([message])); +expectType>(transport.verify()); +expectError(new ScalewayTransport({ secretKey: 'k' })); diff --git a/tests/unit/scaleway.test.js b/tests/unit/scaleway.test.js new file mode 100644 index 0000000..c8c2b6e --- /dev/null +++ b/tests/unit/scaleway.test.js @@ -0,0 +1,207 @@ +// Not ported: tests of ./scaleway, ported from unemail's Scaleway driver, +// against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { ScalewayTransport } = require('../../src/scaleway/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const PROJECT = '11111111-2222-4333-8444-555555555555'; +const transport = (config = {}) => + new ScalewayTransport({ secretKey: 'scw-secret', projectId: PROJECT, retries: 0, ...config }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const created = (...ids) => json({ emails: ids.map((id) => ({ id, status: 'new' })) }); + +test('Scaleway: a message is the request of /emails, with the secret key', async (t) => { + const requests = stubFetch(t, () => created('em-1', 'em-2', 'em-3', 'em-4')); + const receipt = await transport().send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: ['support@example.com', { name: 'Desk', address: 'desk@example.com' }], + priority: 'low', + tags: ['orders'], + metadata: { orderId: '42' }, + list: { unsubscribe: 'https://example.com/u' }, + attachments: { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'em-1', provider: 'scaleway' }); + const [request] = requests; + assert.strictEqual( + request.url, + 'https://api.scaleway.com/transactional-email/v1alpha1/regions/fr-par/emails', + ); + assert.strictEqual(request.headers.get('x-auth-token'), 'scw-secret'); + assert.strictEqual(request.redirect, 'manual'); + assert.deepStrictEqual(request.json, { + from: { email: 'shop@example.com', name: 'Shop' }, + to: [{ email: 'bob@example.com' }, { email: 'carol@xn--bcher-kva.de', name: 'Carol' }], + subject: 'Your order', + project_id: PROJECT, + cc: [{ email: 'dave@example.com' }], + bcc: [{ email: 'erin@example.com' }], + text: 'Shipped', + html: '

Shipped

', + additional_headers: [ + { key: 'X-Priority', value: '5' }, + { key: 'X-MSMail-Priority', value: 'Low' }, + { key: 'Importance', value: 'Low' }, + { key: 'List-Unsubscribe', value: '' }, + { key: 'Reply-To', value: 'support@example.com, Desk ' }, + { key: 'X-Metadata-orderId', value: '42' }, + ], + attachments: [{ name: 'r.txt', type: 'text/plain', content: 'aGk=' }], + }); +}); + +test('Scaleway: the region is a segment of the path, a plain message has no headers', async (t) => { + const requests = stubFetch(t, () => created('em-1')); + await transport({ region: 'nl-ams', baseUrl: 'http://localhost:9000' }).send( + message({ content: { text: 'plain' } }), + ); + assert.strictEqual( + requests[0].url, + 'http://localhost:9000/transactional-email/v1alpha1/regions/nl-ams/emails', + ); + for (const key of ['cc', 'bcc', 'html', 'additional_headers', 'attachments']) { + assert.strictEqual(requests[0].json[key], undefined, key); + } +}); + +test('Scaleway: an error type says what failed', async (t) => { + const answers = [ + json( + { + type: 'invalid_arguments', + message: 'invalid argument(s)', + details: [ + { argument_name: 'from.email', reason: 'constraint' }, + { argument_name: 'subject' }, + { reason: 'unknown' }, + {}, + ], + }, + 400, + ), + json({ type: 'invalid_arguments', details: [{ argument_name: 'to' }] }, 400), + json({ type: 'invalid_arguments', message: 'bad', details: [] }, 400), + json({ type: 'quotas_exceeded', message: 'quota(s) exceeded for this resource' }, 403), + json({ type: 'denied_authentication', message: 'invalid secret key' }, 401), + json({ emails: [] }), + ]; + stubFetch(t, () => answers.shift()); + const invalid = await transport().send(message()); + assert.strictEqual( + invalid.errorMessages[0], + 'Scaleway: invalid argument(s): from.email constraint; subject; unknown', + ); + assert.strictEqual(invalid.errors[0].category, 'validation'); + assert.strictEqual(invalid.errors[0].code, 'scaleway.invalid_arguments'); + assert.strictEqual( + (await transport().send(message())).errorMessages[0], + 'Scaleway: invalid arguments: to', + ); + assert.strictEqual((await transport().send(message())).errorMessages[0], 'Scaleway: bad'); + const quota = await transport().send(message()); + assert.strictEqual(quota.errors[0].category, 'configuration'); + assert.strictEqual(quota.retryable, false); + const auth = await transport().send(message()); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errorMessages[0], 'Scaleway: invalid secret key'); + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); +}); + +test('Scaleway: an inline part, a schedule and 2 MB are refused before a request', async (t) => { + const requests = stubFetch(t, () => created('em-1')); + const refusals = [ + [ + message({ + attachments: { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array(1), + }, + }), + /cannot send an inline attachment/, + ], + [message({ scheduledAt: new Date() }), /Invalid scheduledAt: scaleway/], + [ + message({ + attachments: { + filename: 'big.pdf', + contentType: 'application/pdf', + inline: false, + contentId: '', + content: new Uint8Array(1600 * 1024), + }, + }), + /Scaleway takes at most 2097152/, + ], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('Scaleway: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => created(`em-${index++}`)); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ scheduledAt: new Date() }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('Scaleway: verify lists the domains, the options are checked, the key never shows', async (t) => { + const answers = [ + json({ domains: [], total_count: 0 }), + json({ type: 'denied_authentication' }, 401), + ]; + const requests = stubFetch(t, () => answers.shift()); + await transport().verify(); + assert.strictEqual( + requests[0].url, + `https://api.scaleway.com/transactional-email/v1alpha1/regions/fr-par/domains?project_id=${PROJECT}&page_size=1`, + ); + assert.strictEqual(requests[0].headers.get('x-auth-token'), 'scw-secret'); + await assert.rejects(transport().verify(), /Scaleway refused the credentials/); + assert.throws(() => new ScalewayTransport({ projectId: PROJECT }), /needs secretKey/); + assert.throws(() => new ScalewayTransport({ secretKey: 'k' }), /needs projectId/); + assert.throws(() => transport({ region: 'fr-par/../x' }), /region must be a Scaleway region/); + assert.throws(() => transport({ region: 1 }), /region must be a Scaleway region/); + assert.throws(() => transport({ baseUrl: 'http://api.example.com' }), /baseUrl/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('scw-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('scw-secret')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); +}); From e3528cd70b5aedcf42ca7ca85ec9e7effe70cb6b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:07:32 +0300 Subject: [PATCH 026/143] feat(cloudflare): a transport over the Cloudflare Email Service REST API Ported from unemail's REST driver of Cloudflare Email Service onto the shared client of the HTTP providers, in the shapes of the REST API rather than those of the Workers binding: `{ address, name }`, `reply_to` and `content_id` (U9). The receipt's id is the `message_id` of the answer, not one made of the recipients and the clock; recipients that bounced or are suppressed are listed on a successful receipt, which fails as `rejected` only when none was accepted, and a 200 with `success: false` is a validation failure. `sendRaw` goes through `send_raw`; `verify` asks Cloudflare whether the token is active, as a token of the account and then of a user. The decoding of a raw message as UTF-8, which SparkPost had of its own, moves to `internal/raw.js`. The budget is 8 KB: the raw-message reader adds half a kilobyte. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/cloudflare.md | 92 ++++++++ package.json | 4 + scripts/size-budgets.json | 1 + src/cloudflare/cloudflare-transport.js | 177 +++++++++++++++ src/cloudflare/config.js | 35 +++ src/cloudflare/http-client.js | 50 +++++ src/cloudflare/index.d.ts | 89 ++++++++ src/cloudflare/index.js | 7 + src/cloudflare/message-converter.js | 119 ++++++++++ src/internal/raw.js | 21 +- src/sparkpost/message-converter.js | 15 +- tests/types/cloudflare.test-d.ts | 27 +++ tests/unit/cloudflare.test.js | 293 +++++++++++++++++++++++++ 16 files changed, 926 insertions(+), 13 deletions(-) create mode 100644 docs/transports/cloudflare.md create mode 100644 src/cloudflare/cloudflare-transport.js create mode 100644 src/cloudflare/config.js create mode 100644 src/cloudflare/http-client.js create mode 100644 src/cloudflare/index.d.ts create mode 100644 src/cloudflare/index.js create mode 100644 src/cloudflare/message-converter.js create mode 100644 tests/types/cloudflare.test-d.ts create mode 100644 tests/unit/cloudflare.test.js diff --git a/CLAUDE.md b/CLAUDE.md index bb7e02f..14d0199 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -543,6 +543,13 @@ fails on upstream's behavior): - U8. unemail (`src/drivers/scaleway.ts`): an inline attachment went as an ordinary file without its Content-ID, which the API has no field for, so the `cid:` of the HTML matched nothing. `./scaleway` refuses it before the request. +- U9. unemail (`src/drivers/cloudflare-email-rest.ts`): the REST driver wrote the addresses as + `{ email, name }`, the reply-to as `replyTo` and the Content-ID as `contentId`, the shapes of the + Workers binding, where the REST API takes `{ address, name }`, `reply_to` and `content_id` + (Cloudflare's API reference and SDK 7.3.0); and the receipt's id was made of the delivered + addresses and the clock, which put the recipients into every log of a receipt and could repeat + within a millisecond, while the answer has a `message_id`. `./cloudflare` writes the REST + shapes and takes the `message_id`. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index dcdfba9..ee5565c 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -93,6 +93,7 @@ const guide = [ { text: 'SMTP2GO', link: '/transports/smtp2go' }, { text: 'Azure Communication Services', link: '/transports/azure' }, { text: 'Scaleway', link: '/transports/scaleway' }, + { text: 'Cloudflare', link: '/transports/cloudflare' }, { text: 'JMAP', link: '/transports/jmap' }, { text: 'Mock', link: '/transports/mock' }, { text: 'Custom transports', link: '/transports/custom' }, diff --git a/docs/api/exports.md b/docs/api/exports.md index 2ae38b5..039c870 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -62,6 +62,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/smtp2go` | `Smtp2goTransport` | [SMTP2GO](/transports/smtp2go) | | `@alexify/mailer/azure` | `AzureTransport` | [Azure Communication Services](/transports/azure) | | `@alexify/mailer/scaleway` | `ScalewayTransport` | [Scaleway](/transports/scaleway) | +| `@alexify/mailer/cloudflare` | `CloudflareTransport` | [Cloudflare](/transports/cloudflare) | | `@alexify/mailer/jmap` | `JmapTransport`, `createJmapConfig`, `JmapApiError`, `JMAP_ERROR_TYPES`, `isCapabilityError`, `uploadBlob` | [JMAP](/transports/jmap) | | `@alexify/mailer/mock` | `MockTransport` | [Mock](/transports/mock) | | `@alexify/mailer/pool` | `PoolTransport`, `RoundRobinStrategy`, `WeightedStrategy`, `PriorityStrategy`, `SelectorStrategy` | [Pool](/transports/pool) | diff --git a/docs/transports/cloudflare.md b/docs/transports/cloudflare.md new file mode 100644 index 0000000..6c1b63e --- /dev/null +++ b/docs/transports/cloudflare.md @@ -0,0 +1,92 @@ +# Cloudflare transport + +The Cloudflare transport sends through the REST API of [Cloudflare Email +Service], with an API token, from any runtime with `fetch`. It comes from the +REST driver of unemail. Inside a Worker, the service's binding needs no token +and is one hop shorter; this transport is for everywhere else. + +[Cloudflare Email Service]: https://developers.cloudflare.com/email-service/ + +## Installation + +```bash +pnpm add @alexify/mailer +# or +npm install @alexify/mailer +``` + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { CloudflareTransport } = require('@alexify/mailer/cloudflare'); + +const transport = new CloudflareTransport({ + accountId: process.env.CF_ACCOUNT_ID, + apiToken: process.env.CF_API_TOKEN, +}); + +const receipt = await transport.send( + createMessage({ + from: 'Shop ', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + }), +); +if (receipt.successful) console.log(receipt.messageId); // the message_id +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `accountId` | (required) | The id of the Cloudflare account. | +| `apiToken` | (required) | An API token with the permission to send email. | +| `baseUrl` | `https://api.cloudflare.com/client/v4` | HTTPS, or HTTP to this machine. | +| `concurrency` | `5` | The most requests of `sendMany()` at once. | +| `timeout` | `30000` | Milliseconds a request may take; 0 or `Infinity` for none. | +| `retries` | `3` | Retries of a request that failed before Cloudflare took it. | +| `headers` | | Header fields added to every request. | +| `fetch`, `logger`, `generateId` | | As for every transport. | + +## What a message becomes + +- Addresses go as `{ address, name }`, the reply-to address as `reply_to` (one + only) and an inline attachment with its Content-ID as `content_id`, the + shapes of the REST API. +- `metadata` goes in `X-Metadata-` headers, with custom header fields, + List-* and the threading fields. Cloudflare takes only the header fields it + lists, and answers another with a 400. +- Cloudflare has no tags or per-message tracking, which are not sent, as over + SMTP; it cannot schedule a message, so `scheduledAt` is refused as + `validation`. +- A message has at most 50 recipients across `to`, `cc` and `bcc`; more are + refused before the request. + +Cloudflare answers with the outcome of each recipient: delivered, queued, +bounced permanently, or dropped as on the suppression list. A receipt is +successful when any was delivered or queued, its id the `message_id` of the +answer; it lists the others in `bouncedRecipients` and `suppressedRecipients`. +When none was accepted it fails as `rejected`, not retryable. A 200 with +`success: false` is a request Cloudflare read but did not act on, a +`validation` failure; an error keeps Cloudflare's numeric code +(`cloudflare.10101`), its category chosen by the status. + +## Raw messages + +`sendRaw()` sends a ready MIME message through `send_raw`, routed by its +envelope. The API takes it as a JSON string, so its bytes must be UTF-8; others, +and an envelope without a sender, are refused as `validation` before any +request. + +## Batches + +`sendMany()` sends `concurrency` single requests at a time, each with its own +receipt. + +## Verifying the configuration + +`verify()` asks Cloudflare whether the token is active, as a token of the account +(`/accounts/{id}/tokens/verify`) and, when it is not one, of a user +(`/user/tokens/verify`). diff --git a/package.json b/package.json index a6d29af..a07e1f7 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,10 @@ "types": "./src/scaleway/index.d.ts", "default": "./src/scaleway/index.js" }, + "./cloudflare": { + "types": "./src/cloudflare/index.d.ts", + "default": "./src/cloudflare/index.js" + }, "./pool": { "types": "./src/pool/index.d.ts", "default": "./src/pool/index.js" diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 90efc4a..96cb4d5 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -22,6 +22,7 @@ "./smtp2go": 7, "./azure": 8, "./scaleway": 7, + "./cloudflare": 8, "./pool": 3, "./retry": 3, "./mock": 2, diff --git a/src/cloudflare/cloudflare-transport.js b/src/cloudflare/cloudflare-transport.js new file mode 100644 index 0000000..d5c7f36 --- /dev/null +++ b/src/cloudflare/cloudflare-transport.js @@ -0,0 +1,177 @@ +// Ported from unemail v0.7.0 (src/drivers/cloudflare-email-rest.ts): a +// transport over the Email Service API of Cloudflare, which answers with the +// outcome of each recipient. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the id of a receipt is the `message_id` of the +// answer, where unemail made one of the delivered addresses and the clock, +// which carried the recipients into every log of the receipt and two sends +// in one millisecond could share; recipients that bounced or were +// suppressed are listed on a successful receipt, which fails only when none +// was accepted; `sendRaw` goes through `send_raw`; and `verify` checks the +// token. + +const { generateId } = require('../core/id.js'); +const { createFailedReceipt } = require('../core/receipt.js'); +const { sendConcurrently } = require('../internal/batch.js'); +const { isCallerAbort, providerFailure, ProviderApiError } = require('../internal/api.js'); +const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { createCloudflareConfig } = require('./config.js'); +const { createCloudflareClient, errorCodeOf } = require('./http-client.js'); +const { convertMessage, convertRawMessage } = require('./message-converter.js'); + +const listOf = (value) => (Array.isArray(value) ? value : []); + +/** + * The receipt of `{ success, errors, result: { message_id, delivered, + * queued, permanent_bounces, suppressed_recipients } }`. + */ +const receiptOf = (json, config) => { + // A 200 with `success: false` is a request Cloudflare read but did not act on. + if (json?.success === false) { + const first = Array.isArray(json.errors) ? json.errors[0] : undefined; + return createFailedReceipt(`Cloudflare: ${first?.message ?? 'the send failed.'}`, { + provider: 'cloudflare', + category: 'validation', + retryable: false, + code: errorCodeOf(json), + providerDetails: json, + }); + } + const result = json?.result ?? {}; + const accepted = listOf(result.delivered).length + listOf(result.queued).length; + const bounced = listOf(result.permanent_bounces); + const suppressed = listOf(result.suppressed_recipients); + if (accepted === 0) { + const refused = bounced.length + suppressed.length > 0; + return createFailedReceipt( + refused + ? 'Cloudflare: every recipient bounced permanently or is suppressed.' + : 'Cloudflare accepted no recipient.', + refused + ? { + provider: 'cloudflare', + category: 'rejected', + retryable: false, + code: 'cloudflare.no_recipient_accepted', + providerDetails: result, + } + : { provider: 'cloudflare', deliveryUnknown: true }, + ); + } + const messageId = + typeof result.message_id === 'string' && result.message_id !== '' + ? result.message_id + : generateId('receipt-id', config.generateId); + const receipt = { successful: true, messageId, provider: 'cloudflare' }; + if (bounced.length > 0) receipt.bouncedRecipients = bounced; + if (suppressed.length > 0) receipt.suppressedRecipients = suppressed; + return receipt; +}; + +/** + * A transport over the Email Service API of Cloudflare. + */ +class CloudflareTransport { + id = 'cloudflare'; + + constructor(config) { + this.config = createCloudflareConfig(config); + this.httpClient = createCloudflareClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends one message with `email/sending/send`. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt, with the `message_id` of the answer. + * @throws The reason of the signal if it aborts. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + try { + const payload = await convertMessage(message, options?.signal); + options?.signal?.throwIfAborted(); + return receiptOf((await this.httpClient.send(payload, options)).json, this.config); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return providerFailure('cloudflare', error); + } + } + + /** + * Sends a ready MIME message with `email/sending/send_raw`, to the + * recipients of its envelope. + * + * @param message The raw message; its bytes must be UTF-8. + * @param options Optional abort signal and retry callback. + * @returns The receipt. + * @throws The reason of the signal if it aborts. + */ + async sendRaw(message, options) { + if (message?.content instanceof Promise) message.content.catch(() => {}); + options?.signal?.throwIfAborted(); + try { + const payload = convertRawMessage(await readRawMessage(message, options?.signal)); + return receiptOf((await this.httpClient.sendRaw(payload, options)).json, this.config); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + return rawMessageFailure('cloudflare', error) ?? providerFailure('cloudflare', error); + } + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendConcurrently( + messages, + this.config.concurrency, + (message) => this.send(message, options), + options?.signal, + ); + } + + /** + * Verifies the API token with Cloudflare's own check, as a token of the + * account and, failing that, of a user. + * + * @param options Optional abort signal. + * @throws {Error} If Cloudflare refuses the token, finds it inactive, or + * cannot be reached. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + const check = async (url) => { + const answer = await this.httpClient.request({ + url, + method: 'GET', + isIdempotent: true, + signal: options?.signal, + }); + const status = answer.json?.result?.status; + if (status !== 'active') { + throw new Error(`Cloudflare: the API token is ${String(status ?? 'not active')}.`); + } + }; + try { + await check(`${this.httpClient.account}/tokens/verify`); + } catch (error) { + // A token of a user is not one of the account. + const status = error instanceof ProviderApiError ? error.statusCode : undefined; + if (status === undefined || status >= 500) throw error; + await check(`${this.config.baseUrl}/user/tokens/verify`); + } + } +} + +module.exports = { CloudflareTransport }; diff --git a/src/cloudflare/config.js b/src/cloudflare/config.js new file mode 100644 index 0000000..9ee1e9e --- /dev/null +++ b/src/cloudflare/config.js @@ -0,0 +1,35 @@ +// Ported from unemail v0.7.0 (src/drivers/cloudflare-email-rest.ts): the +// options of the driver of Cloudflare Email Service over its REST API. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkCredential } = require('../internal/api.js'); +const { checkConcurrency, checkRequestOptions, checkSecureUrl } = require('../internal/config.js'); + +/** + * Resolves the options of the Cloudflare transport. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createCloudflareConfig = (config = {}) => { + const resolved = { + accountId: checkCredential(config.accountId, 'accountId', 'CloudflareTransport'), + apiToken: checkCredential(config.apiToken, 'apiToken', 'CloudflareTransport'), + baseUrl: config.baseUrl ?? 'https://api.cloudflare.com/client/v4', + concurrency: config.concurrency ?? 5, + timeout: config.timeout ?? 30000, + retries: config.retries ?? 3, + headers: config.headers ?? {}, + fetch: config.fetch, + generateId: config.generateId, + logger: config.logger, + }; + checkConcurrency(resolved.concurrency); + checkRequestOptions(resolved.timeout, resolved.retries); + checkSecureUrl(resolved.baseUrl, 'baseUrl'); + return resolved; +}; + +module.exports = { createCloudflareConfig }; diff --git a/src/cloudflare/http-client.js b/src/cloudflare/http-client.js new file mode 100644 index 0000000..1705a6e --- /dev/null +++ b/src/cloudflare/http-client.js @@ -0,0 +1,50 @@ +// Ported from unemail v0.7.0 (src/drivers/cloudflare-email-rest.ts): the +// requests of the Email Service API of Cloudflare. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: through the shared client of the HTTP providers, the +// status choosing the category and the numeric code of the error kept. + +const { createApiClient } = require('../internal/api.js'); + +/** + * The code of the first error of an answer, `{ errors: [{ code, message }] }`. + * + * @internal + */ +const errorCodeOf = (json) => { + const code = Array.isArray(json?.errors) ? json.errors[0]?.code : undefined; + return typeof code === 'number' || typeof code === 'string' ? `cloudflare.${code}` : undefined; +}; + +/** + * The client of the Email Service API of Cloudflare. + * + * @internal + */ +const createCloudflareClient = (config) => { + const client = createApiClient(config, { + id: 'cloudflare', + label: 'Cloudflare', + authorize: (headers) => headers.set('Authorization', `Bearer ${config.apiToken}`), + classify: (_status, json) => { + const code = errorCodeOf(json); + return code === undefined ? undefined : { code }; + }, + }); + const account = `${config.baseUrl}/accounts/${encodeURIComponent(config.accountId)}`; + const post = (path, payload, options) => + client.request({ + url: `${account}${path}`, + body: payload, + signal: options?.signal, + onRetry: options?.onRetry, + }); + return { + ...client, + account, + send: (payload, options) => post('/email/sending/send', payload, options), + sendRaw: (payload, options) => post('/email/sending/send_raw', payload, options), + }; +}; + +module.exports = { createCloudflareClient, errorCodeOf }; diff --git a/src/cloudflare/index.d.ts b/src/cloudflare/index.d.ts new file mode 100644 index 0000000..d37fc5b --- /dev/null +++ b/src/cloudflare/index.d.ts @@ -0,0 +1,89 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + TransportDependencies, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link CloudflareTransport}. + */ +export interface CloudflareConfig extends TransportDependencies { + /** The id of the Cloudflare account. */ + readonly accountId: string; + /** An API token with the permission to send email. */ + readonly apiToken: string; + /** + * The base URL of the API, https (or http to this machine). + * @default "https://api.cloudflare.com/client/v4" + */ + readonly baseUrl?: string; + /** + * The most requests of `sendMany()` at once. + * @default 5 + */ + readonly concurrency?: number; + /** + * The time a request may take, in milliseconds; 0 or `Infinity` for none. + * @default 30000 + */ + readonly timeout?: number; + /** + * Retries of a request that failed before Cloudflare took it. + * @default 3 + */ + readonly retries?: number; + /** Header fields added to every request. */ + readonly headers?: Readonly>; +} + +/** + * The receipt of {@link CloudflareTransport}: a successful one lists the + * recipients that bounced permanently or were dropped as suppressed, while + * the others received the message or have it queued. + */ +export type CloudflareReceipt = + | (Extract, { readonly successful: true }> & { + /** The recipients that bounced permanently, when there are any. */ + readonly bouncedRecipients?: readonly string[]; + /** The recipients dropped as on the suppression list, when there are any. */ + readonly suppressedRecipients?: readonly string[]; + }) + | Extract, { readonly successful: false }>; + +/** + * A transport over the Email Service API of Cloudflare, from any runtime with + * `fetch` (in a Worker, its binding needs no token). `metadata` goes in + * `X-Metadata-` headers; Cloudflare has no tags, per-message tracking + * or scheduling; a ready MIME message goes through `send_raw`. + * + * @example + * ```js + * const { CloudflareTransport } = require('@alexify/mailer/cloudflare'); + * const transport = new CloudflareTransport({ + * accountId: process.env.CF_ACCOUNT_ID, + * apiToken: process.env.CF_API_TOKEN, + * }); + * ``` + */ +export declare class CloudflareTransport + implements RawTransport<'cloudflare'>, VerifiableTransport<'cloudflare'> +{ + readonly id = 'cloudflare'; + constructor(config: CloudflareConfig); + send(message: Message, options?: TransportOptions): Promise; + /** Sends a ready MIME message, whose bytes must be UTF-8. */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise; + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable; + /** Verifies the API token with Cloudflare's own check. */ + verify(options?: TransportOptions): Promise; +} diff --git a/src/cloudflare/index.js b/src/cloudflare/index.js new file mode 100644 index 0000000..aee0ce0 --- /dev/null +++ b/src/cloudflare/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/drivers/cloudflare-email-rest.ts): the +// entry of the Cloudflare Email Service transport. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { CloudflareTransport } = require('./cloudflare-transport.js'); + +module.exports = { CloudflareTransport }; diff --git a/src/cloudflare/message-converter.js b/src/cloudflare/message-converter.js new file mode 100644 index 0000000..3955d41 --- /dev/null +++ b/src/cloudflare/message-converter.js @@ -0,0 +1,119 @@ +// Ported from unemail v0.7.0 (src/drivers/cloudflare-email-rest.ts): a message +// as the request of `email/sending/send`, `toPayload`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: an address is `{ address, name }`, the reply-to +// address `reply_to` and the Content-ID `content_id`, as the REST API takes +// them, where unemail wrote the `{ email, name }`, `replyTo` and `contentId` +// of the Workers binding; metadata goes in `X-Metadata-` headers; a +// second reply-to address and a scheduled message are refused; tags, which +// Cloudflare has no field for, are not sent, as over SMTP. + +const { toAsciiAddress } = require('../core/address.js'); +const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { rawMessageText } = require('../internal/raw.js'); +const { RawMessageValidationError } = require('../core/raw-message.js'); +const { refuseScheduledAt } = require('../internal/schedule.js'); + +// "The combined number of addresses across to, cc, and bcc must not exceed 50." +const MAX_RECIPIENTS = 50; + +const addressOf = (address) => { + const result = { address: toAsciiAddress(address.address) }; + if (address.name) result.name = address.name; + return result; +}; + +const addressesOf = (addresses) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); + return list; +}; + +const checkLimits = (message) => { + refuseScheduledAt(message, 'cloudflare'); + const total = + message.recipients.length + message.ccRecipients.length + message.bccRecipients.length; + if (total > MAX_RECIPIENTS) { + throw new RangeError(`Cloudflare takes at most ${MAX_RECIPIENTS} recipients a message.`); + } + if (message.replyRecipients.length > 1) { + throw new RangeError('Cloudflare takes one reply-to address.'); + } +}; + +const headersOf = (message) => { + const headers = {}; + let hasHeaders = false; + for (const field of headerFieldsOf(message)) { + headers[field.name] = field.value; + hasHeaders = true; + } + if (message.metadata !== undefined) { + for (const key in message.metadata) { + headers[`X-Metadata-${key}`] = message.metadata[key]; + hasHeaders = true; + } + } + return hasHeaders ? headers : undefined; +}; + +/** + * Converts a message to a request of `email/sending/send`. + * + * @param message The message. + * @param signal Cancels the reads of the attachments. + * @returns The request. + * @throws {RangeError} If the message is over a limit of Cloudflare. + * @throws {TypeError} If it is scheduled. + */ +const convertMessage = async (message, signal) => { + checkLimits(message); + const payload = { from: addressOf(message.sender), subject: message.subject }; + if (message.recipients.length > 0) payload.to = addressesOf(message.recipients); + if (message.ccRecipients.length > 0) payload.cc = addressesOf(message.ccRecipients); + if (message.bccRecipients.length > 0) payload.bcc = addressesOf(message.bccRecipients); + if (message.replyRecipients.length === 1) { + payload.reply_to = addressOf(message.replyRecipients[0]); + } + if (message.content.text) payload.text = message.content.text; + if ('html' in message.content) payload.html = message.content.html; + const headers = headersOf(message); + if (headers !== undefined) payload.headers = headers; + const attachments = await base64Attachments(message, signal); + if (attachments.length > 0) { + payload.attachments = []; + for (const item of attachments) { + const part = { + content: item.content, + filename: item.attachment.filename, + type: item.attachment.contentType, + disposition: item.attachment.inline ? 'inline' : 'attachment', + }; + if (item.attachment.inline) part.content_id = item.attachment.contentId; + payload.attachments.push(part); + } + } + return payload; +}; + +/** + * Converts a raw message, read with `readRawMessage`, to a request of + * `email/sending/send_raw`, the envelope routing it. + * + * @param raw The envelope and bytes. + * @returns The request. + * @throws {RawMessageValidationError} If the envelope has no sender, or the + * bytes are not UTF-8. + */ +const convertRawMessage = (raw) => { + if (raw.envelope.from === null) { + throw new RawMessageValidationError('Cloudflare needs a sender in the envelope.', 'envelope'); + } + return { + from: raw.envelope.from, + recipients: raw.envelope.to, + mime_message: rawMessageText(raw.bytes, 'Cloudflare'), + }; +}; + +module.exports = { convertMessage, convertRawMessage }; diff --git a/src/internal/raw.js b/src/internal/raw.js index 662e535..80fb3be 100644 --- a/src/internal/raw.js +++ b/src/internal/raw.js @@ -10,6 +10,8 @@ const { toAsciiAddress } = require('../core/address.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { joinChunks } = require('./body.js'); +const UTF8 = new TextDecoder('utf-8', { fatal: true }); + /** * Reads a raw message into one buffer, its lines checked as they are read, * and its envelope with domains as A-labels. @@ -51,4 +53,21 @@ const rawMessageFailure = (provider, error) => ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) : undefined; -module.exports = { rawMessageFailure, readRawMessage }; +/** + * The text of a raw message, for an API that takes it as a JSON string. + * + * @param bytes The bytes of the message. + * @param label The provider's name, for the error. + * @returns The text. + * @throws {RawMessageValidationError} If the bytes are not UTF-8. + * @internal + */ +const rawMessageText = (bytes, label) => { + try { + return UTF8.decode(bytes); + } catch { + throw new RawMessageValidationError(`${label} takes a raw message only as UTF-8.`, 'content'); + } +}; + +module.exports = { rawMessageFailure, rawMessageText, readRawMessage }; diff --git a/src/sparkpost/message-converter.js b/src/sparkpost/message-converter.js index cbcc6d3..9e7004b 100644 --- a/src/sparkpost/message-converter.js +++ b/src/sparkpost/message-converter.js @@ -9,13 +9,12 @@ // message goes as `email_rfc822`. const { formatMailbox, toAsciiAddress } = require('../core/address.js'); -const { RawMessageValidationError } = require('../core/raw-message.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); +const { rawMessageText } = require('../internal/raw.js'); // "Maximum number of tags - 10 per recipient". const MAX_TAGS = 10; -const UTF8 = new TextDecoder('utf-8', { fatal: true }); // `start_time` is "YYYY-MM-DDTHH:MM:SS+-HH:MM", without milliseconds. const startTimeOf = (date) => `${date.toISOString().slice(0, 19)}+00:00`; @@ -114,15 +113,6 @@ const convertMessage = async (message, config, signal) => { return transmission; }; -// The API takes the message as a JSON string, so its bytes must be UTF-8. -const textOf = (bytes) => { - try { - return UTF8.decode(bytes); - } catch { - throw new RawMessageValidationError('SparkPost takes a raw message only as UTF-8.', 'content'); - } -}; - /** * Converts a raw message, read with `readRawMessage`, to a transmission of * its RFC 822 text to the recipients of its envelope. @@ -136,7 +126,8 @@ const convertRawMessage = (raw, config) => { for (const email of raw.envelope.to) recipients.push({ address: { email } }); const transmission = { recipients, - content: { email_rfc822: textOf(raw.bytes) }, + // The API takes the message as a JSON string. + content: { email_rfc822: rawMessageText(raw.bytes, 'SparkPost') }, }; const options = optionsOf(undefined, config); if (Object.keys(options).length > 0) transmission.options = options; diff --git a/tests/types/cloudflare.test-d.ts b/tests/types/cloudflare.test-d.ts new file mode 100644 index 0000000..108649e --- /dev/null +++ b/tests/types/cloudflare.test-d.ts @@ -0,0 +1,27 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type RawTransport, type VerifiableTransport } from '../../index.js'; +import { CloudflareTransport, type CloudflareReceipt } from '../../src/cloudflare/index.js'; + +const transport = new CloudflareTransport({ + accountId: 'account', + apiToken: 'token', + concurrency: 2, +}); +expectType<'cloudflare'>(transport.id); +expectAssignable>(transport); +expectAssignable>(transport); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>(transport.send(message)); +expectType>(transport.sendMany([message])); +expectType>(transport.verify()); +const receipt = await transport.send(message); +if (receipt.successful) { + expectType(receipt.bouncedRecipients); + expectType(receipt.suppressedRecipients); +} +expectError(new CloudflareTransport({ accountId: 'a' })); diff --git a/tests/unit/cloudflare.test.js b/tests/unit/cloudflare.test.js new file mode 100644 index 0000000..0832f25 --- /dev/null +++ b/tests/unit/cloudflare.test.js @@ -0,0 +1,293 @@ +// Not ported: tests of ./cloudflare, ported from unemail's driver of the +// Cloudflare Email Service REST API, against a stubbed fetch. +const { test } = require('node:test'); +const assert = require('node:assert'); +const util = require('node:util'); +const { createMessage } = require('../../index.js'); +const { CloudflareTransport } = require('../../src/cloudflare/index.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); + +const ACCOUNT = '023e105f4ecef8ad9ca31a8372d0c353'; +const BASE = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT}`; +const transport = (config = {}) => + new CloudflareTransport({ + accountId: ACCOUNT, + apiToken: 'cf-secret', + retries: 0, + generateId: () => 'gen-1', + ...config, + }); +const message = (fields = {}) => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: ['bob@example.com'], + subject: 'Your order', + content: { text: 'Shipped', html: '

Shipped

' }, + ...fields, + }); +const sent = (result = {}) => + json({ + success: true, + errors: [], + messages: [], + result: { + message_id: 'cf-1', + delivered: ['bob@example.com'], + queued: [], + permanent_bounces: [], + ...result, + }, + }); + +test('Cloudflare: a message is the request of email/sending/send, in the REST shapes', async (t) => { + const requests = stubFetch(t, () => sent()); + const receipt = await transport().send( + message({ + to: ['bob@example.com', { name: 'Carol', address: 'carol@bücher.de' }], + cc: 'dave@example.com', + bcc: 'erin@example.com', + replyTo: { name: 'Desk', address: 'desk@example.com' }, + priority: 'high', + tags: ['orders'], + metadata: { orderId: '42' }, + list: { unsubscribe: 'https://example.com/u', oneClick: true }, + attachments: [ + { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new TextEncoder().encode('hi'), + }, + { + filename: 'logo.png', + contentType: 'image/png', + inline: true, + contentId: 'logo@example.com', + content: new Uint8Array([1]), + }, + ], + }), + ); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'cf-1', provider: 'cloudflare' }); + const [request] = requests; + assert.strictEqual(request.url, `${BASE}/email/sending/send`); + assert.strictEqual(request.headers.get('authorization'), 'Bearer cf-secret'); + assert.strictEqual(request.redirect, 'manual'); + // Changed in the port: unemail wrote { email, name }, replyTo and contentId. + assert.deepStrictEqual(request.json, { + from: { address: 'shop@example.com', name: 'Shop' }, + subject: 'Your order', + to: [{ address: 'bob@example.com' }, { address: 'carol@xn--bcher-kva.de', name: 'Carol' }], + cc: [{ address: 'dave@example.com' }], + bcc: [{ address: 'erin@example.com' }], + reply_to: { address: 'desk@example.com', name: 'Desk' }, + text: 'Shipped', + html: '

Shipped

', + headers: { + 'X-Priority': '1', + 'X-MSMail-Priority': 'High', + Importance: 'High', + 'List-Unsubscribe': '', + 'List-Unsubscribe-Post': 'List-Unsubscribe=One-Click', + 'X-Metadata-orderId': '42', + }, + attachments: [ + { content: 'aGk=', filename: 'r.txt', type: 'text/plain', disposition: 'attachment' }, + { + content: 'AQ==', + filename: 'logo.png', + type: 'image/png', + disposition: 'inline', + content_id: 'logo@example.com', + }, + ], + }); +}); + +test('Cloudflare: a plain message has no headers, and the account is one segment', async (t) => { + const requests = stubFetch(t, () => sent()); + await transport({ accountId: 'a/b?c', baseUrl: 'http://localhost:9000' }).send( + message({ to: [], bcc: 'erin@example.com', content: { text: 'plain' } }), + ); + const [request] = requests; + assert.strictEqual(request.url, 'http://localhost:9000/accounts/a%2Fb%3Fc/email/sending/send'); + for (const key of ['to', 'cc', 'reply_to', 'html', 'headers', 'attachments']) { + assert.strictEqual(request.json[key], undefined, key); + } +}); + +test('Cloudflare: bounced and suppressed recipients are listed, or fail when none went', async (t) => { + const answers = [ + sent({ delivered: [], queued: ['bob@example.com'], permanent_bounces: ['carol@example.com'] }), + sent({ suppressed_recipients: ['dave@example.com'], message_id: undefined }), + sent({ delivered: [], permanent_bounces: ['bob@example.com'] }), + sent({ delivered: [], suppressed_recipients: ['bob@example.com'] }), + sent({ delivered: [] }), + json({ success: false, errors: [{ code: 10200, message: 'Sender not verified' }] }), + json({ success: false }), + ]; + stubFetch(t, () => answers.shift()); + assert.deepStrictEqual(await transport().send(message()), { + successful: true, + messageId: 'cf-1', + provider: 'cloudflare', + bouncedRecipients: ['carol@example.com'], + }); + // Changed in the port: unemail made the id of the addresses and the clock. + assert.deepStrictEqual(await transport().send(message()), { + successful: true, + messageId: 'gen-1', + provider: 'cloudflare', + suppressedRecipients: ['dave@example.com'], + }); + for (let i = 0; i < 2; i++) { + const none = await transport().send(message()); + assert.strictEqual(none.successful, false); + assert.strictEqual(none.retryable, false); + assert.strictEqual(none.errors[0].category, 'rejected'); + assert.strictEqual(none.errors[0].code, 'cloudflare.no_recipient_accepted'); + assert.doesNotMatch(none.errorMessages[0], /example\.com/); + } + assert.strictEqual((await transport().send(message())).deliveryUnknown, true); + const refused = await transport().send(message()); + assert.strictEqual(refused.errorMessages[0], 'Cloudflare: Sender not verified'); + assert.strictEqual(refused.errors[0].category, 'validation'); + assert.strictEqual(refused.errors[0].code, 'cloudflare.10200'); + assert.strictEqual( + (await transport().send(message())).errorMessages[0], + 'Cloudflare: the send failed.', + ); +}); + +test('Cloudflare: the status of an error chooses its category, its code is kept', async (t) => { + const answers = [ + json({ success: false, errors: [{ code: 10101, message: 'Invalid API token' }] }, 401), + json( + { success: false, errors: [{ code: 10004, message: 'email.sending.error.throttled' }] }, + 429, + ), + json({ success: false, errors: [] }, 503), + ]; + stubFetch(t, () => answers.shift()); + const auth = await transport().send(message()); + assert.strictEqual(auth.errorMessages[0], 'Cloudflare: Invalid API token'); + assert.strictEqual(auth.errors[0].category, 'auth'); + assert.strictEqual(auth.errors[0].code, 'cloudflare.10101'); + const throttled = await transport().send(message()); + assert.strictEqual(throttled.errors[0].category, 'rate-limit'); + assert.strictEqual(throttled.retryable, true); + assert.strictEqual((await transport().send(message())).errors[0].category, 'service-unavailable'); +}); + +test('Cloudflare: its limits and a schedule are refused before a request', async (t) => { + const requests = stubFetch(t, () => sent()); + const many = []; + for (let i = 0; i < 26; i++) many.push(`r${i}@example.com`); + const refusals = [ + [message({ to: many, cc: many }), /at most 50 recipients/], + [message({ replyTo: ['a@example.com', 'b@example.com'] }), /one reply-to/], + [message({ scheduledAt: new Date() }), /Invalid scheduledAt: cloudflare/], + ]; + for (const [refused, error] of refusals) { + assert.match((await transport().send(refused)).errorMessages[0], error); + } + assert.strictEqual(requests.length, 0); +}); + +test('Cloudflare: sendRaw sends the MIME text through send_raw', async (t) => { + const requests = stubFetch(t, () => sent({ message_id: 'cf-raw' })); + const text = 'From: shop@example.com\r\nTo: bob@example.com\r\nSubject: Grüße\r\n\r\nHi\r\n'; + const receipt = await transport().sendRaw({ + envelope: { from: 'shop@example.com', to: ['bob@example.com', 'erin@bücher.de'] }, + content: new TextEncoder().encode(text), + }); + assert.strictEqual(receipt.messageId, 'cf-raw'); + assert.strictEqual(requests[0].url, `${BASE}/email/sending/send_raw`); + assert.deepStrictEqual(requests[0].json, { + from: 'shop@example.com', + recipients: ['bob@example.com', 'erin@xn--bcher-kva.de'], + mime_message: text, + }); + const bounce = await transport().sendRaw({ + envelope: { from: null, to: ['bob@example.com'] }, + content: new TextEncoder().encode(text), + }); + assert.strictEqual(bounce.errors[0].category, 'validation'); + assert.match(bounce.errorMessages[0], /needs a sender in the envelope/); + const latin1 = await transport().sendRaw({ + envelope: { from: 'shop@example.com', to: ['bob@example.com'] }, + content: new Uint8Array([...new TextEncoder().encode('Subject: x\r\n\r\n'), 0xe9, 13, 10]), + }); + assert.match(latin1.errorMessages[0], /Cloudflare takes a raw message only as UTF-8/); + stubFetch(t, () => json({ success: false, errors: [{ code: 10001 }] }, 400)); + const failed = await transport().sendRaw({ + envelope: { from: 'shop@example.com', to: ['bob@example.com'] }, + content: new TextEncoder().encode(text), + }); + assert.strictEqual(failed.errors[0].code, 'cloudflare.10001'); + assert.strictEqual(requests.length, 1); +}); + +test('Cloudflare: sendMany sends concurrently, in order', async (t) => { + let index = 0; + const requests = stubFetch(t, () => sent({ message_id: `cf-${index++}` })); + const receipts = []; + for await (const receipt of transport({ concurrency: 2 }).sendMany([ + message(), + message({ scheduledAt: new Date() }), + message(), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, true], + ); + assert.strictEqual(requests.length, 2); +}); + +test('Cloudflare: verify checks the token of the account, then of a user', async (t) => { + const active = () => json({ success: true, result: { id: 't', status: 'active' } }); + const answers = [ + active(), + json({ success: false, errors: [{ code: 1000, message: 'Invalid API Token' }] }, 401), + active(), + json({ success: false, errors: [{ code: 1000 }] }, 401), + json({ success: false, errors: [{ code: 1000, message: 'Invalid API Token' }] }, 401), + json({ success: true, result: { id: 't', status: 'disabled' } }), + json({ success: true, result: {} }), + json({ success: false }, 500), + ]; + const requests = stubFetch(t, () => answers.shift()); + await transport().verify(); + assert.strictEqual(requests[0].url, `${BASE}/tokens/verify`); + assert.strictEqual(requests[0].method, 'GET'); + assert.strictEqual(requests[0].headers.get('authorization'), 'Bearer cf-secret'); + await transport().verify(); + assert.strictEqual(requests[2].url, 'https://api.cloudflare.com/client/v4/user/tokens/verify'); + await assert.rejects(transport().verify(), /Cloudflare: Invalid API Token/); + await assert.rejects(transport().verify(), /the API token is disabled/); + await assert.rejects(transport().verify(), /the API token is not active/); + await assert.rejects(transport().verify(), (error) => error.statusCode === 500); + assert.strictEqual(requests.length, 8); +}); + +test('Cloudflare: the options are checked, and the token never shows', async () => { + assert.throws(() => new CloudflareTransport({ apiToken: 't' }), /needs accountId/); + assert.throws(() => new CloudflareTransport({ accountId: 'a' }), /needs apiToken/); + assert.throws(() => transport({ baseUrl: 'http://api.example.com' }), /baseUrl/); + const made = transport(); + assert.ok(!JSON.stringify(made).includes('cf-secret')); + assert.ok(!util.inspect(made, { depth: 5 }).includes('cf-secret')); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); + await assert.rejects( + made.sendRaw( + { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }, + { signal: controller.signal }, + ), + /stop/, + ); +}); From 4292dce7b3cba4243ea06904e0a116ee92ce8f66 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:10:48 +0300 Subject: [PATCH 027/143] fix(providers): a refusal before any request is a validation failure A converter refuses a message over a limit of its provider, or with a field the provider cannot express, with a RangeError before any request. Its receipt was classified by its words, as `unknown`; it is a non-retryable `validation` failure, as for a message a transport cannot schedule. `refusalFailure` (`internal/refusal.js`) makes it, for the shared `providerFailure` and for the ported providers' own. Co-Authored-By: Claude Opus 5.5 --- src/internal/api.js | 6 +++++- src/internal/refusal.js | 22 ++++++++++++++++++++++ tests/unit/api.test.js | 9 +++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 src/internal/refusal.js diff --git a/src/internal/api.js b/src/internal/api.js index cb5ad29..e5b3a87 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -11,6 +11,7 @@ const { canonicalHeaderName } = require('./headers.js'); const { convertAttachments, visitCustomHeaders, withCalendarAttachment } = require('./provider.js'); const { isDeliveryUnknown } = require('./delivery-unknown.js'); const { requestWithRetry } = require('./http.js'); +const { refusalFailure } = require('./refusal.js'); const { createLog } = require('./log.js'); const MAX_ERROR_TEXT = 500; @@ -182,7 +183,10 @@ const providerFailure = (provider, error) => { providerDetails: error.providerDetails, }); } - return createFailedReceipt(message, { provider, deliveryUnknown: isDeliveryUnknown(error) }); + return ( + refusalFailure(provider, error) ?? + createFailedReceipt(message, { provider, deliveryUnknown: isDeliveryUnknown(error) }) + ); }; /** diff --git a/src/internal/refusal.js b/src/internal/refusal.js new file mode 100644 index 0000000..3f9bf79 --- /dev/null +++ b/src/internal/refusal.js @@ -0,0 +1,22 @@ +// Not ported: the receipt of a message a provider cannot take as it is, which +// its converter refuses with a RangeError before any request: over a limit of +// the provider, or with a field it cannot express. + +const { createFailedReceipt } = require('../core/receipt.js'); + +/** + * The receipt of a refusal before any request, a non-retryable `validation` + * failure, as for a message a transport cannot schedule; or `undefined` for + * another error. + * + * @param provider The transport. + * @param error What was thrown. + * @returns The failed receipt, or `undefined`. + * @internal + */ +const refusalFailure = (provider, error) => + error instanceof RangeError + ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) + : undefined; + +module.exports = { refusalFailure }; diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js index a1342e5..05da554 100644 --- a/tests/unit/api.test.js +++ b/tests/unit/api.test.js @@ -132,3 +132,12 @@ test('API: the receipt of a failure carries what the error says', () => { assert.throws(() => checkCredential('', 'apiKey', 'DemoTransport'), TypeError); assert.strictEqual(checkCredential('k', 'apiKey', 'DemoTransport'), 'k'); }); + +test('API: a refusal of the converter is a non-retryable validation failure', () => { + const refused = providerFailure('x', new RangeError('X takes at most 5 tags.')); + assert.strictEqual(refused.errors[0].category, 'validation'); + assert.strictEqual(refused.retryable, false); + assert.strictEqual(refused.errorMessages[0], 'X takes at most 5 tags.'); + const other = providerFailure('x', new Error('boom')); + assert.notStrictEqual(other.errors[0].category, 'validation'); +}); From fbfb1065f223b4328577720e3e2052eb5014fccb Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:10:48 +0300 Subject: [PATCH 028/143] feat(resend): scheduledAt and metadata on Resend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scheduledAt` is Resend's `scheduled_at`, read from the message rather than from an option of the conversion that nothing passed. `metadata` goes in the tags, which Resend returns in its events, after `tag1`… of the message's tags; a key or value a tag cannot hold, or a key equal to one of those names, is refused before the request. A message with either goes alone, as the batch API takes neither. Tracking is set per domain, so a message's `tracking` is not sent. Co-Authored-By: Claude Opus 5.5 --- docs/transports/resend.md | 23 ++++++++ src/resend/message-converter.js | 45 +++++++++++----- src/resend/resend-transport.js | 4 ++ tests/unit/message-fields.test.js | 60 +++++++++++++++++---- tests/unit/resend-message-converter.test.js | 7 +-- 5 files changed, 113 insertions(+), 26 deletions(-) diff --git a/docs/transports/resend.md b/docs/transports/resend.md index e186b76..2ee634e 100644 --- a/docs/transports/resend.md +++ b/docs/transports/resend.md @@ -146,6 +146,29 @@ statistics, open rates, and engagement metrics for specific message categories. This helps you understand which types of emails perform best with your audience and optimize your email strategies accordingly. +## Scheduling, metadata and tracking + +`scheduledAt` is Resend's `scheduled_at`. `metadata` goes in its tags, next to +`tag1`, `tag2`… of the message's `tags`, since Resend returns tags in its events: +a key and its value must then be ASCII letters, digits, `_` or `-`, at most 256 +characters, and a key not one of those names; any other is refused before the +request. A message with either goes alone rather than through the batch API, +which takes neither. Resend sets tracking per domain, so a message's `tracking` +is not sent. + +```js +const receipt = await transport.send( + createMessage({ + from: 'shop@example.com', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + metadata: { orderId: '42' }, + }), +); +``` + ## Idempotency and reliability Resend provides built-in idempotency support to prevent duplicate email sends diff --git a/src/resend/message-converter.js b/src/resend/message-converter.js index 9c29a55..0f0ad22 100644 --- a/src/resend/message-converter.js +++ b/src/resend/message-converter.js @@ -5,7 +5,6 @@ const { generateId } = require('../core/id.js'); const { formatMailbox: formatAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); -const { refuseScheduledAt } = require('../internal/schedule.js'); const { convertAttachments, formatAddresses, @@ -14,6 +13,34 @@ const { } = require('../internal/provider.js'); const PRIORITIES = { high: '1', normal: '3', low: '5' }; +// The name and value of a Resend tag: "ASCII letters, numbers, underscores, +// or dashes", at most 256 characters. +const TAG_TEXT = /^[A-Za-z0-9_-]{1,256}$/; + +// The tags of a message, `tag1`… for its tags and the keys of its metadata, +// which Resend returns in its events. +const tagsOf = (message) => { + const tags = []; + const names = new Set(); + for (let index = 0; index < message.tags.length; index++) { + const name = `tag${index + 1}`; + names.add(name); + tags.push({ name, value: message.tags[index] }); + } + if (message.metadata !== undefined) { + for (const key in message.metadata) { + const value = message.metadata[key]; + if (!TAG_TEXT.test(key) || !TAG_TEXT.test(value) || names.has(key)) { + throw new RangeError( + `Resend takes metadata as tags, whose name and value are ASCII letters, digits, ` + + `_ or -, and a name other than those of the tags: ${key}`, + ); + } + tags.push({ name: key, value }); + } + } + return tags; +}; /** * Converts a Upyo Message to Resend API JSON format. @@ -26,6 +53,7 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * @param options - Optional conversion options * @returns JSON object ready for Resend API submission * @throws {TypeError} If the message carries an invalid message identifier. + * @throws {RangeError} If a key or value of its metadata cannot be a tag. * * @example * ```typescript @@ -38,7 +66,6 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * ``` */ const convertMessage = async (message, _config, options = {}) => { - refuseScheduledAt(message, 'resend'); const attachments = withCalendarAttachment(message); const emailData = { @@ -86,14 +113,8 @@ const convertMessage = async (message, _config, options = {}) => { } // Tags - Resend uses different format than Upyo - if (message.tags.length > 0) { - // Resend requires name-value pairs. - const tags = []; - for (let index = 0; index < message.tags.length; index++) { - tags.push({ name: `tag${index + 1}`, value: message.tags[index] }); - } - emailData.tags = tags; - } + const tags = tagsOf(message); + if (tags.length > 0) emailData.tags = tags; let headers; if (message.priority !== 'normal') headers = { 'X-Priority': PRIORITIES[message.priority] }; @@ -104,8 +125,8 @@ const convertMessage = async (message, _config, options = {}) => { if (headers !== undefined) emailData.headers = headers; // Scheduled sending - if (options.scheduledAt) { - emailData.scheduled_at = options.scheduledAt.toISOString(); + if (message.scheduledAt !== undefined) { + emailData.scheduled_at = message.scheduledAt.toISOString(); } return emailData; diff --git a/src/resend/resend-transport.js b/src/resend/resend-transport.js index 4b39eb2..7f11d66 100644 --- a/src/resend/resend-transport.js +++ b/src/resend/resend-transport.js @@ -13,6 +13,7 @@ const { const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); +const { refusalFailure } = require('../internal/refusal.js'); /** * Resend transport implementation for sending emails via Resend API. @@ -330,6 +331,7 @@ class ResendTransport { if ( message.attachments.length > 0 || message.tags.length > 0 || + message.metadata !== undefined || message.calendar != null || message.scheduledAt != null ) { @@ -341,6 +343,8 @@ class ResendTransport { } const createResendFailure = (message, error) => { + const refusal = refusalFailure('resend', error); + if (refusal !== undefined) return refusal; if (error instanceof ResendApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index d277967..a8e9d4d 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -106,7 +106,6 @@ const providers = { }), sendgrid: () => new SendGridTransport({ apiKey: 'SG.key', retries: 0 }), mailgun: () => new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com', retries: 0 }), - resend: () => new ResendTransport({ apiKey: 'key', retries: 0 }), mailtrap: () => new MailtrapTransport({ apiToken: 'token', retries: 0 }), maileroo: () => new MailerooTransport({ apiKey: 'key', retries: 0 }), lettermint: () => new LettermintTransport({ apiToken: 'token', retries: 0 }), @@ -130,16 +129,13 @@ for (const [name, make] of Object.entries(providers)) { }); } -test('Fields: Resend sends the others of a chunk in one batch request', async (t) => { +test('Fields: Resend sends a scheduled message alone, with scheduled_at', async (t) => { const bodies = []; t.mock.method(globalThis, 'fetch', async (input, init) => { bodies.push({ url: String(input), body: JSON.parse(init.body) }); - if (String(input).endsWith('/emails/batch')) { - return Response.json({ data: [{ id: 'r1' }, { id: 'r2' }] }); - } - return Response.json({ id: 'single' }); + return Response.json({ id: `r${bodies.length}` }); }); - const transport = providers.resend(); + const transport = new ResendTransport({ apiKey: 'key', retries: 0 }); const receipts = []; for await (const receipt of transport.sendMany([ createMessage(base), @@ -148,13 +144,55 @@ test('Fields: Resend sends the others of a chunk in one batch request', async (t ])) { receipts.push(receipt); } - // A scheduled message keeps the chunk off the batch API, so each goes alone. + // The batch API takes no scheduled message, so each of the chunk goes alone. assert.deepStrictEqual( receipts.map((receipt) => receipt.successful), - [true, false, true], + [true, true, true], ); - assertRefused(receipts[1]); - assert.strictEqual(bodies.length, 2); + assert.deepStrictEqual( + bodies.map((body) => body.url), + [ + 'https://api.resend.com/emails', + 'https://api.resend.com/emails', + 'https://api.resend.com/emails', + ], + ); + assert.strictEqual(bodies[1].body.scheduled_at, scheduled().scheduledAt.toISOString()); + assert.strictEqual(bodies[0].body.scheduled_at, undefined); +}); + +test('Fields: Resend sends metadata as tags, alone, and refuses what a tag cannot hold', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (input, init) => { + bodies.push({ url: String(input), body: JSON.parse(init.body) }); + return Response.json({ id: `r${bodies.length}` }); + }); + const transport = new ResendTransport({ apiKey: 'key', retries: 0 }); + const receipts = []; + for await (const receipt of transport.sendMany([ + createMessage({ ...base, tags: ['welcome'], metadata: { orderId: '42', tenant: 'acme-1' } }), + createMessage({ ...base, metadata: { 'order.id': '42' } }), + createMessage({ ...base, metadata: { note: 'two words' } }), + createMessage({ ...base, tags: ['a'], metadata: { tag1: 'x' } }), + ])) { + receipts.push(receipt); + } + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, false, false, false], + ); + assert.strictEqual(bodies.length, 1); + assert.strictEqual(bodies[0].url, 'https://api.resend.com/emails'); + assert.deepStrictEqual(bodies[0].body.tags, [ + { name: 'tag1', value: 'welcome' }, + { name: 'orderId', value: '42' }, + { name: 'tenant', value: 'acme-1' }, + ]); + for (const receipt of receipts.slice(1)) { + assert.match(receipt.errorMessages[0], /Resend takes metadata as tags/); + assert.strictEqual(receipt.errors[0].category, 'validation'); + assert.strictEqual(receipt.retryable, false); + } }); const CORE = 'urn:ietf:params:jmap:core'; diff --git a/tests/unit/resend-message-converter.test.js b/tests/unit/resend-message-converter.test.js index 3f26916..f287a2c 100644 --- a/tests/unit/resend-message-converter.test.js +++ b/tests/unit/resend-message-converter.test.js @@ -248,6 +248,8 @@ describe('Message Converter', () => { assert.equal(result.headers?.['x-another'], 'another-value'); }); + // Changed after the port: the time is the message's `scheduledAt`, not an + // option of the conversion. it('should handle scheduled sending', async () => { const scheduledDate = new Date('2024-12-01T10:00:00Z'); const message = { @@ -262,11 +264,10 @@ describe('Message Converter', () => { priority: 'normal', tags: [], headers: new Headers(), + scheduledAt: scheduledDate, }; - const result = await convertMessage(message, config, { - scheduledAt: scheduledDate, - }); + const result = await convertMessage(message, config); assert.equal(result.scheduled_at, '2024-12-01T10:00:00.000Z'); }); From df928d264f53d6cdff574c010d7e5ec676ae55b9 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:11:58 +0300 Subject: [PATCH 029/143] feat(sendgrid): scheduledAt, tracking and metadata on SendGrid `scheduledAt` is `send_at`, at most 72 hours ahead by an injected clock (`now`), a later one refused before the request; a message's `tracking` sets its open and click tracking over the configuration's; `metadata` is `custom_args`, which SendGrid returns in its events. Co-Authored-By: Claude Opus 5.5 --- docs/transports/sendgrid.md | 23 +++++++++++++++++ src/sendgrid/config.js | 1 + src/sendgrid/index.d.ts | 6 +++++ src/sendgrid/message-converter.js | 34 ++++++++++++++++++------- src/sendgrid/sendgrid-transport.js | 3 +++ tests/unit/message-fields.test.js | 40 +++++++++++++++++++++++++++++- 6 files changed, 97 insertions(+), 10 deletions(-) diff --git a/docs/transports/sendgrid.md b/docs/transports/sendgrid.md index 38a83d3..ca46faa 100644 --- a/docs/transports/sendgrid.md +++ b/docs/transports/sendgrid.md @@ -207,6 +207,29 @@ while retry configuration determines how many times to retry failed requests. The transport uses exponential backoff for retries, reducing load on SendGrid's servers during temporary outages and improving overall reliability. + +## Scheduling, metadata and tracking + +`scheduledAt` is SendGrid's `send_at`, at most 72 hours ahead (the `now` option +is the clock it is measured by); a later one is refused before the request. +`metadata` is `custom_args`, which SendGrid returns in its events. A message's +`tracking` sets its open and click tracking over `openTracking` and +`clickTracking` of the configuration. + +```js +const receipt = await transport.send( + createMessage({ + from: 'shop@example.com', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + tracking: { opens: false }, + metadata: { orderId: '42' }, + }), +); +``` + ## Verifying the configuration `verify()` reads the scopes of the API key (`GET /v3/scopes`) and checks that one diff --git a/src/sendgrid/config.js b/src/sendgrid/config.js index 659087b..40038c1 100644 --- a/src/sendgrid/config.js +++ b/src/sendgrid/config.js @@ -32,6 +32,7 @@ const createSendGridConfig = (config) => { openTracking: config.openTracking ?? true, subscriptionTracking: config.subscriptionTracking ?? false, googleAnalytics: config.googleAnalytics ?? false, + now: config.now ?? Date.now, }; checkRequestOptions(resolved.timeout, resolved.retries); checkConcurrency(resolved.concurrency); diff --git a/src/sendgrid/index.d.ts b/src/sendgrid/index.d.ts index cc8095d..a2483f1 100644 --- a/src/sendgrid/index.d.ts +++ b/src/sendgrid/index.d.ts @@ -97,6 +97,12 @@ export interface SendGridConfig extends TransportDependencies { * @default false */ readonly googleAnalytics?: boolean; + /** + * The clock, in milliseconds, for the 72 hours of `scheduledAt`. + * + * @default Date.now + */ + readonly now?: () => number; } /** * Resolved SendGrid configuration with all optional fields filled with default values. diff --git a/src/sendgrid/message-converter.js b/src/sendgrid/message-converter.js index 9308f71..ceb77c3 100644 --- a/src/sendgrid/message-converter.js +++ b/src/sendgrid/message-converter.js @@ -4,7 +4,6 @@ const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); -const { refuseScheduledAt } = require('../internal/schedule.js'); const { collectCustomHeaders, convertAttachments, @@ -12,6 +11,9 @@ const { withCalendarAttachment, } = require('../internal/provider.js'); +// `send_at` may be at most 72 hours ahead. +const SCHEDULE_WINDOW_MS = 72 * 60 * 60 * 1000; + /** * Converts a Upyo Message to SendGrid API JSON format. * @@ -22,6 +24,7 @@ const { * @param config - The resolved SendGrid configuration * @returns Promise that resolves to a SendGrid mail object * @throws {TypeError} If the message carries an invalid message identifier. + * @throws {RangeError} If it is scheduled more than 72 hours ahead. * * @example * ```typescript @@ -30,7 +33,12 @@ const { * ``` */ const convertMessage = async (message, config, signal) => { - refuseScheduledAt(message, 'sendgrid'); + if ( + message.scheduledAt !== undefined && + message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS + ) { + throw new RangeError('SendGrid schedules a message at most 72 hours ahead.'); + } const attachments = withCalendarAttachment(message); const personalization = { to: formatAddresses(message.recipients, formatAddress), @@ -72,8 +80,13 @@ const convertMessage = async (message, config, signal) => { sendGridMail.attachments = await convertAttachments(attachments, convertAttachment, signal); } - const trackingSettings = trackingSettingsOf(config); + const trackingSettings = trackingSettingsOf(config, message.tracking); if (trackingSettings !== undefined) sendGridMail.tracking_settings = trackingSettings; + if (message.scheduledAt !== undefined) { + sendGridMail.send_at = Math.floor(message.scheduledAt.getTime() / 1000); + } + // SendGrid returns custom arguments in its events. + if (message.metadata !== undefined) sendGridMail.custom_args = { ...message.metadata }; return sendGridMail; }; @@ -94,16 +107,19 @@ const contentOf = (message) => { return content; }; -// The tracking settings the configuration sets, or `undefined` for none. -const trackingSettingsOf = (config) => { +// The tracking settings of the message over those of the configuration, or +// `undefined` for none. +const trackingSettingsOf = (config, tracking) => { let settings; - if (config.clickTracking !== undefined) { + const clicks = tracking?.clicks ?? config.clickTracking; + const opens = tracking?.opens ?? config.openTracking; + if (clicks !== undefined) { settings ??= {}; - settings.click_tracking = { enable: config.clickTracking, enable_text: config.clickTracking }; + settings.click_tracking = { enable: clicks, enable_text: clicks }; } - if (config.openTracking !== undefined) { + if (opens !== undefined) { settings ??= {}; - settings.open_tracking = { enable: config.openTracking }; + settings.open_tracking = { enable: opens }; } if (config.subscriptionTracking !== undefined) { settings ??= {}; diff --git a/src/sendgrid/sendgrid-transport.js b/src/sendgrid/sendgrid-transport.js index 9287046..7d05acd 100644 --- a/src/sendgrid/sendgrid-transport.js +++ b/src/sendgrid/sendgrid-transport.js @@ -7,6 +7,7 @@ const { createSendGridConfig } = require('./config.js'); const { verifyByRequest } = require('../internal/verify.js'); const { SendGridApiError, SendGridHttpClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { refusalFailure } = require('../internal/refusal.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); @@ -258,6 +259,8 @@ class SendGridTransport { } const createSendGridFailure = (message, error) => { + const refusal = refusalFailure('sendgrid', error); + if (refusal !== undefined) return refusal; if (error instanceof SendGridApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index a8e9d4d..b57d546 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -104,7 +104,6 @@ const providers = { authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, retries: 0, }), - sendgrid: () => new SendGridTransport({ apiKey: 'SG.key', retries: 0 }), mailgun: () => new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com', retries: 0 }), mailtrap: () => new MailtrapTransport({ apiToken: 'token', retries: 0 }), maileroo: () => new MailerooTransport({ apiKey: 'key', retries: 0 }), @@ -195,6 +194,45 @@ test('Fields: Resend sends metadata as tags, alone, and refuses what a tag canno } }); +test('Fields: SendGrid sends send_at, the tracking of the message and custom_args', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + bodies.push(JSON.parse(init.body)); + return new Response(null, { status: 202, headers: { 'x-message-id': `sg${bodies.length}` } }); + }); + const now = () => Date.parse('2026-10-17T12:00:00Z'); + const transport = new SendGridTransport({ apiKey: 'SG.key', retries: 0, now }); + const sent = await transport.send( + createMessage({ + ...base, + scheduledAt: tomorrow, + tracking: { opens: false }, + metadata: { orderId: '42' }, + }), + ); + assert.strictEqual(sent.successful, true); + assert.strictEqual(bodies[0].send_at, tomorrow.getTime() / 1000); + assert.deepStrictEqual(bodies[0].custom_args, { orderId: '42' }); + assert.deepStrictEqual(bodies[0].tracking_settings.open_tracking, { enable: false }); + assert.deepStrictEqual(bodies[0].tracking_settings.click_tracking, { + enable: true, + enable_text: true, + }); + await transport.send(createMessage({ ...base, tracking: { clicks: false } })); + assert.deepStrictEqual(bodies[1].tracking_settings.click_tracking, { + enable: false, + enable_text: false, + }); + assert.strictEqual(bodies[1].send_at, undefined); + assert.strictEqual(bodies[1].custom_args, undefined); + const late = await transport.send( + createMessage({ ...base, scheduledAt: new Date(now() + 73 * 3600_000) }), + ); + assert.match(late.errorMessages[0], /at most 72 hours ahead/); + assert.strictEqual(late.errors[0].category, 'validation'); + assert.strictEqual(bodies.length, 2); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From 80e93296c6a35f66a53987e42dcb94628c98277d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:13:24 +0300 Subject: [PATCH 030/143] feat(mailgun): scheduledAt, tracking and metadata on Mailgun `scheduledAt` is `o:deliverytime`, an RFC 2822 date; a message's `tracking` sets `o:tracking-opens` and `o:tracking-clicks` over the configuration's; `metadata` goes in `v:` variables, which Mailgun returns in its events. Co-Authored-By: Claude Opus 5.5 --- docs/transports/mailgun.md | 22 ++++++++++++++++++++ src/mailgun/message-converter.js | 24 +++++++++++++++------- tests/unit/message-fields.test.js | 34 ++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/transports/mailgun.md b/docs/transports/mailgun.md index 68b996a..c3b3543 100644 --- a/docs/transports/mailgun.md +++ b/docs/transports/mailgun.md @@ -150,6 +150,28 @@ Tags appear in Mailgun's analytics dashboard, allowing you to track open rates, click rates, and delivery statistics for specific message categories. This helps you understand which types of emails perform best with your audience. +## Scheduling, metadata and tracking + +`scheduledAt` is Mailgun's `o:deliverytime`; how far ahead depends on the plan +(three days, or seven), and Mailgun refuses a later one. `metadata` goes in +`v:` variables, which Mailgun returns in its events as user variables. A +message's `tracking` sets `o:tracking-opens` and `o:tracking-clicks` over +`openTracking` and `clickTracking` of the configuration. + +```js +const receipt = await transport.send( + createMessage({ + from: 'shop@example.com', + to: 'user@example.com', + subject: 'Your order has shipped', + content: { html: '

On its way.

' }, + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + tracking: { clicks: false }, + metadata: { orderId: '42' }, + }), +); +``` + ## Raw MIME `sendRaw()` sends serialized MIME through the `messages.mime` endpoint, to the diff --git a/src/mailgun/message-converter.js b/src/mailgun/message-converter.js index 60794a0..5236ffa 100644 --- a/src/mailgun/message-converter.js +++ b/src/mailgun/message-converter.js @@ -4,7 +4,6 @@ const { formatQuotedMailbox: formatAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { visitCustomHeaders, withCalendarAttachment } = require('../internal/provider.js'); -const { refuseScheduledAt } = require('../internal/schedule.js'); const PRIORITIES = { high: '1', normal: '3', low: '5' }; @@ -26,7 +25,6 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * ``` */ const convertMessage = async (message, config, signal) => { - refuseScheduledAt(message, 'mailgun'); const formData = new FormData(); const attachments = withCalendarAttachment(message); @@ -89,17 +87,29 @@ const convertMessage = async (message, config, signal) => { await appendAttachment(formData, attachment, signal); } - // Tracking options + // Tracking options, those of the message over those of the configuration if (config.tracking !== undefined) { formData.append('o:tracking', config.tracking ? 'yes' : 'no'); } - if (config.clickTracking !== undefined) { - formData.append('o:tracking-clicks', config.clickTracking ? 'yes' : 'no'); + const clicks = message.tracking?.clicks ?? config.clickTracking; + if (clicks !== undefined) { + formData.append('o:tracking-clicks', clicks ? 'yes' : 'no'); } - if (config.openTracking !== undefined) { - formData.append('o:tracking-opens', config.openTracking ? 'yes' : 'no'); + const opens = message.tracking?.opens ?? config.openTracking; + if (opens !== undefined) { + formData.append('o:tracking-opens', opens ? 'yes' : 'no'); + } + + // Scheduled delivery, as an RFC 2822 date + if (message.scheduledAt !== undefined) { + formData.append('o:deliverytime', message.scheduledAt.toUTCString()); + } + + // Metadata, as variables Mailgun returns in its events + if (message.metadata !== undefined) { + for (const key in message.metadata) formData.append(`v:${key}`, message.metadata[key]); } return formData; diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index b57d546..79c400d 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -104,7 +104,6 @@ const providers = { authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, retries: 0, }), - mailgun: () => new MailgunTransport({ apiKey: 'key', domain: 'mg.example.com', retries: 0 }), mailtrap: () => new MailtrapTransport({ apiToken: 'token', retries: 0 }), maileroo: () => new MailerooTransport({ apiKey: 'key', retries: 0 }), lettermint: () => new LettermintTransport({ apiToken: 'token', retries: 0 }), @@ -233,6 +232,39 @@ test('Fields: SendGrid sends send_at, the tracking of the message and custom_arg assert.strictEqual(bodies.length, 2); }); +test('Fields: Mailgun sends o:deliverytime, the tracking of the message and v: variables', async (t) => { + const forms = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + forms.push(init.body); + return Response.json({ id: ``, message: 'Queued' }); + }); + const transport = new MailgunTransport({ + apiKey: 'key', + domain: 'mg.example.com', + retries: 0, + clickTracking: true, + }); + const sent = await transport.send( + createMessage({ + ...base, + scheduledAt: tomorrow, + tracking: { opens: false, clicks: false }, + metadata: { orderId: '42', tenant: 'acme' }, + }), + ); + assert.strictEqual(sent.successful, true); + const [form] = forms; + assert.strictEqual(form.get('o:deliverytime'), 'Sun, 18 Oct 2026 09:00:00 GMT'); + assert.strictEqual(form.get('o:tracking-opens'), 'no'); + assert.strictEqual(form.get('o:tracking-clicks'), 'no'); + assert.strictEqual(form.get('v:orderId'), '42'); + assert.strictEqual(form.get('v:tenant'), 'acme'); + await transport.send(createMessage(base)); + assert.strictEqual(forms[1].get('o:tracking-clicks'), 'yes'); + assert.strictEqual(forms[1].get('o:tracking-opens'), 'yes'); + assert.strictEqual(forms[1].get('o:deliverytime'), null); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From 49b9d6cd236cbad5e527e272204355edd0d601b6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:14:06 +0300 Subject: [PATCH 031/143] feat(mailtrap): the metadata of the message on Mailtrap A message's `metadata` goes in `custom_variables`, which Mailtrap returns in its events, its keys over those of the configuration's `metadata`. Mailtrap cannot schedule a message, which stays refused, and sets tracking per domain. Co-Authored-By: Claude Opus 5.5 --- docs/transports/mailtrap.md | 5 +++++ src/mailtrap/message-converter.js | 9 +++++++-- tests/unit/message-fields.test.js | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/transports/mailtrap.md b/docs/transports/mailtrap.md index 9df7f3e..14200ec 100644 --- a/docs/transports/mailtrap.md +++ b/docs/transports/mailtrap.md @@ -121,6 +121,11 @@ const message = createMessage({ }); ``` +A message's own `metadata` goes in `custom_variables` too, its keys over those of +the configuration. Mailtrap cannot schedule a message, so `scheduledAt` is +refused as `validation`, and it sets tracking per domain, so a message's +`tracking` is not sent. + ## Batch sending Use `sendMany()` to send multiple messages. The transport batches up to 500 diff --git a/src/mailtrap/message-converter.js b/src/mailtrap/message-converter.js index 28a5f85..dbfb25f 100644 --- a/src/mailtrap/message-converter.js +++ b/src/mailtrap/message-converter.js @@ -78,7 +78,7 @@ const convertMessage = async (message, config, signal) => { throw new RangeError('Mailtrap requires at least one of text or HTML content.'); } - const customVariables = buildCustomVariables(message.tags, config.metadata); + const customVariables = buildCustomVariables(message.tags, config.metadata, message.metadata); if (Object.keys(customVariables).length > 0) { emailData.custom_variables = customVariables; } @@ -104,7 +104,9 @@ const resolveCategory = (tags, fallback) => { return tags[0] ?? fallback; }; -const buildCustomVariables = (tags, metadata) => { +// The metadata of the configuration, that of the message over it, then the +// tags after the first. +const buildCustomVariables = (tags, metadata, messageMetadata) => { const customVariables = {}; if (metadata != null) { @@ -112,6 +114,9 @@ const buildCustomVariables = (tags, metadata) => { customVariables[key] = value; } } + if (messageMetadata !== undefined) { + for (const key in messageMetadata) customVariables[key] = messageMetadata[key]; + } for (const tag of tags.slice(1)) { customVariables[`tag_${tag}`] = tag; diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index 79c400d..e28d6ff 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -265,6 +265,33 @@ test('Fields: Mailgun sends o:deliverytime, the tracking of the message and v: v assert.strictEqual(forms[1].get('o:deliverytime'), null); }); +test('Fields: Mailtrap sends the metadata of the message over that of the configuration', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + bodies.push(JSON.parse(init.body)); + return Response.json({ success: true, message_ids: [`mt${bodies.length}`] }); + }); + const transport = new MailtrapTransport({ + apiToken: 'token', + retries: 0, + metadata: { tenant: 'acme', stream: 'orders' }, + }); + const sent = await transport.send( + createMessage({ + ...base, + tags: ['first', 'second'], + metadata: { tenant: 'globex', orderId: '42' }, + }), + ); + assert.strictEqual(sent.successful, true); + assert.deepStrictEqual(bodies[0].custom_variables, { + tenant: 'globex', + stream: 'orders', + orderId: '42', + tag_second: 'second', + }); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From c55a85176d25d61114b7963c568de82b02316a64 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:15:05 +0300 Subject: [PATCH 032/143] feat(ses): the metadata of the message as SES tags A message's `metadata` goes in the tags of the request, which SES publishes with its events, after the `category` tags and over a default tag of the same name; a key or value a tag cannot hold, or a key `category` or `priority`, is refused before the request. SES cannot schedule a message, which stays refused, and sets tracking by the configuration set. The budget is 8 KB: the check of the tags and the refusal add 0.1 KB over 7.5. Co-Authored-By: Claude Opus 5.5 --- docs/transports/ses.md | 8 ++++++++ scripts/size-budgets.json | 2 +- src/ses/message-converter.js | 28 +++++++++++++++++++++++++++- src/ses/ses-transport.js | 3 +++ tests/unit/message-fields.test.js | 29 +++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/transports/ses.md b/docs/transports/ses.md index efb0f7d..9bcb02b 100644 --- a/docs/transports/ses.md +++ b/docs/transports/ses.md @@ -340,6 +340,14 @@ instructions and available features. [SES Configuration Sets documentation]: https://docs.aws.amazon.com/ses/latest/dg/using-configuration-sets.html +A message's `metadata` goes in its tags, which SES publishes with its events, +after the `category` tags of its `tags` and over a `defaultTags` entry of the +same name. A key and its value must then be ASCII letters, digits, `_` or `-`, +fewer than 256 characters, and a key neither `category` nor `priority`; any +other is refused before the request. Tracking is set by the configuration set, +so a message's `tracking` is not sent, and SES cannot schedule a message, so +`scheduledAt` is refused as `validation`. + ## Advanced configuration and reliability The SES transport includes comprehensive configuration options for timeout diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 96cb4d5..0e84083 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -5,7 +5,7 @@ "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, - "./ses": 7.5, + "./ses": 8, "./resend": 6.5, "./sendgrid": 6.5, "./mailgun": 6.5, diff --git a/src/ses/message-converter.js b/src/ses/message-converter.js index 5aef5a0..99e944a 100644 --- a/src/ses/message-converter.js +++ b/src/ses/message-converter.js @@ -28,6 +28,26 @@ const SES_HEADERS = new Set([ 'subject', 'to', ]); +// The name and value of an SES tag: "only ASCII letters (a–z, A–Z), numbers +// (0–9), underscores (_), or dashes (-)", fewer than 256 characters. +const TAG_TEXT = /^[A-Za-z0-9_-]{1,255}$/; +// The tags this converter writes of its own. +const OWN_TAGS = new Set(['category', 'priority']); + +// The metadata of a message as tags, which SES returns in its events. +const metadataTagsOf = (metadata, tags) => { + for (const key in metadata) { + const value = metadata[key]; + if (!TAG_TEXT.test(key) || !TAG_TEXT.test(value) || OWN_TAGS.has(key)) { + throw new RangeError( + 'SES takes metadata as tags, whose name and value are ASCII letters, digits, _ or -, ' + + `and a name other than category and priority: ${key}`, + ); + } + tags.push({ Name: key, Value: value }); + } +}; + // The most fields SES takes in Headers, and the longest field it takes. const MAX_SES_HEADERS = 15; const MAX_SES_HEADER_LENGTH = 996; @@ -76,8 +96,14 @@ const convertMessage = async (message, config, signal) => { tags.push({ Name: 'priority', Value: message.priority }); } + if (message.metadata !== undefined) metadataTagsOf(message.metadata, tags); + + // A default tag gives way to a key of the metadata. for (const name in config.defaultTags) { - if (Object.hasOwn(config.defaultTags, name)) { + if ( + Object.hasOwn(config.defaultTags, name) && + (message.metadata === undefined || !Object.hasOwn(message.metadata, name)) + ) { tags.push({ Name: name, Value: config.defaultTags[name] }); } } diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 01d4ad3..fb48bbe 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -10,6 +10,7 @@ const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); +const { refusalFailure } = require('../internal/refusal.js'); const { verifyByRequest } = require('../internal/verify.js'); /** @@ -278,6 +279,8 @@ const checkAuthentication = (authentication) => { }; const createSesFailure = (message, error) => { + const refusal = refusalFailure('ses', error); + if (refusal !== undefined) return refusal; if (error instanceof SesApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index e28d6ff..48b5a31 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -292,6 +292,35 @@ test('Fields: Mailtrap sends the metadata of the message over that of the config }); }); +test('Fields: SES sends metadata as tags, over the default ones, and refuses what a tag cannot hold', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + bodies.push(JSON.parse(init.body)); + return Response.json({ MessageId: `ses${bodies.length}` }); + }); + const transport = new SesTransport({ + authentication: { type: 'credentials', accessKeyId: 'AKID', secretAccessKey: 'secret' }, + retries: 0, + defaultTags: { tenant: 'default', app: 'shop' }, + }); + const sent = await transport.send( + createMessage({ ...base, tags: ['welcome'], metadata: { orderId: '42', tenant: 'acme' } }), + ); + assert.strictEqual(sent.successful, true); + assert.deepStrictEqual(bodies[0].Tags, [ + { Name: 'category', Value: 'welcome' }, + { Name: 'orderId', Value: '42' }, + { Name: 'tenant', Value: 'acme' }, + { Name: 'app', Value: 'shop' }, + ]); + for (const metadata of [{ 'order.id': '1' }, { note: 'two words' }, { category: 'x' }]) { + const refused = await transport.send(createMessage({ ...base, metadata })); + assert.match(refused.errorMessages[0], /SES takes metadata as tags/); + assert.strictEqual(refused.errors[0].category, 'validation'); + } + assert.strictEqual(bodies.length, 1); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From f5d5752d4e6f2e5734205892deb56fbaf5d5b875 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:18:29 +0300 Subject: [PATCH 033/143] feat(maileroo): scheduledAt, tracking and metadata on Maileroo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scheduledAt` is `scheduled_at`; a message's `tracking` sets the one switch Maileroo has for opens and clicks over the configuration's, one asking for opens without clicks, or the reverse, refused (the rule of `./azure`, now `internal/tracking.js`); `metadata` goes in the tags, over the configuration's, a key `tag1`, `tag2`… or a value over 768 characters refused before the request. A converter's refusal is now a `RefusalError`, which alone makes a `validation` receipt: any RangeError did, so the Maileroo client's refusal of a 2xx body over 4 MiB lost its unknown delivery. The budget of `./maileroo` is 6.5 KB. Co-Authored-By: Claude Opus 5.5 --- docs/transports/maileroo.md | 6 ++++ scripts/size-budgets.json | 2 +- src/azure/message-converter.js | 12 +++----- src/brevo/message-converter.js | 5 +-- src/cloudflare/message-converter.js | 5 +-- src/internal/refusal.js | 16 +++++++--- src/internal/tracking.js | 25 +++++++++++++++ src/maileroo/maileroo-transport.js | 3 ++ src/maileroo/message-converter.js | 35 ++++++++++++++++++--- src/mailersend/message-converter.js | 9 +++--- src/mailjet/message-converter.js | 11 ++++--- src/postmark/message-converter.js | 3 +- src/resend/message-converter.js | 3 +- src/scaleway/message-converter.js | 5 +-- src/sendgrid/message-converter.js | 3 +- src/ses/message-converter.js | 7 +++-- src/smtp2go/message-converter.js | 5 +-- src/sparkpost/message-converter.js | 3 +- src/zeptomail/message-converter.js | 9 +++--- tests/unit/api.test.js | 6 +++- tests/unit/message-fields.test.js | 47 ++++++++++++++++++++++++++++- 21 files changed, 173 insertions(+), 47 deletions(-) create mode 100644 src/internal/tracking.js diff --git a/docs/transports/maileroo.md b/docs/transports/maileroo.md index 4f6601b..94c0693 100644 --- a/docs/transports/maileroo.md +++ b/docs/transports/maileroo.md @@ -91,6 +91,12 @@ await transport.send(message); `Message.tags` values are added as generated Maileroo tag keys such as `tag1` and `tag2`, alongside any default tags configured on the transport. +A message's `metadata` goes in the tags too, its keys over those of the +transport; a key may not be `tag1`, `tag2`… and a value has at most 768 +characters. Maileroo has one switch for open and click tracking: a message's +`tracking` sets it over the transport's `tracking`, and one that asks for opens +without clicks, or the reverse, is refused. `scheduledAt` is `scheduled_at`. + ## Sending multiple emails The `sendMany()` method sends messages through the single-message Maileroo diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 0e84083..a922913 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -10,7 +10,7 @@ "./sendgrid": 6.5, "./mailgun": 6.5, "./mailtrap": 6.5, - "./maileroo": 6, + "./maileroo": 6.5, "./lettermint": 6, "./plunk": 5.5, "./postmark": 7, diff --git a/src/azure/message-converter.js b/src/azure/message-converter.js index cd51a2b..fea0e34 100644 --- a/src/azure/message-converter.js +++ b/src/azure/message-converter.js @@ -9,6 +9,8 @@ const { toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { trackingSwitchOf } = require('../internal/tracking.js'); +const { RefusalError } = require('../internal/refusal.js'); /** Azure takes a request of at most 10 MB, its attachments included. */ const MAX_REQUEST_BYTES = 10 * 1024 * 1024; @@ -28,13 +30,7 @@ const addressesOf = (addresses) => { // `userEngagementTrackingDisabled` turns opens and clicks off together, and // can only turn off what the resource has on. const trackingDisabledOf = (message, config) => { - const tracking = message.tracking; - if (tracking === undefined) return config.userEngagementTrackingDisabled; - const { opens, clicks } = tracking; - if (opens !== undefined && clicks !== undefined && opens !== clicks) { - throw new RangeError('Azure turns open and click tracking on or off together.'); - } - const wanted = opens ?? clicks; + const wanted = trackingSwitchOf(message.tracking, 'Azure'); return wanted === undefined ? config.userEngagementTrackingDisabled : !wanted; }; @@ -96,7 +92,7 @@ const convertMessage = async (message, config, signal) => { } const bytes = Buffer.byteLength(JSON.stringify(payload), 'utf8'); if (bytes > MAX_REQUEST_BYTES) { - throw new RangeError( + throw new RefusalError( `The request is ${bytes} bytes; Azure takes at most ${MAX_REQUEST_BYTES}, attachments included.`, ); } diff --git a/src/brevo/message-converter.js b/src/brevo/message-converter.js index c41ad2a..fd44f2e 100644 --- a/src/brevo/message-converter.js +++ b/src/brevo/message-converter.js @@ -15,6 +15,7 @@ const { generateId } = require('../core/id.js'); const { toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { toUuid } = require('../internal/uuid.js'); +const { RefusalError } = require('../internal/refusal.js'); const addressOf = (address) => { const email = toAsciiAddress(address.address); @@ -39,7 +40,7 @@ const addressesOf = (addresses) => { */ const convertMessage = async (message, config, signal) => { if (message.replyRecipients.length > 1) { - throw new RangeError('Brevo takes one reply-to address.'); + throw new RefusalError('Brevo takes one reply-to address.'); } const payload = { sender: addressOf(message.sender), @@ -66,7 +67,7 @@ const convertMessage = async (message, config, signal) => { payload.attachment = []; for (const item of attachments) { if (item.attachment.inline) { - throw new RangeError('Brevo cannot send an inline attachment, for a cid: URL.'); + throw new RefusalError('Brevo cannot send an inline attachment, for a cid: URL.'); } payload.attachment.push({ name: item.attachment.filename, content: item.content }); } diff --git a/src/cloudflare/message-converter.js b/src/cloudflare/message-converter.js index 3955d41..2de0c75 100644 --- a/src/cloudflare/message-converter.js +++ b/src/cloudflare/message-converter.js @@ -13,6 +13,7 @@ const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { rawMessageText } = require('../internal/raw.js'); const { RawMessageValidationError } = require('../core/raw-message.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); // "The combined number of addresses across to, cc, and bcc must not exceed 50." const MAX_RECIPIENTS = 50; @@ -34,10 +35,10 @@ const checkLimits = (message) => { const total = message.recipients.length + message.ccRecipients.length + message.bccRecipients.length; if (total > MAX_RECIPIENTS) { - throw new RangeError(`Cloudflare takes at most ${MAX_RECIPIENTS} recipients a message.`); + throw new RefusalError(`Cloudflare takes at most ${MAX_RECIPIENTS} recipients a message.`); } if (message.replyRecipients.length > 1) { - throw new RangeError('Cloudflare takes one reply-to address.'); + throw new RefusalError('Cloudflare takes one reply-to address.'); } }; diff --git a/src/internal/refusal.js b/src/internal/refusal.js index 3f9bf79..c33e63a 100644 --- a/src/internal/refusal.js +++ b/src/internal/refusal.js @@ -1,9 +1,17 @@ // Not ported: the receipt of a message a provider cannot take as it is, which -// its converter refuses with a RangeError before any request: over a limit of -// the provider, or with a field it cannot express. +// its converter refuses before any request: over a limit of the provider, or +// with a field it cannot express. const { createFailedReceipt } = require('../core/receipt.js'); +/** + * The refusal of a converter, a RangeError that only a converter throws, so + * that another, such as a body over its limit, keeps its own receipt. + * + * @internal + */ +class RefusalError extends RangeError {} + /** * The receipt of a refusal before any request, a non-retryable `validation` * failure, as for a message a transport cannot schedule; or `undefined` for @@ -15,8 +23,8 @@ const { createFailedReceipt } = require('../core/receipt.js'); * @internal */ const refusalFailure = (provider, error) => - error instanceof RangeError + error instanceof RefusalError ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) : undefined; -module.exports = { refusalFailure }; +module.exports = { RefusalError, refusalFailure }; diff --git a/src/internal/tracking.js b/src/internal/tracking.js new file mode 100644 index 0000000..6e251a0 --- /dev/null +++ b/src/internal/tracking.js @@ -0,0 +1,25 @@ +// Not ported: the `tracking` of a message for a provider with one switch for +// open and click tracking together (Azure, Maileroo). + +const { RefusalError } = require('./refusal.js'); + +/** + * The one switch a message's `tracking` asks for: what it says of opens or of + * clicks, or `undefined` when it says nothing. + * + * @param tracking The `tracking` of the message. + * @param label The provider's name, for the error. + * @returns `true`, `false` or `undefined`. + * @throws {RangeError} If it asks for one kind without the other. + * @internal + */ +const trackingSwitchOf = (tracking, label) => { + if (tracking === undefined) return undefined; + const { opens, clicks } = tracking; + if (opens !== undefined && clicks !== undefined && opens !== clicks) { + throw new RefusalError(`${label} turns open and click tracking on or off together.`); + } + return opens ?? clicks; +}; + +module.exports = { trackingSwitchOf }; diff --git a/src/maileroo/maileroo-transport.js b/src/maileroo/maileroo-transport.js index fef35ca..d8afe3c 100644 --- a/src/maileroo/maileroo-transport.js +++ b/src/maileroo/maileroo-transport.js @@ -8,6 +8,7 @@ const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); +const { refusalFailure } = require('../internal/refusal.js'); /** * Maileroo transport implementation for sending emails via Maileroo API. @@ -143,6 +144,8 @@ const responseToReceipt = (response) => { }; const createMailerooFailure = (message, error) => { + const refusal = refusalFailure('maileroo', error); + if (refusal !== undefined) return refusal; if (error instanceof MailerooApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/maileroo/message-converter.js b/src/maileroo/message-converter.js index 61c8d25..be60086 100644 --- a/src/maileroo/message-converter.js +++ b/src/maileroo/message-converter.js @@ -5,13 +5,14 @@ const { generateId } = require('../core/id.js'); const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); -const { refuseScheduledAt } = require('../internal/schedule.js'); +const { trackingSwitchOf } = require('../internal/tracking.js'); const { convertAttachments, formatAddresses, visitCustomHeaders, withCalendarAttachment, } = require('../internal/provider.js'); +const { RefusalError } = require('../internal/refusal.js'); const STANDARD_HEADERS = new Set([ 'from', @@ -36,11 +37,12 @@ const STANDARD_HEADERS = new Set([ * @param signal Optional abort signal for cancellation. * @returns JSON object ready for Maileroo API submission. * @throws {TypeError} If the message carries an invalid message identifier. + * @throws {RangeError} If it asks for opens without clicks, or the reverse, + * or its metadata cannot be tags. * @throws {Error} If the caller aborts the operation. */ const convertMessage = async (message, config, signal) => { signal?.throwIfAborted(); - refuseScheduledAt(message, 'maileroo'); const attachments = withCalendarAttachment(message); @@ -71,8 +73,14 @@ const convertMessage = async (message, config, signal) => { emailData.plain = message.content.text; } - if (config.tracking !== undefined) { - emailData.tracking = config.tracking; + // One switch for opens and clicks, the message's over the configuration's. + const tracking = trackingSwitchOf(message.tracking, 'Maileroo') ?? config.tracking; + if (tracking !== undefined) { + emailData.tracking = tracking; + } + + if (message.scheduledAt !== undefined) { + emailData.scheduled_at = message.scheduledAt.toISOString(); } const tags = convertTags(message, config); @@ -136,9 +144,28 @@ const convertOptionalAddressList = (addresses) => { return addresses.length === 0 ? undefined : convertAddressList(addresses); }; +// "Maximum key length is 128 characters, and maximum value length is 768". +const MAX_TAG_VALUE = 768; +const NUMBERED_TAG = /^tag\d+$/; + +// The tags of the configuration, the metadata of the message over them, then +// `tag1`… of its tags, which a key of the metadata may not take. const convertTags = (message, config) => { const tags = config.tags == null ? {} : { ...config.tags }; + if (message.metadata !== undefined) { + for (const key in message.metadata) { + const value = message.metadata[key]; + if (value.length > MAX_TAG_VALUE || NUMBERED_TAG.test(key)) { + throw new RefusalError( + `Maileroo takes metadata as tags, whose value has at most ${MAX_TAG_VALUE} ` + + `characters and whose name is not tag1, tag2…: ${key}`, + ); + } + tags[key] = value; + } + } + for (let index = 0; index < message.tags.length; index++) { tags[`tag${index + 1}`] = message.tags[index]; } diff --git a/src/mailersend/message-converter.js b/src/mailersend/message-converter.js index 9ef9965..df57000 100644 --- a/src/mailersend/message-converter.js +++ b/src/mailersend/message-converter.js @@ -12,6 +12,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { parseMessageId } = require('../core/message-id.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { RefusalError } = require('../internal/refusal.js'); const LIMITS = [ ['recipients', 'to', 50], @@ -44,20 +45,20 @@ const addressesOf = (addresses) => { const checkLimits = (message, config) => { for (const [field, name, limit] of LIMITS) { if (message[field].length > limit) { - throw new RangeError(`MailerSend takes at most ${limit} ${name} recipients.`); + throw new RefusalError(`MailerSend takes at most ${limit} ${name} recipients.`); } } if (message.tags.length > MAX_TAGS) { - throw new RangeError(`MailerSend takes at most ${MAX_TAGS} tags.`); + throw new RefusalError(`MailerSend takes at most ${MAX_TAGS} tags.`); } if (message.replyRecipients.length > 1) { - throw new RangeError('MailerSend takes one reply-to address.'); + throw new RefusalError('MailerSend takes one reply-to address.'); } if ( message.scheduledAt !== undefined && message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS ) { - throw new RangeError('MailerSend schedules a message at most 72 hours ahead.'); + throw new RefusalError('MailerSend schedules a message at most 72 hours ahead.'); } }; diff --git a/src/mailjet/message-converter.js b/src/mailjet/message-converter.js index 3865fbe..4a39bbd 100644 --- a/src/mailjet/message-converter.js +++ b/src/mailjet/message-converter.js @@ -9,6 +9,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); // "MAX RECIPIENTS: 50", across To, Cc and Bcc of the whole request. const MAX_RECIPIENTS = 50; @@ -86,15 +87,15 @@ const recipientCount = (message) => { const checkLimits = (message) => { refuseScheduledAt(message, 'mailjet'); if (message.subject.length > MAX_SUBJECT) { - throw new RangeError(`Mailjet takes a subject of at most ${MAX_SUBJECT} characters.`); + throw new RefusalError(`Mailjet takes a subject of at most ${MAX_SUBJECT} characters.`); } if (recipientCount(message) > MAX_RECIPIENTS) { - throw new RangeError(`Mailjet takes at most ${MAX_RECIPIENTS} recipients a message.`); + throw new RefusalError(`Mailjet takes at most ${MAX_RECIPIENTS} recipients a message.`); } if (message.replyRecipients.length > 1) { - throw new RangeError('Mailjet takes one reply-to address.'); + throw new RefusalError('Mailjet takes one reply-to address.'); } - if (message.tags.length > 1) throw new RangeError('Mailjet takes one tag per message.'); + if (message.tags.length > 1) throw new RefusalError('Mailjet takes one tag per message.'); }; const trackingOf = (message, config, hasHtml) => { @@ -136,7 +137,7 @@ const convertMessage = async (message, config, signal) => { let hasHeaders = false; for (const field of headerFieldsOf(message)) { if (FORBIDDEN_HEADERS.has(field.name.toLowerCase())) { - throw new RangeError(`Mailjet sets ${field.name} itself; the message cannot give it.`); + throw new RefusalError(`Mailjet sets ${field.name} itself; the message cannot give it.`); } headers[field.name] = field.value; hasHeaders = true; diff --git a/src/postmark/message-converter.js b/src/postmark/message-converter.js index 705af92..18a2125 100644 --- a/src/postmark/message-converter.js +++ b/src/postmark/message-converter.js @@ -13,6 +13,7 @@ const { formatMailbox } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); const addressList = (addresses) => formatAddresses(addresses, formatMailbox).join(', '); @@ -34,7 +35,7 @@ const trackingOf = (message, config) => { */ const convertMessage = async (message, config, signal) => { refuseScheduledAt(message, 'postmark'); - if (message.tags.length > 1) throw new RangeError('Postmark takes one tag per message.'); + if (message.tags.length > 1) throw new RefusalError('Postmark takes one tag per message.'); const payload = { From: formatMailbox(message.sender), To: addressList(message.recipients), diff --git a/src/resend/message-converter.js b/src/resend/message-converter.js index 0f0ad22..9a01a53 100644 --- a/src/resend/message-converter.js +++ b/src/resend/message-converter.js @@ -11,6 +11,7 @@ const { visitCustomHeaders, withCalendarAttachment, } = require('../internal/provider.js'); +const { RefusalError } = require('../internal/refusal.js'); const PRIORITIES = { high: '1', normal: '3', low: '5' }; // The name and value of a Resend tag: "ASCII letters, numbers, underscores, @@ -31,7 +32,7 @@ const tagsOf = (message) => { for (const key in message.metadata) { const value = message.metadata[key]; if (!TAG_TEXT.test(key) || !TAG_TEXT.test(value) || names.has(key)) { - throw new RangeError( + throw new RefusalError( `Resend takes metadata as tags, whose name and value are ASCII letters, digits, ` + `_ or -, and a name other than those of the tags: ${key}`, ); diff --git a/src/scaleway/message-converter.js b/src/scaleway/message-converter.js index e383c35..118c40f 100644 --- a/src/scaleway/message-converter.js +++ b/src/scaleway/message-converter.js @@ -10,6 +10,7 @@ const { formatMailbox, toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); // "Maximum email size (API): 2 MB, including the email and all attachments". const MAX_EMAIL_BYTES = 2 * 1024 * 1024; @@ -60,7 +61,7 @@ const convertMessage = async (message, config, signal) => { refuseScheduledAt(message, 'scaleway'); for (const attachment of message.attachments) { if (attachment.inline) { - throw new RangeError( + throw new RefusalError( 'Scaleway cannot send an inline attachment, which it has no Content-ID for.', ); } @@ -90,7 +91,7 @@ const convertMessage = async (message, config, signal) => { } const bytes = Buffer.byteLength(JSON.stringify(payload), 'utf8'); if (bytes > MAX_EMAIL_BYTES) { - throw new RangeError( + throw new RefusalError( `The request is ${bytes} bytes; Scaleway takes at most ${MAX_EMAIL_BYTES}, attachments included.`, ); } diff --git a/src/sendgrid/message-converter.js b/src/sendgrid/message-converter.js index ceb77c3..99d7e5b 100644 --- a/src/sendgrid/message-converter.js +++ b/src/sendgrid/message-converter.js @@ -10,6 +10,7 @@ const { formatAddresses, withCalendarAttachment, } = require('../internal/provider.js'); +const { RefusalError } = require('../internal/refusal.js'); // `send_at` may be at most 72 hours ahead. const SCHEDULE_WINDOW_MS = 72 * 60 * 60 * 1000; @@ -37,7 +38,7 @@ const convertMessage = async (message, config, signal) => { message.scheduledAt !== undefined && message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS ) { - throw new RangeError('SendGrid schedules a message at most 72 hours ahead.'); + throw new RefusalError('SendGrid schedules a message at most 72 hours ahead.'); } const attachments = withCalendarAttachment(message); const personalization = { diff --git a/src/ses/message-converter.js b/src/ses/message-converter.js index 99e944a..1daed50 100644 --- a/src/ses/message-converter.js +++ b/src/ses/message-converter.js @@ -12,6 +12,7 @@ const { visitCustomHeaders, withCalendarAttachment, } = require('../internal/provider.js'); +const { RefusalError } = require('../internal/refusal.js'); // The header fields SES sets itself and refuses in Headers, lower-cased. const SES_HEADERS = new Set([ @@ -39,7 +40,7 @@ const metadataTagsOf = (metadata, tags) => { for (const key in metadata) { const value = metadata[key]; if (!TAG_TEXT.test(key) || !TAG_TEXT.test(value) || OWN_TAGS.has(key)) { - throw new RangeError( + throw new RefusalError( 'SES takes metadata as tags, whose name and value are ASCII letters, digits, _ or -, ' + `and a name other than category and priority: ${key}`, ); @@ -199,13 +200,13 @@ const convertHeaders = (message) => { if (value === '') return; const encoded = encodeHeaderValue(value); if (name.length + encoded.length > MAX_SES_HEADER_LENGTH) { - throw new RangeError(`SES takes a header field of at most 996 characters: ${name}`); + throw new RefusalError(`SES takes a header field of at most 996 characters: ${name}`); } headers.push({ Name: name, Value: encoded }); }, ); if (headers.length > MAX_SES_HEADERS) { - throw new RangeError( + throw new RefusalError( `SES takes at most ${MAX_SES_HEADERS} custom header fields, and the message has ${headers.length}.`, ); } diff --git a/src/smtp2go/message-converter.js b/src/smtp2go/message-converter.js index b417dbb..82d3ca8 100644 --- a/src/smtp2go/message-converter.js +++ b/src/smtp2go/message-converter.js @@ -8,6 +8,7 @@ const { formatMailbox } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); +const { RefusalError } = require('../internal/refusal.js'); // "An array of names and email addresses (up to 100)", in each field. const MAX_RECIPIENTS = 100; @@ -21,14 +22,14 @@ const checkLimits = (message, config) => { ['bccRecipients', 'bcc'], ]) { if (message[field].length > MAX_RECIPIENTS) { - throw new RangeError(`SMTP2GO takes at most ${MAX_RECIPIENTS} ${name} recipients.`); + throw new RefusalError(`SMTP2GO takes at most ${MAX_RECIPIENTS} ${name} recipients.`); } } if ( message.scheduledAt !== undefined && message.scheduledAt.getTime() - config.now() > SCHEDULE_WINDOW_MS ) { - throw new RangeError('SMTP2GO schedules a message at most 3 days ahead.'); + throw new RefusalError('SMTP2GO schedules a message at most 3 days ahead.'); } }; diff --git a/src/sparkpost/message-converter.js b/src/sparkpost/message-converter.js index 9e7004b..01a27dc 100644 --- a/src/sparkpost/message-converter.js +++ b/src/sparkpost/message-converter.js @@ -12,6 +12,7 @@ const { formatMailbox, toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); const { rawMessageText } = require('../internal/raw.js'); +const { RefusalError } = require('../internal/refusal.js'); // "Maximum number of tags - 10 per recipient". const MAX_TAGS = 10; @@ -100,7 +101,7 @@ const contentOf = async (message, signal) => { */ const convertMessage = async (message, config, signal) => { if (message.tags.length > MAX_TAGS) { - throw new RangeError(`SparkPost takes at most ${MAX_TAGS} tags.`); + throw new RefusalError(`SparkPost takes at most ${MAX_TAGS} tags.`); } const transmission = { recipients: recipientsOf(message), diff --git a/src/zeptomail/message-converter.js b/src/zeptomail/message-converter.js index c91d534..94757d2 100644 --- a/src/zeptomail/message-converter.js +++ b/src/zeptomail/message-converter.js @@ -7,6 +7,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); // At most 500 addresses in each of to, cc and bcc (TM_8001, SM_127). const MAX_RECIPIENTS = 500; @@ -33,17 +34,17 @@ const checkLimits = (message) => { ['bccRecipients', 'bcc'], ]) { if (message[field].length > MAX_RECIPIENTS) { - throw new RangeError(`ZeptoMail takes at most ${MAX_RECIPIENTS} ${name} recipients.`); + throw new RefusalError(`ZeptoMail takes at most ${MAX_RECIPIENTS} ${name} recipients.`); } } if (message.attachments.length > MAX_ATTACHMENTS) { - throw new RangeError(`ZeptoMail takes at most ${MAX_ATTACHMENTS} attachments.`); + throw new RefusalError(`ZeptoMail takes at most ${MAX_ATTACHMENTS} attachments.`); } if (message.subject.length > MAX_SUBJECT) { - throw new RangeError(`ZeptoMail takes a subject of at most ${MAX_SUBJECT} characters.`); + throw new RefusalError(`ZeptoMail takes a subject of at most ${MAX_SUBJECT} characters.`); } if (message.tags.length > 1) { - throw new RangeError('ZeptoMail takes one tag per message, as its client_reference.'); + throw new RefusalError('ZeptoMail takes one tag per message, as its client_reference.'); } }; diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js index 05da554..d4bb7f1 100644 --- a/tests/unit/api.test.js +++ b/tests/unit/api.test.js @@ -8,6 +8,7 @@ const { createApiClient, providerFailure, } = require('../../src/internal/api.js'); +const { RefusalError } = require('../../src/internal/refusal.js'); const { isDeliveryUnknown } = require('../../src/internal/delivery-unknown.js'); const { json, stubFetch } = require('../helpers/providers/fetch.js'); @@ -134,10 +135,13 @@ test('API: the receipt of a failure carries what the error says', () => { }); test('API: a refusal of the converter is a non-retryable validation failure', () => { - const refused = providerFailure('x', new RangeError('X takes at most 5 tags.')); + const refused = providerFailure('x', new RefusalError('X takes at most 5 tags.')); assert.strictEqual(refused.errors[0].category, 'validation'); assert.strictEqual(refused.retryable, false); assert.strictEqual(refused.errorMessages[0], 'X takes at most 5 tags.'); const other = providerFailure('x', new Error('boom')); assert.notStrictEqual(other.errors[0].category, 'validation'); + // Another RangeError, such as a body over its limit, keeps its own receipt. + const range = providerFailure('x', new RangeError('The body is over 4 MiB.')); + assert.notStrictEqual(range.errors[0].category, 'validation'); }); diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index 48b5a31..5b61700 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -105,7 +105,6 @@ const providers = { retries: 0, }), mailtrap: () => new MailtrapTransport({ apiToken: 'token', retries: 0 }), - maileroo: () => new MailerooTransport({ apiKey: 'key', retries: 0 }), lettermint: () => new LettermintTransport({ apiToken: 'token', retries: 0 }), plunk: () => new PlunkTransport({ apiKey: 'key', retries: 0 }), }; @@ -321,6 +320,52 @@ test('Fields: SES sends metadata as tags, over the default ones, and refuses wha assert.strictEqual(bodies.length, 1); }); +test('Fields: Maileroo sends scheduled_at, one tracking switch and metadata as tags', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + bodies.push(JSON.parse(init.body)); + return Response.json({ success: true, data: { reference_id: `ref${bodies.length}` } }); + }); + const transport = new MailerooTransport({ + apiKey: 'key', + retries: 0, + tracking: true, + tags: { app: 'shop', tenant: 'default' }, + }); + const sent = await transport.send( + createMessage({ + ...base, + tags: ['welcome'], + scheduledAt: tomorrow, + tracking: { clicks: false }, + metadata: { orderId: '42', tenant: 'acme' }, + }), + ); + assert.strictEqual(sent.successful, true); + assert.strictEqual(bodies[0].scheduled_at, '2026-10-18T09:00:00.000Z'); + assert.strictEqual(bodies[0].tracking, false); + assert.deepStrictEqual(bodies[0].tags, { + app: 'shop', + tenant: 'acme', + orderId: '42', + tag1: 'welcome', + }); + await transport.send(createMessage(base)); + assert.strictEqual(bodies[1].tracking, true); + assert.strictEqual(bodies[1].scheduled_at, undefined); + const refusals = [ + [{ tracking: { opens: true, clicks: false } }, /on or off together/], + [{ metadata: { tag2: 'x' } }, /Maileroo takes metadata as tags/], + [{ metadata: { note: 'x'.repeat(769) } }, /at most 768/], + ]; + for (const [fields, error] of refusals) { + const refused = await transport.send(createMessage({ ...base, ...fields })); + assert.match(refused.errorMessages[0], error); + assert.strictEqual(refused.errors[0].category, 'validation'); + } + assert.strictEqual(bodies.length, 2); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From 18732c31fcf5efbeb80425126568ff4e12622774 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:19:26 +0300 Subject: [PATCH 034/143] feat(lettermint): the metadata and tracking of the message on Lettermint A message's `metadata` goes in `metadata`, its keys over those of the configuration, and its `tracking` sets `track_opens` and `track_clicks` over the configuration's `settings`. Lettermint cannot schedule a message, which stays refused. Plunk's page says it takes none of the three fields. Co-Authored-By: Claude Opus 5.5 --- docs/transports/lettermint.md | 5 +++++ docs/transports/plunk.md | 6 ++++++ src/lettermint/message-converter.js | 23 ++++++++++++---------- tests/unit/message-fields.test.js | 30 +++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/transports/lettermint.md b/docs/transports/lettermint.md index 7d0fd88..da2fb93 100644 --- a/docs/transports/lettermint.md +++ b/docs/transports/lettermint.md @@ -98,6 +98,11 @@ Lettermint accepts one tag per message. If a message has exactly one has more than one tag, the transport returns a failed receipt instead of sending the message. +A message's own `metadata` goes in `metadata` too, its keys over those of the +transport, and its `tracking` sets `track_opens` and `track_clicks` over the +transport's `settings`. Lettermint cannot schedule a message, so `scheduledAt` +is refused as `validation`. + ## Batch sending The `sendMany()` method uses Lettermint's batch endpoint and automatically diff --git a/docs/transports/plunk.md b/docs/transports/plunk.md index 2fff4ee..ad482f5 100644 --- a/docs/transports/plunk.md +++ b/docs/transports/plunk.md @@ -155,6 +155,12 @@ can start and gives the receipts in the order of the messages. const transport = new PlunkTransport({ apiKey: process.env.PLUNK_API_KEY, concurrency: 4 }); ``` +## Scheduling, metadata and tracking + +Plunk takes none of the three. A message with `scheduledAt` is refused as a +`validation` failure rather than sent at once, and its `tracking` and +`metadata` are not sent, as over SMTP. + ## Advanced features and reliability The Plunk transport includes comprehensive configuration options for timeout diff --git a/src/lettermint/message-converter.js b/src/lettermint/message-converter.js index 66ca64a..3212505 100644 --- a/src/lettermint/message-converter.js +++ b/src/lettermint/message-converter.js @@ -80,11 +80,13 @@ const convertMessage = async (message, config, signal) => { emailData.tag = config.tag; } - if (config.metadata != null && Object.keys(config.metadata).length > 0) { - emailData.metadata = { ...config.metadata }; + // The metadata of the configuration, that of the message over it. + const metadata = { ...config.metadata, ...message.metadata }; + if (Object.keys(metadata).length > 0) { + emailData.metadata = metadata; } - const settings = convertSettings(config); + const settings = convertSettings(config, message.tracking); if (settings != null) { emailData.settings = settings; } @@ -104,17 +106,18 @@ const convertMessage = async (message, config, signal) => { return emailData; }; -const convertSettings = (config) => { - if (config.settings == null) return undefined; - +// The tracking of the message over the settings of the configuration. +const convertSettings = (config, tracking) => { const settings = {}; - if (config.settings.trackOpens !== undefined) { - settings.track_opens = config.settings.trackOpens; + const opens = tracking?.opens ?? config.settings?.trackOpens; + if (opens !== undefined) { + settings.track_opens = opens; } - if (config.settings.trackClicks !== undefined) { - settings.track_clicks = config.settings.trackClicks; + const clicks = tracking?.clicks ?? config.settings?.trackClicks; + if (clicks !== undefined) { + settings.track_clicks = clicks; } return Object.keys(settings).length > 0 ? settings : undefined; diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index 5b61700..71eb029 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -366,6 +366,36 @@ test('Fields: Maileroo sends scheduled_at, one tracking switch and metadata as t assert.strictEqual(bodies.length, 2); }); +test('Fields: Lettermint sends the metadata and tracking of the message over its settings', async (t) => { + const bodies = []; + t.mock.method(globalThis, 'fetch', async (_input, init) => { + bodies.push(JSON.parse(init.body)); + return Response.json({ message_id: `lm${bodies.length}`, status: 'pending' }, { status: 202 }); + }); + const transport = new LettermintTransport({ + apiToken: 'token', + retries: 0, + metadata: { tenant: 'default', app: 'shop' }, + settings: { trackOpens: true, trackClicks: true }, + }); + const sent = await transport.send( + createMessage({ + ...base, + tracking: { clicks: false }, + metadata: { tenant: 'acme', orderId: '42' }, + }), + ); + assert.strictEqual(sent.successful, true); + assert.deepStrictEqual(bodies[0].metadata, { tenant: 'acme', app: 'shop', orderId: '42' }); + assert.deepStrictEqual(bodies[0].settings, { track_opens: true, track_clicks: false }); + const plain = new LettermintTransport({ apiToken: 'token', retries: 0 }); + await plain.send(createMessage({ ...base, tracking: { opens: false } })); + assert.deepStrictEqual(bodies[1].settings, { track_opens: false }); + assert.strictEqual(bodies[1].metadata, undefined); + await plain.send(createMessage(base)); + assert.strictEqual(bodies[2].settings, undefined); +}); + const CORE = 'urn:ietf:params:jmap:core'; const MAIL = 'urn:ietf:params:jmap:mail'; const SUBMISSION = 'urn:ietf:params:jmap:submission'; From 4b6f19c15dc24f2ea0d29b57aca96bfc897e7792 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:34:40 +0300 Subject: [PATCH 035/143] feat(breaker): a circuit breaker decorator Ported from unemail's circuit breaker as a decorator of one transport. After `failureThreshold` failures of the provider in a row (network, timeout, server error, unavailable, or a thrown error other than the caller's abort) the circuit opens and answers each send at once with a retryable `service-unavailable` receipt and the time left, which `./pool` fails over with and `./retry` waits for. After `resetTimeoutMs` exactly one probe goes through, where unemail let every send through while half-open (U10); its success closes the circuit, its failure opens it again. A late result of a send made in another state decides nothing. The state is that of the instance. `sendMany` goes through the wrapped `sendMany`, its batching kept, a message read while the circuit is open answered in place; that is `internal/gate.js`, which the other decorators of this phase share. A listener that throws becomes a process warning (`internal/callback.js`). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/breaker.md | 70 +++++++++ package.json | 4 + scripts/size-budgets.json | 3 +- src/breaker/breaker-transport.js | 244 +++++++++++++++++++++++++++++ src/breaker/config.js | 38 +++++ src/breaker/index.d.ts | 115 ++++++++++++++ src/breaker/index.js | 7 + src/internal/callback.js | 19 +++ src/internal/gate.js | 119 ++++++++++++++ tests/helpers/decorators/fake.js | 62 ++++++++ tests/types/breaker.test-d.ts | 35 +++++ tests/unit/breaker.test.js | 258 +++++++++++++++++++++++++++++++ tests/unit/gate.test.js | 65 ++++++++ 16 files changed, 1046 insertions(+), 1 deletion(-) create mode 100644 docs/transports/breaker.md create mode 100644 src/breaker/breaker-transport.js create mode 100644 src/breaker/config.js create mode 100644 src/breaker/index.d.ts create mode 100644 src/breaker/index.js create mode 100644 src/internal/callback.js create mode 100644 src/internal/gate.js create mode 100644 tests/helpers/decorators/fake.js create mode 100644 tests/types/breaker.test-d.ts create mode 100644 tests/unit/breaker.test.js create mode 100644 tests/unit/gate.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 14d0199..340e09b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -550,6 +550,12 @@ fails on upstream's behavior): addresses and the clock, which put the recipients into every log of a receipt and could repeat within a millisecond, while the answer has a `message_id`. `./cloudflare` writes the REST shapes and takes the `message_id`. +- U10. unemail (`src/middleware/circuit-breaker.ts`): half-open let every send through, not one + probe, so a provider coming back met the whole backlog at once; any failure but a malformed + message counted, a rate limit and a caller's abort among them. `./breaker` lets one probe + through, answers the others as while open, counts only network, timeout, server and + unavailability failures and a thrown error that is not the caller's abort, and ignores a late + result of a send made in another state. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ee5565c..ddfb14a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -106,6 +106,7 @@ const guide = [ { text: 'Retry', link: '/transports/retry' }, { text: 'OpenTelemetry', link: '/transports/otel' }, { text: 'Logger', link: '/transports/logger' }, + { text: 'Circuit breaker', link: '/transports/breaker' }, ], }, ]; diff --git a/docs/api/exports.md b/docs/api/exports.md index 039c870..ec28f3f 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -69,6 +69,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/retry` | `RetryTransport`, `createRetryTransport` | [Retry](/transports/retry) | | `@alexify/mailer/otel` | `OpenTelemetryTransport`, `createOpenTelemetryTransport`, `createEmailAttributeExtractor`, `createErrorClassifier`, `defaultErrorClassifier` | [OpenTelemetry](/transports/otel) | | `@alexify/mailer/logger` | `LoggerTransport` | [Logger](/transports/logger) | +| `@alexify/mailer/breaker` | `CircuitBreakerTransport` | [Circuit breaker](/transports/breaker) | The types of each subpath (configurations, receipts with provider details, the structural `TracerProvider`, `MeterProvider` and `Logger` of the decorators) are in diff --git a/docs/transports/breaker.md b/docs/transports/breaker.md new file mode 100644 index 0000000..f0117c3 --- /dev/null +++ b/docs/transports/breaker.md @@ -0,0 +1,70 @@ +# Circuit breaker + +`CircuitBreakerTransport` stops sending through a transport that keeps failing, +answers at once while it is down, and lets one probe through to learn whether it +is back. It comes from the circuit breaker of unemail. + +## Basic usage + +```js +const { CircuitBreakerTransport } = require('@alexify/mailer/breaker'); +const { SesTransport } = require('@alexify/mailer/ses'); + +const transport = new CircuitBreakerTransport(new SesTransport(sesConfig), { + failureThreshold: 5, + resetTimeoutMs: 30_000, + onStateChange: (state, previous) => console.log(`SES circuit ${previous} -> ${state}`), +}); +``` + +## States + +- **closed:** every send goes through. A failed receipt counts when one of its + errors is of the category `network`, `timeout`, `server-error` or + `service-unavailable`, and so does a thrown error other than the caller's + abort. A message the provider refused, a rate limit or a success answers + that the provider is up, and starts the count again. +- **open:** after `failureThreshold` failures in a row. A send is answered at + once, nothing sent, with a retryable `service-unavailable` receipt (code + `breaker.open`) whose `retryAfterMilliseconds` is the time left. +- **half-open:** after `resetTimeoutMs`. One send goes through as a probe; the + others are answered as while open until it ends. Its success closes the + circuit, its failure opens it again. A probe the caller aborts decides + nothing. + +A result counts only in the state it was sent in: a late one of a send made +before the circuit opened changes nothing. The state is that of the instance; +`breaker.state` reads it. + +## With a pool and retries + +The receipt of an open circuit is one `./pool` fails over with and `./retry` +waits for, so a breaker around each provider of a pool takes a failing one out +of the rotation until its probe succeeds: + +```js +const { PoolTransport } = require('@alexify/mailer/pool'); + +const pool = new PoolTransport({ + transports: [ + { transport: new CircuitBreakerTransport(ses), priority: 1 }, + { transport: new CircuitBreakerTransport(postmark), priority: 2 }, + ], + strategy: 'priority', +}); +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `failureThreshold` | `5` | The failures in a row that open the circuit. | +| `resetTimeoutMs` | `30000` | How long it stays open before a probe. | +| `isFailure` | the categories above | Whether a receipt counts as a failure. | +| `onStateChange` | | Hears each change of state; one that throws becomes a process warning (`MAILER_BREAKER_LISTENER_FAILED`). | +| `now` | `Date.now` | The clock. | + +`sendMany()` goes through the `sendMany()` of the wrapped transport while the +circuit is closed, so its batching is kept; a message read while it is open is +answered in place, the receipts in the order of the messages. `sendRaw()` is +guarded as `send()` is, and `verify()` goes through whatever the state. diff --git a/package.json b/package.json index a07e1f7..23615e4 100644 --- a/package.json +++ b/package.json @@ -135,6 +135,10 @@ "./logger": { "types": "./src/logger/index.d.ts", "default": "./src/logger/index.js" + }, + "./breaker": { + "types": "./src/breaker/index.d.ts", + "default": "./src/breaker/index.js" } }, "directories": { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index a922913..d05ac19 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -27,5 +27,6 @@ "./retry": 3, "./mock": 2, "./otel": 5, - "./logger": 2.5 + "./logger": 2.5, + "./breaker": 2 } diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js new file mode 100644 index 0000000..6c40525 --- /dev/null +++ b/src/breaker/breaker-transport.js @@ -0,0 +1,244 @@ +// Ported from unemail v0.7.0 (src/middleware/circuit-breaker.ts): a circuit +// breaker that stops sending through a transport that keeps failing, and lets +// one probe through once it has waited. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: half-open lets exactly one probe through, where +// unemail let every send through while it was half-open; the circuit is that +// of the instance; only a failure of the provider counts (network, timeout, +// server error, unavailable), not a refused message or a caller's abort; and +// an open circuit answers with a retryable `service-unavailable` receipt +// carrying the time left, which `./pool` fails over with and `./retry` waits +// for. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { warnThrown } = require('../internal/callback.js'); +const { refuseRaw } = require('../internal/capabilities.js'); +const { gatedSendMany } = require('../internal/gate.js'); +const { createBreakerConfig } = require('./config.js'); + +// The categories that say the provider is failing rather than the message. +const FAILURE_CATEGORIES = new Set(['network', 'timeout', 'server-error', 'service-unavailable']); + +const isProviderFailure = (receipt) => { + if (receipt.successful) return false; + for (const error of receipt.errors) { + if (FAILURE_CATEGORIES.has(error.category)) return true; + } + return false; +}; + +const isAbortOf = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +/** + * A transport that stops sending through the one it wraps after it failed + * `failureThreshold` times in a row, answers at once while open, and after + * `resetTimeoutMs` lets one probe through: its success closes the circuit, + * its failure opens it again. + */ +class CircuitBreakerTransport { + id; + config; + wrappedTransport; + #state = 'closed'; + #failures = 0; + #openedAt = 0; + #probing = false; + + /** + * @param transport The transport to guard. + * @param config `failureThreshold`, `resetTimeoutMs`, `isFailure`, + * `onStateChange` and `now`. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport, config) { + this.wrappedTransport = transport; + this.id = transport.id; + this.config = createBreakerConfig(config); + } + + /** + * The state of the circuit: `closed`, `open`, or `half-open` when the next + * send would be a probe. + */ + get state() { + if (this.#state === 'open' && this.#remaining() <= 0) return 'half-open'; + return this.#state; + } + + /** + * Sends one message, unless the circuit is open. + * + * @param message The message. + * @param options Optional abort signal and retry callback. + * @returns The receipt of the wrapped transport, or that of an open circuit. + */ + send(message, options) { + return this.#guard(options, (sendOptions) => this.wrappedTransport.send(message, sendOptions)); + } + + /** + * Sends a raw message, unless the circuit is open. A wrapped transport + * without `sendRaw()` gives a failed receipt of the category + * `configuration`, which the circuit does not count. + */ + async sendRaw(message, options) { + if (!isRawTransport(this.wrappedTransport)) { + options?.signal?.throwIfAborted(); + return refuseRaw(this.wrappedTransport); + } + return this.#guard(options, (sendOptions) => + this.wrappedTransport.sendRaw(message, sendOptions), + ); + } + + /** + * Sends messages through the `sendMany()` of the wrapped transport while + * the circuit is closed; a message read while it is open is answered at + * once, and one while it is half-open goes as its probe. + */ + sendMany(messages, options) { + return gatedSendMany(this.wrappedTransport, messages, options, { + admit: (message) => { + const entry = this.#enter(); + return entry.refusal === undefined ? { message, probe: entry.probe } : entry.refusal; + }, + settle: (receipt, admitted) => { + this.#record(this.#counts(receipt), admitted.probe); + return receipt; + }, + fail: (error, admitted) => this.#thrown(error, options?.signal, admitted.probe), + }); + } + + /** + * Verifies the wrapped transport, whatever the state of the circuit; it has + * nothing to check when that transport cannot verify. + */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** + * Disposes of the wrapped transport when it supports explicit resource + * management. + */ + async [Symbol.asyncDispose]() { + const transport = Object(this.wrappedTransport); + if (typeof transport[Symbol.asyncDispose] === 'function') { + await transport[Symbol.asyncDispose](); + } else if (typeof transport[Symbol.dispose] === 'function') { + transport[Symbol.dispose](); + } + } + + async #guard(options, send) { + options?.signal?.throwIfAborted(); + const entry = this.#enter(); + if (entry.refusal !== undefined) return entry.refusal.receipt; + let receipt; + try { + receipt = await send(options); + } catch (error) { + this.#thrown(error, options?.signal, entry.probe); + throw error; + } + this.#record(this.#counts(receipt), entry.probe); + return receipt; + } + + #remaining() { + return this.#openedAt + this.config.resetTimeoutMs - this.config.now(); + } + + // Whether a send may go now, and whether it is the probe. + #enter() { + if (this.#state === 'open') { + const remaining = this.#remaining(); + if (remaining > 0) return { refusal: this.#refusal(remaining) }; + this.#transition('half-open'); + } + if (this.#state === 'half-open') { + if (this.#probing) return { refusal: this.#refusal(undefined) }; + this.#probing = true; + return { probe: true }; + } + return { probe: false }; + } + + #refusal(remaining) { + const receipt = createFailedReceipt( + `The circuit of ${this.id} is open after ${this.config.failureThreshold} failures in a ` + + 'row; the message was not sent.', + { + provider: this.id, + category: 'service-unavailable', + retryable: true, + retryAfterMilliseconds: remaining === undefined ? undefined : Math.ceil(remaining), + code: 'breaker.open', + }, + ); + return { receipt }; + } + + #counts(receipt) { + return (this.config.isFailure ?? isProviderFailure)(receipt) === true; + } + + #thrown(error, signal, probe) { + if (isAbortOf(error, signal)) { + // The caller gave up: nothing is learned of the provider. + if (probe) this.#probing = false; + return; + } + this.#record(true, probe); + } + + // A result counts only in the state it was sent in: a late one of a send + // made before the circuit opened decides nothing. + #record(failed, probe) { + if (probe) { + this.#probing = false; + if (failed) this.#open(); + else this.#close(); + return; + } + if (this.#state !== 'closed') return; + if (!failed) { + this.#failures = 0; + return; + } + this.#failures++; + if (this.#failures >= this.config.failureThreshold) this.#open(); + } + + #open() { + this.#openedAt = this.config.now(); + this.#failures = 0; + this.#transition('open'); + } + + #close() { + this.#failures = 0; + this.#transition('closed'); + } + + #transition(state) { + if (this.#state === state) return; + const previous = this.#state; + this.#state = state; + try { + this.config.onStateChange?.(state, previous); + } catch (error) { + warnThrown( + 'The onStateChange of CircuitBreakerTransport', + error, + 'MAILER_BREAKER_LISTENER_FAILED', + ); + } + } +} + +module.exports = { CircuitBreakerTransport }; diff --git a/src/breaker/config.js b/src/breaker/config.js new file mode 100644 index 0000000..b50063c --- /dev/null +++ b/src/breaker/config.js @@ -0,0 +1,38 @@ +// Ported from unemail v0.7.0 (src/middleware/circuit-breaker.ts): the options +// of the circuit breaker. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const checkFunction = (value, name) => { + if (value !== undefined && typeof value !== 'function') { + throw new TypeError(`${name} must be a function.`); + } + return value; +}; + +/** + * Resolves the options of the circuit breaker. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createBreakerConfig = (config = {}) => { + const failureThreshold = config.failureThreshold ?? 5; + if (!Number.isInteger(failureThreshold) || failureThreshold < 1) { + throw new TypeError(`failureThreshold must be a positive integer: ${String(failureThreshold)}`); + } + const resetTimeoutMs = config.resetTimeoutMs ?? 30000; + if (typeof resetTimeoutMs !== 'number' || !(resetTimeoutMs > 0) || resetTimeoutMs === Infinity) { + throw new TypeError(`resetTimeoutMs must be a positive number: ${String(resetTimeoutMs)}`); + } + return { + failureThreshold, + resetTimeoutMs, + isFailure: checkFunction(config.isFailure, 'isFailure'), + onStateChange: checkFunction(config.onStateChange, 'onStateChange'), + now: checkFunction(config.now, 'now') ?? Date.now, + }; +}; + +module.exports = { checkFunction, createBreakerConfig }; diff --git a/src/breaker/index.d.ts b/src/breaker/index.d.ts new file mode 100644 index 0000000..c91d15a --- /dev/null +++ b/src/breaker/index.d.ts @@ -0,0 +1,115 @@ +// Types ported from unemail v0.7.0 (src/middleware) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +/// + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * `closed` lets every send through, `open` answers each at once, and + * `half-open` lets one probe through to learn whether the provider is back. + */ +export type CircuitState = 'closed' | 'open' | 'half-open'; + +/** + * The options of {@link CircuitBreakerTransport}. + */ +export interface CircuitBreakerConfig { + /** + * The failures in a row that open the circuit. + * @default 5 + */ + readonly failureThreshold?: number; + /** + * How long the circuit stays open before it lets a probe through, in + * milliseconds. + * @default 30000 + */ + readonly resetTimeoutMs?: number; + /** + * Whether a receipt counts as a failure of the provider. By default a + * failed receipt counts when one of its errors is of the category + * `network`, `timeout`, `server-error` or `service-unavailable`; a refused + * message, a rate limit or a caller's abort says nothing of its health. + * A thrown error, other than the caller's abort, always counts. + */ + readonly isFailure?: (receipt: Receipt) => boolean; + /** + * Hears every change of state; one that throws becomes a process warning + * (`MAILER_BREAKER_LISTENER_FAILED`). + */ + readonly onStateChange?: (state: CircuitState, previous: CircuitState) => void; + /** The clock, in milliseconds. */ + readonly now?: () => number; +} + +/** + * The options of {@link CircuitBreakerTransport}, with their defaults. + */ +export interface ResolvedCircuitBreakerConfig { + readonly failureThreshold: number; + readonly resetTimeoutMs: number; + readonly isFailure?: (receipt: Receipt) => boolean; + readonly onStateChange?: (state: CircuitState, previous: CircuitState) => void; + readonly now: () => number; +} + +/** + * A transport that stops sending through the one it wraps after it failed + * `failureThreshold` times in a row. While open, a send is answered at once + * with a retryable `service-unavailable` receipt (code `breaker.open`) and + * the time left in `retryAfterMilliseconds`, which `./pool` fails over with + * and `./retry` waits for; after `resetTimeoutMs` one probe goes through, + * whose success closes the circuit and whose failure opens it again. The + * state is that of the instance. + * + * @example + * ```js + * const { CircuitBreakerTransport } = require('@alexify/mailer/breaker'); + * const guarded = new CircuitBreakerTransport(transport, { failureThreshold: 5 }); + * ``` + */ +export declare class CircuitBreakerTransport + implements RawTransport, VerifiableTransport, AsyncDisposable +{ + readonly id: TProviderId; + /** The resolved options. */ + readonly config: ResolvedCircuitBreakerConfig; + /** + * @param transport The transport to guard. + * @param config The options. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport: Transport, config?: CircuitBreakerConfig); + /** The state of the circuit, `half-open` when the next send would be a probe. */ + readonly state: CircuitState; + /** Sends one message, unless the circuit is open. */ + send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends a raw message, unless the circuit is open. A wrapped transport + * without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Sends messages through the `sendMany()` of the wrapped transport while the + * circuit is closed; one read while it is open is answered at once, the + * receipts in the order of the messages. + */ + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the wrapped transport, whatever the state of the circuit. */ + verify(options?: TransportOptions): Promise; + /** Disposes of the wrapped transport when it supports disposal. */ + [Symbol.asyncDispose](): Promise; +} diff --git a/src/breaker/index.js b/src/breaker/index.js new file mode 100644 index 0000000..fe00ccc --- /dev/null +++ b/src/breaker/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/middleware/circuit-breaker.ts): the entry of +// the circuit breaker. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { CircuitBreakerTransport } = require('./breaker-transport.js'); + +module.exports = { CircuitBreakerTransport }; diff --git a/src/internal/callback.js b/src/internal/callback.js new file mode 100644 index 0000000..2b8bdd4 --- /dev/null +++ b/src/internal/callback.js @@ -0,0 +1,19 @@ +// Not ported: a function the application passes to a decorator (a listener, a +// hook, a store) that throws: the send and its receipt go on, and the failure +// becomes a process warning rather than vanishing, as a logger's does. + +/** + * Emits a process warning for a function of the application that threw. + * + * @param what What threw, such as "The onStateChange of CircuitBreakerTransport". + * @param error What it threw. + * @param code The code of the warning. + * @internal + */ +const warnThrown = (what, error, code) => { + process.emitWarning(`${what} threw: ${error instanceof Error ? error.message : String(error)}`, { + code, + }); +}; + +module.exports = { warnThrown }; diff --git a/src/internal/gate.js b/src/internal/gate.js new file mode 100644 index 0000000..5411b0c --- /dev/null +++ b/src/internal/gate.js @@ -0,0 +1,119 @@ +// Not ported: `sendMany()` of a decorator that answers some messages itself (a +// refusal, a stored receipt) and hands the others to the transport it wraps, +// through that transport's own `sendMany()` so its batching is kept, with the +// receipts in the order of the messages. + +/** + * A first-in first-out queue whose `shift` takes constant time. + * + * @internal + */ +class Fifo { + #items = []; + #head = 0; + + get size() { + return this.#items.length - this.#head; + } + + push(item) { + this.#items.push(item); + } + + shift() { + const item = this.#items[this.#head]; + this.#items[this.#head] = undefined; + this.#head++; + // The consumed half is dropped once it is the larger part. + if (this.#head > 1024 && this.#head * 2 > this.#items.length) { + this.#items = this.#items.slice(this.#head); + this.#head = 0; + } + return item; + } +} + +const iteratorOf = (messages) => + typeof messages[Symbol.asyncIterator] === 'function' + ? messages[Symbol.asyncIterator]() + : messages[Symbol.iterator](); + +/** + * Sends messages through a gate. `gate.admit(message)` decides each one: `{ + * receipt }` answers it in place; `{ message, …state }` hands `message` to the + * wrapped transport. The admitted ones go through `transport.sendMany()` in + * runs, a run ending at a message answered in place, and each receipt goes + * through `gate.settle(receipt, admitted)` before it is yielded. When the + * wrapped `sendMany()` throws, `gate.fail(error, admitted)` hears of each + * message still waiting for its receipt. + * + * @param transport The wrapped transport. + * @param messages The messages, sync or async. + * @param options The caller's options, passed on. + * @param gate `admit`, and optional `settle` and `fail`. + * @returns The receipts, one per message, in order. + * @internal + */ +const gatedSendMany = async function* (transport, messages, options, gate) { + options?.signal?.throwIfAborted(); + const iterator = iteratorOf(messages); + let first; + try { + for (;;) { + if (first === undefined) { + const step = await iterator.next(); + if (step.done) return; + const decision = await gate.admit(step.value); + if (decision.receipt !== undefined) { + yield decision.receipt; + continue; + } + first = decision; + } + const run = { iterator, first, fed: new Fifo(), refusal: undefined, exhausted: false }; + first = undefined; + yield* sendRun(transport, run, options, gate); + if (run.refusal !== undefined) yield run.refusal; + if (run.exhausted) return; + } + } finally { + await iterator.return?.(); + } +}; + +// The admitted messages of a run, read until one is answered in place or the +// input ends. +const feedRun = async function* (run, gate) { + run.fed.push(run.first); + yield run.first.message; + for (;;) { + const step = await run.iterator.next(); + if (step.done) { + run.exhausted = true; + return; + } + const decision = await gate.admit(step.value); + if (decision.receipt !== undefined) { + run.refusal = decision.receipt; + return; + } + run.fed.push(decision); + yield decision.message; + } +}; + +const sendRun = async function* (transport, run, options, gate) { + try { + for await (const receipt of transport.sendMany(feedRun(run, gate), options)) { + const admitted = run.fed.shift(); + yield gate.settle === undefined ? receipt : await gate.settle(receipt, admitted); + } + } catch (error) { + if (gate.fail !== undefined) { + while (run.fed.size > 0) await gate.fail(error, run.fed.shift()); + } + throw error; + } +}; + +module.exports = { Fifo, gatedSendMany }; diff --git a/tests/helpers/decorators/fake.js b/tests/helpers/decorators/fake.js new file mode 100644 index 0000000..1fbca6a --- /dev/null +++ b/tests/helpers/decorators/fake.js @@ -0,0 +1,62 @@ +// Not ported: a transport for the tests of the decorators, which answers each +// message with the receipt `answer(message, index)` gives, or throws what it +// returns as `{ throw }`. + +const sent = (messageId) => ({ successful: true, messageId, provider: 'fake' }); + +const failed = (category, extra = {}) => ({ + successful: false, + errorMessages: [`${category} failure`], + errors: [{ message: `${category} failure`, category, retryable: true, provider: 'fake' }], + retryable: true, + provider: 'fake', + ...extra, +}); + +/** + * A fake transport. `calls` holds what each send was given; `sendMany` + * reads its input `readAhead` messages ahead, as a batching transport does. + */ +const fakeTransport = (answer = (_message, index) => sent(`m${index}`), extras = {}) => { + const calls = []; + const respond = async (message, options, operation) => { + const index = calls.length; + calls.push({ message, options, operation }); + const result = await answer(message, index, options); + if (result?.throw !== undefined) throw result.throw; + return result; + }; + const transport = { + id: 'fake', + calls, + send: (message, options) => respond(message, options, 'send'), + sendRaw: (message, options) => respond(message, options, 'sendRaw'), + async *sendMany(messages, options) { + const readAhead = extras.readAhead ?? 0; + const buffer = []; + for await (const message of messages) { + buffer.push(message); + if (buffer.length > readAhead) yield await respond(buffer.shift(), options, 'sendMany'); + } + while (buffer.length > 0) yield await respond(buffer.shift(), options, 'sendMany'); + }, + verified: 0, + async verify() { + transport.verified++; + }, + disposed: 0, + async [Symbol.asyncDispose]() { + transport.disposed++; + }, + ...extras.overrides, + }; + return transport; +}; + +const collect = async (iterable) => { + const items = []; + for await (const item of iterable) items.push(item); + return items; +}; + +module.exports = { collect, failed, fakeTransport, sent }; diff --git a/tests/types/breaker.test-d.ts b/tests/types/breaker.test-d.ts new file mode 100644 index 0000000..19d8044 --- /dev/null +++ b/tests/types/breaker.test-d.ts @@ -0,0 +1,35 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type Transport, + type VerifiableTransport, +} from '../../index.js'; +import { CircuitBreakerTransport, type CircuitState } from '../../src/breaker/index.js'; + +declare const inner: Transport<'ses'>; +const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 3, + resetTimeoutMs: 10_000, + isFailure: (receipt) => !receipt.successful, + onStateChange: (state, previous) => { + expectType(state); + expectType(previous); + }, + now: () => Date.now(), +}); +expectType<'ses'>(breaker.id); +expectType(breaker.state); +expectAssignable>(breaker); +expectAssignable>(breaker); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(breaker.send(message)); +expectType>>(breaker.sendMany([message])); +new CircuitBreakerTransport(inner); +expectError(new CircuitBreakerTransport(inner, { failureThreshold: '5' })); diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js new file mode 100644 index 0000000..eb4805c --- /dev/null +++ b/tests/unit/breaker.test.js @@ -0,0 +1,258 @@ +// Not ported: tests of ./breaker, ported from unemail's circuit breaker, +// against a fake transport. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { CircuitBreakerTransport } = require('../../src/breaker/index.js'); +const { collect, failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const message = (subject = 'Hello') => + createMessage({ from: 'a@example.com', to: 'b@example.com', subject, content: { text: 'x' } }); + +const clock = () => { + let time = 0; + return { now: () => time, advance: (ms) => (time += ms) }; +}; + +const deferred = () => { + let resolve; + const promise = new Promise((_resolve) => { + resolve = _resolve; + }); + return { promise, resolve }; +}; + +test('Breaker: failures of the provider in a row open the circuit; others do not count', async () => { + const answers = ['network', 'timeout', 'validation', 'server-error', 'rate-limit', 'network']; + const inner = fakeTransport((_m, index) => + index < answers.length ? failed(answers[index]) : sent('ok'), + ); + const time = clock(); + const states = []; + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 3, + now: time.now, + onStateChange: (state, previous) => states.push(`${previous}->${state}`), + }); + // network, timeout: 2; validation answers, so the count starts again; + // server-error: 1; rate-limit is no failure of health: 0; network: 1. + for (let i = 0; i < answers.length; i++) await breaker.send(message()); + assert.strictEqual(breaker.state, 'closed'); + inner.calls.length = 0; + const failing = fakeTransport(() => failed('service-unavailable')); + const tripped = new CircuitBreakerTransport(failing, { failureThreshold: 2, now: time.now }); + await tripped.send(message()); + await tripped.send(message()); + assert.strictEqual(tripped.state, 'open'); + assert.deepStrictEqual(states, []); +}); + +test('Breaker: an open circuit answers at once, with the time left', async () => { + const time = clock(); + const inner = fakeTransport(() => failed('network')); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 10_000, + now: time.now, + }); + await breaker.send(message()); + time.advance(2_500); + const refused = await breaker.send(message()); + assert.strictEqual(inner.calls.length, 1); + assert.strictEqual(refused.successful, false); + assert.strictEqual(refused.retryable, true); + assert.strictEqual(refused.provider, 'fake'); + assert.strictEqual(refused.errors[0].category, 'service-unavailable'); + assert.strictEqual(refused.errors[0].code, 'breaker.open'); + assert.strictEqual(refused.errors[0].retryAfterMilliseconds, 7_500); + assert.match(refused.errorMessages[0], /circuit of fake is open after 1 failures/); +}); + +test('Breaker: half-open lets one probe through, and its success closes the circuit', async () => { + const time = clock(); + const probe = deferred(); + let failing = true; + const inner = fakeTransport(() => (failing ? failed('network') : probe.promise)); + const states = []; + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 1_000, + now: time.now, + onStateChange: (state, previous) => states.push(`${previous}->${state}`), + }); + await breaker.send(message()); + time.advance(1_000); + assert.strictEqual(breaker.state, 'half-open'); + failing = false; + const first = breaker.send(message('probe')); + // Changed in the port: unemail let every send through while half-open. + const second = await breaker.send(message('during the probe')); + assert.strictEqual(second.errors[0].code, 'breaker.open'); + assert.strictEqual(second.errors[0].retryAfterMilliseconds, undefined); + assert.strictEqual(inner.calls.length, 2); + probe.resolve(sent('probe-ok')); + assert.strictEqual((await first).messageId, 'probe-ok'); + assert.strictEqual(breaker.state, 'closed'); + assert.deepStrictEqual(states, ['closed->open', 'open->half-open', 'half-open->closed']); +}); + +test('Breaker: a failed or thrown probe opens it again; an abort frees the probe', async () => { + const time = clock(); + const outcomes = [failed('network'), failed('timeout'), { throw: new Error('socket hang up') }]; + const inner = fakeTransport((_m, index, options) => { + if (index < outcomes.length) return outcomes[index]; + return new Promise((_resolve, reject) => { + options.signal.addEventListener('abort', () => reject(options.signal.reason)); + }); + }); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 1_000, + now: time.now, + }); + await breaker.send(message()); + time.advance(1_000); + await breaker.send(message()); + assert.strictEqual(breaker.state, 'open'); + time.advance(1_000); + await assert.rejects(breaker.send(message()), /socket hang up/); + assert.strictEqual(breaker.state, 'open'); + time.advance(1_000); + const controller = new AbortController(); + const aborted = breaker.send(message(), { signal: controller.signal }); + controller.abort(new Error('caller gave up')); + await assert.rejects(aborted, /caller gave up/); + // The abort learned nothing: the next send is a probe again. + assert.strictEqual(breaker.state, 'half-open'); + await assert.rejects( + breaker.send(message(), { signal: AbortSignal.abort(new Error('already')) }), + /already/, + ); +}); + +test('Breaker: sendMany keeps the batching of the wrapped transport and stops when it opens', async () => { + const time = clock(); + const inner = fakeTransport((_m, index) => (index < 2 ? sent(`m${index}`) : failed('network')), { + readAhead: 1, + }); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 2, + resetTimeoutMs: 1_000, + now: time.now, + }); + const messages = []; + for (let i = 0; i < 7; i++) messages.push(message(`m${i}`)); + const receipts = await collect(breaker.sendMany(messages)); + assert.strictEqual(receipts.length, 7); + // Two go, two fail and open the circuit; the one read ahead goes too. + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful || receipt.errors[0].category), + [true, true, 'network', 'network', 'network', 'service-unavailable', 'service-unavailable'], + ); + assert.ok(inner.calls.every((call) => call.operation === 'sendMany')); + assert.strictEqual(inner.calls.length, 5); + // Half-open: the first message is the probe, the others wait for it. + time.advance(1_000); + const answer = await collect(breaker.sendMany([message('p'), message('q')])); + assert.deepStrictEqual( + answer.map((receipt) => receipt.successful || receipt.errors[0].code || 'network'), + ['network', 'breaker.open'], + ); + assert.strictEqual(breaker.state, 'open'); +}); + +test('Breaker: a sendMany that throws counts, and fails the messages still out', async () => { + const time = clock(); + const inner = fakeTransport(() => sent('x'), { + overrides: { + // oxlint-disable-next-line require-yield + async *sendMany(messages) { + await messages[Symbol.asyncIterator]().next(); + throw new Error('batch exploded'); + }, + }, + }); + const breaker = new CircuitBreakerTransport(inner, { failureThreshold: 1, now: time.now }); + await assert.rejects(collect(breaker.sendMany([message()])), /batch exploded/); + assert.strictEqual(breaker.state, 'open'); + const empty = await collect(breaker.sendMany([])); + assert.deepStrictEqual(empty, []); +}); + +test('Breaker: a custom isFailure, sendRaw, verify and dispose', async () => { + const time = clock(); + const inner = fakeTransport(() => failed('rejected')); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + isFailure: (receipt) => !receipt.successful, + now: time.now, + }); + const raw = { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }; + await breaker.sendRaw(raw); + assert.strictEqual(inner.calls[0].operation, 'sendRaw'); + assert.strictEqual(breaker.state, 'open'); + assert.strictEqual((await breaker.sendRaw(raw)).errors[0].code, 'breaker.open'); + await breaker.verify(); + assert.strictEqual(inner.verified, 1); + await breaker[Symbol.asyncDispose](); + assert.strictEqual(inner.disposed, 1); + const plain = { id: 'plain', send: async () => sent('p'), async *sendMany() {} }; + const wrapped = new CircuitBreakerTransport(plain); + assert.strictEqual((await wrapped.sendRaw(raw)).errors[0].category, 'configuration'); + await wrapped.verify(); + await wrapped[Symbol.asyncDispose](); + let disposedSync = 0; + await new CircuitBreakerTransport({ ...plain, [Symbol.dispose]: () => disposedSync++ })[ + Symbol.asyncDispose + ](); + assert.strictEqual(disposedSync, 1); + await assert.rejects(wrapped.sendRaw(raw, { signal: AbortSignal.abort(new Error('no')) }), /no/); +}); + +test('Breaker: a listener that throws is a warning, and the options are checked', async (t) => { + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const breaker = new CircuitBreakerTransport( + fakeTransport(() => failed('network')), + { + failureThreshold: 1, + onStateChange: () => { + throw new Error('listener broke'); + }, + }, + ); + const receipt = await breaker.send(message()); + assert.strictEqual(receipt.errors[0].category, 'network'); + assert.deepStrictEqual(warnings, [ + [ + 'The onStateChange of CircuitBreakerTransport threw: listener broke', + 'MAILER_BREAKER_LISTENER_FAILED', + ], + ]); + const inner = fakeTransport(); + assert.throws( + () => new CircuitBreakerTransport(inner, { failureThreshold: 0 }), + /failureThreshold/, + ); + assert.throws( + () => new CircuitBreakerTransport(inner, { failureThreshold: 1.5 }), + /failureThreshold/, + ); + assert.throws(() => new CircuitBreakerTransport(inner, { resetTimeoutMs: 0 }), /resetTimeoutMs/); + assert.throws( + () => new CircuitBreakerTransport(inner, { resetTimeoutMs: Infinity }), + /resetTimeoutMs/, + ); + assert.throws( + () => new CircuitBreakerTransport(inner, { resetTimeoutMs: '1' }), + /resetTimeoutMs/, + ); + assert.throws( + () => new CircuitBreakerTransport(inner, { isFailure: true }), + /isFailure must be a function/, + ); + const defaults = new CircuitBreakerTransport(inner); + assert.strictEqual(defaults.config.failureThreshold, 5); + assert.strictEqual(defaults.config.resetTimeoutMs, 30_000); + assert.strictEqual(defaults.id, 'fake'); +}); diff --git a/tests/unit/gate.test.js b/tests/unit/gate.test.js new file mode 100644 index 0000000..ad80bbb --- /dev/null +++ b/tests/unit/gate.test.js @@ -0,0 +1,65 @@ +// Not ported: tests of the gated sendMany the decorators share. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { Fifo, gatedSendMany } = require('../../src/internal/gate.js'); +const { collect, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +test('Gate: answers in place keep their order among those of the wrapped transport', async () => { + const inner = fakeTransport((message) => sent(`sent-${message}`), { readAhead: 2 }); + const receipts = await collect( + gatedSendMany(inner, [1, 2, 3, 4, 5, 6, 7], undefined, { + admit: (n) => (n % 3 === 0 ? { receipt: sent(`own-${n}`) } : { message: n }), + }), + ); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.messageId), + ['sent-1', 'sent-2', 'own-3', 'sent-4', 'sent-5', 'own-6', 'sent-7'], + ); +}); + +test('Gate: a long batch goes through, and a break closes the input', async () => { + const inner = fakeTransport((message) => sent(String(message))); + const many = []; + for (let i = 0; i < 3000; i++) many.push(i); + let settled = 0; + const receipts = await collect( + gatedSendMany(inner, many, undefined, { + admit: (n) => ({ message: n }), + settle: (receipt) => { + settled++; + return receipt; + }, + }), + ); + assert.strictEqual(receipts.length, 3000); + assert.strictEqual(receipts[2999].messageId, '2999'); + assert.strictEqual(settled, 3000); + let closed = false; + const endless = { + [Symbol.asyncIterator]() { + let n = 0; + return { + next: async () => ({ done: false, value: n++ }), + return: async () => { + closed = true; + return { done: true }; + }, + }; + }, + }; + for await (const receipt of gatedSendMany(inner, endless, undefined, { + admit: (n) => ({ message: n }), + })) { + if (receipt.messageId === '3') break; + } + assert.strictEqual(closed, true); +}); + +test('Gate: the queue keeps its order across its compaction', () => { + const queue = new Fifo(); + for (let i = 0; i < 5000; i++) queue.push(i); + for (let i = 0; i < 4000; i++) assert.strictEqual(queue.shift(), i); + assert.strictEqual(queue.size, 1000); + queue.push(5000); + assert.strictEqual(queue.shift(), 4000); +}); From 79d82d69e3c483ab86ec9175ba4297dc7dab390f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:38:04 +0300 Subject: [PATCH 036/143] feat(ratelimit): a token-bucket rate limit decorator Ported from unemail's rate limit as a decorator of one transport: a bucket per `key` holds `burst` tokens and gains `limit` every `intervalMs`, and a message takes `cost(message)`. In `wait` mode the waiters are served first in, first out from one timer, and an abort of the caller takes a message out of the queue, where unemail's sleep did not hear the signal; a message the queue ahead would hold past `maxWaitMs`, counted from its arrival, or any that would wait in `reject` mode, gets a retryable `rate-limit` receipt with the wait; a cost over `burst`, which unemail cut to it, is refused (U11). Buckets full again and idle are dropped past `maxKeys`. `sendMany` goes through the wrapped `sendMany`, a message handed to it once it has its tokens. The option checks of the decorators move to `internal/options.js`, which carries no URL check into their bundles. The budget is 2.5 KB. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/ratelimit.md | 66 ++++++++ package.json | 4 + scripts/size-budgets.json | 3 +- src/breaker/config.js | 37 ++--- src/internal/gate.js | 4 + src/internal/options.js | 40 +++++ src/ratelimit/config.js | 42 +++++ src/ratelimit/index.d.ts | 131 ++++++++++++++++ src/ratelimit/index.js | 7 + src/ratelimit/ratelimit-transport.js | 227 +++++++++++++++++++++++++++ tests/types/ratelimit.test-d.ts | 35 +++++ tests/unit/ratelimit.test.js | 210 +++++++++++++++++++++++++ 15 files changed, 789 insertions(+), 25 deletions(-) create mode 100644 docs/transports/ratelimit.md create mode 100644 src/internal/options.js create mode 100644 src/ratelimit/config.js create mode 100644 src/ratelimit/index.d.ts create mode 100644 src/ratelimit/index.js create mode 100644 src/ratelimit/ratelimit-transport.js create mode 100644 tests/types/ratelimit.test-d.ts create mode 100644 tests/unit/ratelimit.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 340e09b..407695b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -556,6 +556,12 @@ fails on upstream's behavior): through, answers the others as while open, counts only network, timeout, server and unavailability failures and a thrown error that is not the caller's abort, and ignores a late result of a send made in another state. +- U11. unemail (`src/middleware/rate-limit.ts`): the wait for tokens did not hear the caller's + signal, so an aborted send kept its place and its tokens; `maxWaitMs` counted from the turn of + a message, not its arrival, so one queued behind others waited past it; and a batch costing + more than the capacity was cut to it and went through, over the limit. `./ratelimit` serves the + waiters in turn from one timer and drops one whose caller aborts, refuses at once a message the + queue ahead would hold past `maxWaitMs`, and refuses a cost over `burst`. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ddfb14a..cd79279 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -107,6 +107,7 @@ const guide = [ { text: 'OpenTelemetry', link: '/transports/otel' }, { text: 'Logger', link: '/transports/logger' }, { text: 'Circuit breaker', link: '/transports/breaker' }, + { text: 'Rate limit', link: '/transports/ratelimit' }, ], }, ]; diff --git a/docs/api/exports.md b/docs/api/exports.md index ec28f3f..b994581 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -70,6 +70,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/otel` | `OpenTelemetryTransport`, `createOpenTelemetryTransport`, `createEmailAttributeExtractor`, `createErrorClassifier`, `defaultErrorClassifier` | [OpenTelemetry](/transports/otel) | | `@alexify/mailer/logger` | `LoggerTransport` | [Logger](/transports/logger) | | `@alexify/mailer/breaker` | `CircuitBreakerTransport` | [Circuit breaker](/transports/breaker) | +| `@alexify/mailer/ratelimit` | `RateLimitTransport` | [Rate limit](/transports/ratelimit) | The types of each subpath (configurations, receipts with provider details, the structural `TracerProvider`, `MeterProvider` and `Logger` of the decorators) are in diff --git a/docs/transports/ratelimit.md b/docs/transports/ratelimit.md new file mode 100644 index 0000000..7353732 --- /dev/null +++ b/docs/transports/ratelimit.md @@ -0,0 +1,66 @@ +# Rate limit + +`RateLimitTransport` sends through a transport at most `limit` messages every +`intervalMs`, with bursts up to `burst`: a token bucket in front of it, per key. +It comes from the rate limit of unemail. + +## Basic usage + +```js +const { RateLimitTransport } = require('@alexify/mailer/ratelimit'); + +// SES at 14 messages a second, which it counts per recipient. +const transport = new RateLimitTransport(ses, { + limit: 14, + cost: (message) => + message.recipients.length + message.ccRecipients.length + message.bccRecipients.length, +}); +``` + +## Waiting and refusing + +A bucket holds `burst` tokens and gains `limit` every `intervalMs`; a message +takes `cost(message)` of them, 1 without `cost`. + +- In `wait` mode, the default, a message waits for its tokens, first in, first + out, behind those already waiting. An abort of the caller takes it out of the + queue and rejects with the signal's reason. +- A message the queue ahead of it would hold longer than `maxWaitMs`, counted + from its arrival, is answered at once, as is any that would wait in `reject` + mode: a retryable `rate-limit` receipt (code `ratelimit.limited`) with the + wait in `retryAfterMilliseconds`, which `./retry` waits for. +- A cost over `burst`, which no bucket could ever hold, is refused as + `configuration`, not retryable. + +## Per key + +`key(message)` chooses the bucket, such as a tenant's, so one tenant's burst +does not hold back the others. Buckets that are full again and have no waiter +are dropped once there are `maxKeys` of them, as one made anew would be the +same. + +```js +const perTenant = new RateLimitTransport(transport, { + limit: 10, + key: (message) => message.metadata?.tenant ?? 'default', +}); +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `limit` | (required) | The tokens a bucket gains every `intervalMs`. | +| `intervalMs` | `1000` | The interval of `limit`, in milliseconds. | +| `burst` | `limit` | The tokens a bucket holds. | +| `mode` | `'wait'` | `'wait'` or `'reject'`. | +| `maxWaitMs` | `30000` | The longest wait, from the arrival. | +| `key` | one bucket | The bucket of a message. | +| `cost` | `1` | The tokens a message takes. | +| `maxKeys` | `10000` | The buckets kept before full idle ones are dropped. | +| `now` | `Date.now` | The clock. | + +`sendMany()` goes through the `sendMany()` of the wrapped transport, each +message handed to it once it has its tokens, so a batch is throttled like as +many sends; a transport that reads a batch ahead may still send it at once. +`sendRaw()` is throttled as `send()` is, and `verify()` takes no tokens. diff --git a/package.json b/package.json index 23615e4..c77ef41 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,10 @@ "./breaker": { "types": "./src/breaker/index.d.ts", "default": "./src/breaker/index.js" + }, + "./ratelimit": { + "types": "./src/ratelimit/index.d.ts", + "default": "./src/ratelimit/index.js" } }, "directories": { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d05ac19..9dc54d9 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -28,5 +28,6 @@ "./mock": 2, "./otel": 5, "./logger": 2.5, - "./breaker": 2 + "./breaker": 2, + "./ratelimit": 2.5 } diff --git a/src/breaker/config.js b/src/breaker/config.js index b50063c..3ee90ed 100644 --- a/src/breaker/config.js +++ b/src/breaker/config.js @@ -2,12 +2,11 @@ // of the circuit breaker. // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind -const checkFunction = (value, name) => { - if (value !== undefined && typeof value !== 'function') { - throw new TypeError(`${name} must be a function.`); - } - return value; -}; +const { + checkOptionalFunction, + checkPositiveInteger, + checkPositiveNumber, +} = require('../internal/options.js'); /** * Resolves the options of the circuit breaker. @@ -17,22 +16,12 @@ const checkFunction = (value, name) => { * @throws {TypeError} If an option is invalid. * @internal */ -const createBreakerConfig = (config = {}) => { - const failureThreshold = config.failureThreshold ?? 5; - if (!Number.isInteger(failureThreshold) || failureThreshold < 1) { - throw new TypeError(`failureThreshold must be a positive integer: ${String(failureThreshold)}`); - } - const resetTimeoutMs = config.resetTimeoutMs ?? 30000; - if (typeof resetTimeoutMs !== 'number' || !(resetTimeoutMs > 0) || resetTimeoutMs === Infinity) { - throw new TypeError(`resetTimeoutMs must be a positive number: ${String(resetTimeoutMs)}`); - } - return { - failureThreshold, - resetTimeoutMs, - isFailure: checkFunction(config.isFailure, 'isFailure'), - onStateChange: checkFunction(config.onStateChange, 'onStateChange'), - now: checkFunction(config.now, 'now') ?? Date.now, - }; -}; +const createBreakerConfig = (config = {}) => ({ + failureThreshold: checkPositiveInteger(config.failureThreshold ?? 5, 'failureThreshold'), + resetTimeoutMs: checkPositiveNumber(config.resetTimeoutMs ?? 30000, 'resetTimeoutMs'), + isFailure: checkOptionalFunction(config.isFailure, 'isFailure'), + onStateChange: checkOptionalFunction(config.onStateChange, 'onStateChange'), + now: checkOptionalFunction(config.now, 'now') ?? Date.now, +}); -module.exports = { checkFunction, createBreakerConfig }; +module.exports = { createBreakerConfig }; diff --git a/src/internal/gate.js b/src/internal/gate.js index 5411b0c..a4a8f35 100644 --- a/src/internal/gate.js +++ b/src/internal/gate.js @@ -20,6 +20,10 @@ class Fifo { this.#items.push(item); } + peek() { + return this.#items[this.#head]; + } + shift() { const item = this.#items[this.#head]; this.#items[this.#head] = undefined; diff --git a/src/internal/options.js b/src/internal/options.js new file mode 100644 index 0000000..f75de3b --- /dev/null +++ b/src/internal/options.js @@ -0,0 +1,40 @@ +// Not ported: the checks of the options of the decorators, apart from those of +// the HTTP providers so that a decorator's bundle carries no URL checks. + +/** + * Checks an optional function. + * + * @internal + */ +const checkOptionalFunction = (value, name) => { + if (value !== undefined && typeof value !== 'function') { + throw new TypeError(`${name} must be a function.`); + } + return value; +}; + +/** + * Checks a finite number above zero. + * + * @internal + */ +const checkPositiveNumber = (value, name) => { + if (typeof value !== 'number' || !(value > 0) || value === Infinity) { + throw new TypeError(`${name} must be a positive number: ${String(value)}`); + } + return value; +}; + +/** + * Checks a whole number above zero. + * + * @internal + */ +const checkPositiveInteger = (value, name) => { + if (!Number.isInteger(value) || value < 1) { + throw new TypeError(`${name} must be a positive integer: ${String(value)}`); + } + return value; +}; + +module.exports = { checkOptionalFunction, checkPositiveInteger, checkPositiveNumber }; diff --git a/src/ratelimit/config.js b/src/ratelimit/config.js new file mode 100644 index 0000000..f8c6de2 --- /dev/null +++ b/src/ratelimit/config.js @@ -0,0 +1,42 @@ +// Ported from unemail v0.7.0 (src/middleware/rate-limit.ts): the options of +// the token bucket. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { + checkOptionalFunction, + checkPositiveInteger, + checkPositiveNumber, +} = require('../internal/options.js'); + +/** + * Resolves the options of the rate limiter. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createRateLimitConfig = (config = {}) => { + const limit = checkPositiveNumber(config.limit, 'limit'); + const mode = config.mode ?? 'wait'; + if (mode !== 'wait' && mode !== 'reject') { + throw new TypeError(`mode must be 'wait' or 'reject': ${String(mode)}`); + } + const maxWaitMs = config.maxWaitMs ?? 30000; + if (typeof maxWaitMs !== 'number' || !(maxWaitMs >= 0)) { + throw new TypeError(`maxWaitMs must be a non-negative number: ${String(maxWaitMs)}`); + } + return { + limit, + intervalMs: checkPositiveNumber(config.intervalMs ?? 1000, 'intervalMs'), + burst: checkPositiveNumber(config.burst ?? limit, 'burst'), + mode, + maxWaitMs, + key: checkOptionalFunction(config.key, 'key'), + cost: checkOptionalFunction(config.cost, 'cost'), + maxKeys: checkPositiveInteger(config.maxKeys ?? 10000, 'maxKeys'), + now: checkOptionalFunction(config.now, 'now') ?? Date.now, + }; +}; + +module.exports = { createRateLimitConfig }; diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts new file mode 100644 index 0000000..808a009 --- /dev/null +++ b/src/ratelimit/index.d.ts @@ -0,0 +1,131 @@ +// Types ported from unemail v0.7.0 (src/middleware) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +/// + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The options of {@link RateLimitTransport}. + */ +export interface RateLimitConfig { + /** The tokens a bucket gains every `intervalMs`: the sustained rate. */ + readonly limit: number; + /** + * The interval of `limit`, in milliseconds. + * @default 1000 + */ + readonly intervalMs?: number; + /** + * The tokens a bucket holds, the most sent at once after a quiet spell. + * @default limit + */ + readonly burst?: number; + /** + * `wait` holds a message until its tokens come, first in first out; `reject` + * answers at once with a retryable `rate-limit` receipt. + * @default "wait" + */ + readonly mode?: 'wait' | 'reject'; + /** + * The longest a message may wait, counted from its arrival; one the queue + * ahead of it would hold longer is answered at once, as in `reject` mode. + * @default 30000 + */ + readonly maxWaitMs?: number; + /** + * The bucket of a message, such as its tenant; one bucket for all without + * it. + */ + readonly key?: (message: Message | RawMessage) => string; + /** + * The tokens a message takes, such as its number of recipients for SES; + * 1 without it. A cost over `burst` is refused. + */ + readonly cost?: (message: Message | RawMessage) => number; + /** + * The buckets kept before those full and idle are dropped. + * @default 10000 + */ + readonly maxKeys?: number; + /** The clock, in milliseconds. */ + readonly now?: () => number; +} + +/** + * The options of {@link RateLimitTransport}, with their defaults. + */ +export interface ResolvedRateLimitConfig { + readonly limit: number; + readonly intervalMs: number; + readonly burst: number; + readonly mode: 'wait' | 'reject'; + readonly maxWaitMs: number; + readonly key?: (message: Message | RawMessage) => string; + readonly cost?: (message: Message | RawMessage) => number; + readonly maxKeys: number; + readonly now: () => number; +} + +/** + * A transport that sends through the one it wraps at most `limit` messages + * every `intervalMs`, with bursts up to `burst`: a token bucket per `key`, + * its waiters served first in, first out. A message that would wait longer + * than `maxWaitMs`, or any that must wait in `reject` mode, is answered at + * once with a retryable `rate-limit` receipt (code `ratelimit.limited`) and + * the wait in `retryAfterMilliseconds`; an abort of the caller takes a + * waiting message out of the queue. + * + * @example + * ```js + * const { RateLimitTransport } = require('@alexify/mailer/ratelimit'); + * // SES at 14 messages a second, counted per recipient. + * const throttled = new RateLimitTransport(ses, { + * limit: 14, + * cost: (message) => + * message.recipients.length + message.ccRecipients.length + message.bccRecipients.length, + * }); + * ``` + */ +export declare class RateLimitTransport + implements RawTransport, VerifiableTransport, AsyncDisposable +{ + readonly id: TProviderId; + /** The resolved options. */ + readonly config: ResolvedRateLimitConfig; + /** + * @param transport The transport to throttle. + * @param config The options. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport: Transport, config: RateLimitConfig); + /** Sends one message once its bucket has its tokens. */ + send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends a raw message once its bucket has its tokens. A wrapped transport + * without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Sends messages through the `sendMany()` of the wrapped transport, each + * handed to it once its bucket has its tokens; one refused is answered in + * place, the receipts in the order of the messages. + */ + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the wrapped transport, without taking tokens. */ + verify(options?: TransportOptions): Promise; + /** Disposes of the wrapped transport when it supports disposal. */ + [Symbol.asyncDispose](): Promise; +} diff --git a/src/ratelimit/index.js b/src/ratelimit/index.js new file mode 100644 index 0000000..4c0b362 --- /dev/null +++ b/src/ratelimit/index.js @@ -0,0 +1,7 @@ +// Ported from unemail v0.7.0 (src/middleware/rate-limit.ts): the entry of the +// rate limiter. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { RateLimitTransport } = require('./ratelimit-transport.js'); + +module.exports = { RateLimitTransport }; diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js new file mode 100644 index 0000000..dc03319 --- /dev/null +++ b/src/ratelimit/ratelimit-transport.js @@ -0,0 +1,227 @@ +// Ported from unemail v0.7.0 (src/middleware/rate-limit.ts): a token bucket +// in front of a transport, which waits for its tokens or refuses. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the waiters are served first in, first out by one +// timer, and a waiter whose caller aborts leaves the queue at once, where +// unemail's sleep did not hear the signal; the longest wait is counted from +// the arrival, so a message is refused at once when the queue ahead of it +// would hold it longer, where unemail started counting only at its turn; a +// cost over the capacity is refused, where unemail cut it to the capacity and +// sent more than the limit; and the buckets of the keys are bounded. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('../internal/capabilities.js'); +const { Fifo, gatedSendMany } = require('../internal/gate.js'); +const { abortReason } = require('../internal/sleep.js'); +const { createRateLimitConfig } = require('./config.js'); + +const disposeOf = async (transport) => { + const target = Object(transport); + if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); + else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); +}; + +/** + * A transport that sends through the one it wraps at most `limit` messages + * every `intervalMs`, with bursts up to `burst`, a bucket per `key`. + */ +class RateLimitTransport { + id; + config; + wrappedTransport; + #buckets = new Map(); + #sweepAt; + #refillPerMs; + + /** + * @param transport The transport to throttle. + * @param config `limit`, `intervalMs`, `burst`, `mode`, `maxWaitMs`, `key`, + * `cost`, `maxKeys` and `now`. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport, config) { + this.wrappedTransport = transport; + this.id = transport.id; + this.config = createRateLimitConfig(config); + this.#refillPerMs = this.config.limit / this.config.intervalMs; + this.#sweepAt = this.config.maxKeys; + } + + /** + * Sends one message once the bucket has its tokens. + * + * @returns The receipt of the wrapped transport, or a `rate-limit` one when + * the message would wait longer than `maxWaitMs` (at once in `reject` mode). + * @throws The reason of the signal if it aborts while the message waits. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + const refusal = await this.#admit(message, options?.signal); + if (refusal !== undefined) return refusal; + return this.wrappedTransport.send(message, options); + } + + /** Sends a raw message once the bucket has its tokens. */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const refusal = await this.#admit(message, options?.signal); + if (refusal !== undefined) return refusal; + return this.wrappedTransport.sendRaw(message, options); + } + + /** + * Sends messages through the `sendMany()` of the wrapped transport, each + * handed to it once the bucket has its tokens, so a batch is throttled like + * as many sends; one refused is answered in place. + */ + sendMany(messages, options) { + return gatedSendMany(this.wrappedTransport, messages, options, { + admit: async (message) => { + const refusal = await this.#admit(message, options?.signal); + return refusal === undefined ? { message } : { receipt: refusal }; + }, + }); + } + + /** Verifies the wrapped transport, without taking tokens. */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** Disposes of the wrapped transport when it supports disposal. */ + async [Symbol.asyncDispose]() { + await disposeOf(this.wrappedTransport); + } + + // Takes the tokens of a message, waiting for them: `undefined` once taken, + // or the receipt of a refusal. + #admit(message, signal) { + const { cost: costOf, key: keyOf } = this.config; + const cost = costOf === undefined ? 1 : costOf(message); + if (typeof cost !== 'number' || !(cost >= 0) || cost === Infinity) { + return this.#refused(`The cost of a message must be a non-negative number: ${cost}`, { + category: 'configuration', + code: 'ratelimit.invalid_cost', + }); + } + if (cost > this.config.burst) { + return this.#refused( + `The cost ${cost} of the message is over the ${this.config.burst} tokens the bucket holds.`, + { category: 'configuration', code: 'ratelimit.cost_over_capacity' }, + ); + } + const bucket = this.#bucketOf(keyOf === undefined ? '' : String(keyOf(message))); + this.#refill(bucket); + if (bucket.waiters.size === 0 && bucket.tokens >= cost) { + bucket.tokens -= cost; + return undefined; + } + // Served in turn, this one gets its tokens once those ahead have theirs. + const waitMs = Math.max( + 1, + Math.ceil((bucket.queuedCost + cost - bucket.tokens) / this.#refillPerMs), + ); + if (this.config.mode === 'reject' || waitMs > this.config.maxWaitMs) { + return this.#refused( + `The local rate limit of ${this.id} (${this.config.limit} per ${this.config.intervalMs} ms) ` + + `would hold the message ${waitMs} ms.`, + { category: 'rate-limit', retryable: true, retryAfterMilliseconds: waitMs }, + ); + } + return this.#wait(bucket, cost, signal); + } + + #refused(text, options) { + return Promise.resolve( + createFailedReceipt(text, { + provider: this.id, + retryable: false, + code: 'ratelimit.limited', + ...options, + }), + ); + } + + #wait(bucket, cost, signal) { + return new Promise((resolve, reject) => { + const waiter = { cost, resolve, cancelled: false, signal, onAbort: undefined }; + if (signal !== undefined) { + waiter.onAbort = () => { + waiter.cancelled = true; + bucket.queuedCost -= cost; + reject(abortReason(signal)); + this.#pump(bucket); + }; + signal.addEventListener('abort', waiter.onAbort, { once: true }); + } + bucket.waiters.push(waiter); + bucket.queuedCost += cost; + this.#pump(bucket); + }); + } + + // Serves the waiters in turn, and sets one timer for the next. + #pump(bucket) { + clearTimeout(bucket.timer); + bucket.timer = undefined; + this.#refill(bucket); + while (bucket.waiters.size > 0) { + const head = bucket.waiters.peek(); + if (head.cancelled) { + bucket.waiters.shift(); + continue; + } + if (head.cost > bucket.tokens) { + const delay = Math.max(1, Math.ceil((head.cost - bucket.tokens) / this.#refillPerMs)); + bucket.timer = setTimeout(() => this.#pump(bucket), delay); + return; + } + bucket.waiters.shift(); + bucket.tokens -= head.cost; + bucket.queuedCost -= head.cost; + head.signal?.removeEventListener('abort', head.onAbort); + head.resolve(undefined); + } + } + + #refill(bucket) { + const now = this.config.now(); + const elapsed = Math.max(0, now - bucket.last); + bucket.tokens = Math.min(this.config.burst, bucket.tokens + elapsed * this.#refillPerMs); + bucket.last = now; + } + + #bucketOf(key) { + let bucket = this.#buckets.get(key); + if (bucket === undefined) { + if (this.#buckets.size >= this.#sweepAt) this.#sweep(); + bucket = { + tokens: this.config.burst, + last: this.config.now(), + waiters: new Fifo(), + queuedCost: 0, + timer: undefined, + }; + this.#buckets.set(key, bucket); + } + return bucket; + } + + // Drops the buckets that are full again and have no waiter: one made anew + // would be the same. When too few are, the next sweep waits for twice as + // many, so a crowd of busy keys does not sweep on every new one. + #sweep() { + for (const [key, bucket] of this.#buckets) { + this.#refill(bucket); + if (bucket.waiters.size === 0 && bucket.tokens >= this.config.burst) { + this.#buckets.delete(key); + } + } + this.#sweepAt = Math.max(this.config.maxKeys, this.#buckets.size * 2); + } +} + +module.exports = { RateLimitTransport }; diff --git a/tests/types/ratelimit.test-d.ts b/tests/types/ratelimit.test-d.ts new file mode 100644 index 0000000..9b6259c --- /dev/null +++ b/tests/types/ratelimit.test-d.ts @@ -0,0 +1,35 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type Transport, + type VerifiableTransport, +} from '../../index.js'; +import { RateLimitTransport } from '../../src/ratelimit/index.js'; + +declare const inner: Transport<'ses'>; +const limiter = new RateLimitTransport(inner, { + limit: 14, + intervalMs: 1000, + burst: 28, + mode: 'wait', + maxWaitMs: 5000, + key: () => 'tenant', + cost: () => 1, + maxKeys: 100, + now: () => Date.now(), +}); +expectType<'ses'>(limiter.id); +expectAssignable>(limiter); +expectAssignable>(limiter); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(limiter.send(message)); +expectType>>(limiter.sendMany([message])); +expectError(new RateLimitTransport(inner, {})); +expectError(new RateLimitTransport(inner, { limit: 1, mode: 'drop' })); diff --git a/tests/unit/ratelimit.test.js b/tests/unit/ratelimit.test.js new file mode 100644 index 0000000..7cf0f14 --- /dev/null +++ b/tests/unit/ratelimit.test.js @@ -0,0 +1,210 @@ +// Not ported: tests of ./ratelimit, ported from unemail's rate limit, against a +// fake transport, a manual clock and mocked timers. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { RateLimitTransport } = require('../../src/ratelimit/index.js'); +const { collect, fakeTransport } = require('../helpers/decorators/fake.js'); + +const message = (subject = 'Hello', to = 'b@example.com') => + createMessage({ from: 'a@example.com', to, subject, content: { text: 'x' } }); + +// A clock that moves the mocked timers with it. +const timeline = (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let time = 0; + return { + now: () => time, + advance: async (ms) => { + time += ms; + t.mock.timers.tick(ms); + // Lets the promises settled by the timers run, which setImmediate, + // not mocked, follows. + await new Promise((resolve) => setImmediate(resolve)); + }, + }; +}; + +test('RateLimit: a burst goes at once, then one message per refill, in order', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 2, intervalMs: 1000, now: clock.now }); + const order = []; + const sends = []; + for (let i = 0; i < 5; i++) { + sends.push(limiter.send(message(`m${i}`)).then(() => order.push(i))); + } + await clock.advance(0); + assert.deepStrictEqual(order, [0, 1]); + await clock.advance(499); + assert.deepStrictEqual(order, [0, 1]); + await clock.advance(1); + assert.deepStrictEqual(order, [0, 1, 2]); + await clock.advance(1000); + assert.deepStrictEqual(order, [0, 1, 2, 3, 4]); + await Promise.all(sends); + assert.strictEqual(inner.calls.length, 5); +}); + +test('RateLimit: the longest wait counts from the arrival, behind the queue', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { + limit: 1, + intervalMs: 1000, + maxWaitMs: 1500, + now: clock.now, + }); + const first = limiter.send(message()); + const second = limiter.send(message()); + // Changed in the port: unemail counted the wait only from the turn of a + // message, so this one would have waited behind the second as well. + const third = await limiter.send(message()); + assert.strictEqual(third.successful, false); + assert.strictEqual(third.retryable, true); + assert.strictEqual(third.errors[0].category, 'rate-limit'); + assert.strictEqual(third.errors[0].code, 'ratelimit.limited'); + assert.strictEqual(third.errors[0].retryAfterMilliseconds, 2000); + await clock.advance(1000); + await Promise.all([first, second]); + assert.strictEqual(inner.calls.length, 2); +}); + +test('RateLimit: reject mode answers at once, and a cost over the burst is refused', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const recipients = (m) => m.recipients.length; + const limiter = new RateLimitTransport(inner, { + limit: 3, + mode: 'reject', + cost: recipients, + now: clock.now, + }); + assert.strictEqual( + (await limiter.send(message('a', ['x@example.com', 'y@example.com']))).successful, + true, + ); + const limited = await limiter.send(message('b', ['x@example.com', 'y@example.com'])); + assert.strictEqual(limited.errors[0].retryAfterMilliseconds, 334); + // Changed in the port: unemail cut a cost over the capacity to it. + const tooBig = await limiter.send( + message('c', ['a@example.com', 'b@example.com', 'c@example.com', 'd@example.com']), + ); + assert.strictEqual(tooBig.errors[0].category, 'configuration'); + assert.strictEqual(tooBig.errors[0].code, 'ratelimit.cost_over_capacity'); + assert.strictEqual(tooBig.retryable, false); + const invalid = new RateLimitTransport(inner, { limit: 1, cost: () => -1, now: clock.now }); + assert.strictEqual((await invalid.send(message())).errors[0].code, 'ratelimit.invalid_cost'); + assert.strictEqual(inner.calls.length, 1); +}); + +test('RateLimit: an abort takes a waiting message out of the queue', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 1, now: clock.now }); + await limiter.send(message('first')); + const controller = new AbortController(); + const waiting = limiter.send(message('aborted'), { signal: controller.signal }); + const after = limiter.send(message('after')); + controller.abort(new Error('caller left')); + // Changed in the port: unemail's sleep did not hear the signal. + await assert.rejects(waiting, /caller left/); + await clock.advance(1000); + await after; + assert.deepStrictEqual( + inner.calls.map((call) => call.message.subject), + ['first', 'after'], + ); + await assert.rejects( + limiter.send(message(), { signal: AbortSignal.abort(new Error('already')) }), + /already/, + ); +}); + +test('RateLimit: a bucket per key, and full idle ones are dropped', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { + limit: 1, + mode: 'reject', + key: (m) => m.recipients[0].address, + maxKeys: 2, + now: clock.now, + }); + assert.strictEqual((await limiter.send(message('a', 'x@example.com'))).successful, true); + assert.strictEqual((await limiter.send(message('b', 'y@example.com'))).successful, true); + assert.strictEqual((await limiter.send(message('c', 'x@example.com'))).successful, false); + await clock.advance(1000); + // Both buckets are full again: the third key drops them. + assert.strictEqual((await limiter.send(message('d', 'z@example.com'))).successful, true); + assert.strictEqual((await limiter.send(message('e', 'z@example.com'))).successful, false); + // A bucket still waiting to refill is kept, and the next sweep waits longer. + assert.strictEqual((await limiter.send(message('f', 'w@example.com'))).successful, true); + assert.strictEqual((await limiter.send(message('g', 'v@example.com'))).successful, true); +}); + +test('RateLimit: sendMany is throttled like as many sends, through the wrapped sendMany', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(undefined, { readAhead: 1 }); + const limiter = new RateLimitTransport(inner, { + limit: 2, + maxWaitMs: 600, + now: clock.now, + }); + const batch = collect(limiter.sendMany([message('0'), message('1'), message('2'), message('3')])); + await clock.advance(0); + assert.strictEqual(inner.calls.length, 1); + await clock.advance(500); + await clock.advance(500); + const receipts = await batch; + assert.deepStrictEqual( + receipts.map((receipt) => receipt.successful), + [true, true, true, true], + ); + assert.ok(inner.calls.every((call) => call.operation === 'sendMany')); + // Over the longest wait, the rest is answered in place. + const refused = await collect( + new RateLimitTransport(fakeTransport(), { limit: 1, maxWaitMs: 0, now: clock.now }).sendMany([ + message(), + message(), + ]), + ); + assert.deepStrictEqual( + refused.map((receipt) => receipt.successful), + [true, false], + ); +}); + +test('RateLimit: sendRaw, verify, dispose and the options', async (t) => { + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 1, mode: 'reject', now: clock.now }); + const raw = { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }; + assert.strictEqual((await limiter.sendRaw(raw)).successful, true); + assert.strictEqual((await limiter.sendRaw(raw)).errors[0].category, 'rate-limit'); + await limiter.verify(); + assert.strictEqual(inner.verified, 1); + await limiter[Symbol.asyncDispose](); + assert.strictEqual(inner.disposed, 1); + const plain = { id: 'plain', send: async () => ({ successful: true, messageId: 'p' }) }; + const bare = new RateLimitTransport(plain, { limit: 1 }); + assert.strictEqual((await bare.sendRaw(raw)).errors[0].category, 'configuration'); + await bare.verify(); + await bare[Symbol.asyncDispose](); + let disposed = 0; + await new RateLimitTransport({ ...plain, [Symbol.dispose]: () => disposed++ }, { limit: 1 })[ + Symbol.asyncDispose + ](); + assert.strictEqual(disposed, 1); + assert.throws(() => new RateLimitTransport(inner, {}), /limit must be a positive number/); + assert.throws(() => new RateLimitTransport(inner, { limit: 1, mode: 'drop' }), /mode/); + assert.throws(() => new RateLimitTransport(inner, { limit: 1, maxWaitMs: -1 }), /maxWaitMs/); + assert.throws(() => new RateLimitTransport(inner, { limit: 1, maxKeys: 0 }), /maxKeys/); + assert.throws(() => new RateLimitTransport(inner, { limit: 1, key: 'tenant' }), /key must be/); + const defaults = new RateLimitTransport(inner, { limit: 5 }); + assert.strictEqual(defaults.config.burst, 5); + assert.strictEqual(defaults.config.intervalMs, 1000); + assert.strictEqual(defaults.config.maxWaitMs, 30000); + assert.strictEqual(defaults.config.mode, 'wait'); + assert.strictEqual(defaults.id, 'fake'); +}); From 452d7746d8f203f3d9230770423d9a38d5c9f3b4 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:44:20 +0300 Subject: [PATCH 037/143] feat(idempotency): an idempotency decorator with reserved keys Ported from unemail's idempotency middleware as a decorator of one transport. The key of a message (its `idempotencyKey`, or what `key` gives) is reserved in a store before the send: a send under a key in flight is answered as of unknown delivery, one under a key that went with the receipt of that send, and the same key for another message, told by a fingerprint of the message, is refused (U12). A send that failed frees its key; one of unknown delivery, or aborted after it began, keeps it. A store that fails fails the message it was asked about alone, or, after a send that went, becomes a process warning. `createMemoryIdempotencyStore` is the in-process store, bounded by `maxEntries`; a shared store implements `reserve`, `complete` and `release`, `reserve` atomic. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/idempotency.md | 91 +++++++ package.json | 4 + scripts/size-budgets.json | 3 +- src/idempotency/config.js | 38 +++ src/idempotency/fingerprint.js | 70 ++++++ src/idempotency/idempotency-transport.js | 223 +++++++++++++++++ src/idempotency/index.d.ts | 161 ++++++++++++ src/idempotency/index.js | 8 + src/idempotency/memory-store.js | 64 +++++ tests/types/idempotency.test-d.ts | 44 ++++ tests/unit/idempotency.test.js | 302 +++++++++++++++++++++++ 14 files changed, 1016 insertions(+), 1 deletion(-) create mode 100644 docs/transports/idempotency.md create mode 100644 src/idempotency/config.js create mode 100644 src/idempotency/fingerprint.js create mode 100644 src/idempotency/idempotency-transport.js create mode 100644 src/idempotency/index.d.ts create mode 100644 src/idempotency/index.js create mode 100644 src/idempotency/memory-store.js create mode 100644 tests/types/idempotency.test-d.ts create mode 100644 tests/unit/idempotency.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 407695b..f44ea27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -562,6 +562,13 @@ fails on upstream's behavior): more than the capacity was cut to it and went through, over the limit. `./ratelimit` serves the waiters in turn from one timer and drops one whose caller aborts, refuses at once a message the queue ahead would hold past `maxWaitMs`, and refuses a cost over `burst`. +- U12. unemail (`src/middleware/idempotency.ts`): the key was looked up before the send and + written after it, so two concurrent sends under one key both went; the same key for another + message was answered with the first's receipt; and only successes were kept, so a send of + unknown delivery went again on a retry. `./idempotency` reserves the key in the store first + (`reserve`, `complete`, `release`), answers a send in flight as of unknown delivery, refuses a + key whose fingerprint differs, keeps a key whose delivery is unknown, and fails only the + message whose store call failed. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index cd79279..06ae2ca 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -108,6 +108,7 @@ const guide = [ { text: 'Logger', link: '/transports/logger' }, { text: 'Circuit breaker', link: '/transports/breaker' }, { text: 'Rate limit', link: '/transports/ratelimit' }, + { text: 'Idempotency', link: '/transports/idempotency' }, ], }, ]; diff --git a/docs/api/exports.md b/docs/api/exports.md index b994581..d578a1e 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -71,6 +71,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/logger` | `LoggerTransport` | [Logger](/transports/logger) | | `@alexify/mailer/breaker` | `CircuitBreakerTransport` | [Circuit breaker](/transports/breaker) | | `@alexify/mailer/ratelimit` | `RateLimitTransport` | [Rate limit](/transports/ratelimit) | +| `@alexify/mailer/idempotency` | `IdempotencyTransport`, `createMemoryIdempotencyStore` | [Idempotency](/transports/idempotency) | The types of each subpath (configurations, receipts with provider details, the structural `TracerProvider`, `MeterProvider` and `Logger` of the decorators) are in diff --git a/docs/transports/idempotency.md b/docs/transports/idempotency.md new file mode 100644 index 0000000..8b7081d --- /dev/null +++ b/docs/transports/idempotency.md @@ -0,0 +1,91 @@ +# Idempotency + +`IdempotencyTransport` sends a message with an idempotency key once. A later +send under the same key answers with the receipt of the first, so a job that +runs twice, or a retry after a crash, does not send the message twice. It comes +from the idempotency middleware of unemail. + +## Basic usage + +```js +const { createMessage } = require('@alexify/mailer'); +const { IdempotencyTransport } = require('@alexify/mailer/idempotency'); + +const transport = new IdempotencyTransport(ses); + +await transport.send( + createMessage({ + from: 'shop@example.com', + to: user.email, + subject: 'Welcome', + content: { html: '

Welcome aboard.

' }, + idempotencyKey: `welcome:${user.id}`, + }), +); +``` + +## How a key is kept + +The key is reserved in the store before the message is sent, so of two sends +under one key, only one goes: + +| Found under the key | Answer | +| --- | --- | +| nothing | the key is reserved, the message sent | +| a send still in flight | a receipt of unknown delivery (code `idempotency.in_flight`), not sent | +| a send that went, or may have gone | the receipt of that send | +| another message | refused as `validation` (code `idempotency.key_reused`) | + +A send that failed frees its key for another attempt; one of unknown delivery +(`deliveryUnknown`, or one the caller aborted after it began) keeps it, as the +message may have gone. A message is told from another by a fingerprint of its +addresses, subject, body, header fields, the names and types of its +attachments (not their bytes), its threading, schedule, tracking and metadata. +A message without a key goes as it is. + +When the store fails to reserve a key, that message alone fails, retryable +(`idempotency.store_failed`); when it fails to keep the receipt of a send that +went, the receipt still comes back and the failure becomes a process warning +(`MAILER_IDEMPOTENCY_STORE_FAILED`). + +## Stores + +The default store is in this process: it deduplicates across the transports +given it, not across processes, and forgets on a restart. For several +processes, pass a store on a shared database whose `reserve` is atomic: + +```js +const redisStore = { + // SET NX reserves the key; the record is JSON. + async reserve(key, fingerprint, ttlMs) { + const pending = JSON.stringify({ state: 'pending', fingerprint }); + const ok = await redis.set(key, pending, { NX: true, PX: ttlMs }); + if (ok) return undefined; + return JSON.parse(await redis.get(key)); + }, + async complete(key, { fingerprint, receipt }, ttlMs) { + await redis.set(key, JSON.stringify({ state: 'done', fingerprint, receipt }), { PX: ttlMs }); + }, + async release(key) { + await redis.del(key); + }, +}; +``` + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `store` | in-process | `reserve`, `complete` and `release` of the keys. | +| `ttlMs` | `86400000` | How long a key is kept. | +| `key` | `message.idempotencyKey` | The key of a message. | +| `namespace` | the wrapped transport's id | The prefix of the keys in the store. | + +`createMemoryIdempotencyStore({ maxEntries, now })` makes the in-process store, +which keeps at most `maxEntries` keys (100000), dropping the oldest first. + +Put this decorator outside `./retry` and `./pool`, so that a message is reserved +once for all their attempts. `sendMany()` goes through the `sendMany()` of the +wrapped transport, a message whose key was used before, in the batch too, +answered in place. A raw message has no `idempotencyKey`, so `sendRaw()` checks +only the key `key` gives it, its fingerprint its envelope. diff --git a/package.json b/package.json index c77ef41..c4b9f14 100644 --- a/package.json +++ b/package.json @@ -143,6 +143,10 @@ "./ratelimit": { "types": "./src/ratelimit/index.d.ts", "default": "./src/ratelimit/index.js" + }, + "./idempotency": { + "types": "./src/idempotency/index.d.ts", + "default": "./src/idempotency/index.js" } }, "directories": { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 9dc54d9..1cac1ce 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -29,5 +29,6 @@ "./otel": 5, "./logger": 2.5, "./breaker": 2, - "./ratelimit": 2.5 + "./ratelimit": 2.5, + "./idempotency": 3 } diff --git a/src/idempotency/config.js b/src/idempotency/config.js new file mode 100644 index 0000000..dd975c0 --- /dev/null +++ b/src/idempotency/config.js @@ -0,0 +1,38 @@ +// Ported from unemail v0.7.0 (src/middleware/idempotency.ts): the options of +// the idempotency decorator. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { checkOptionalFunction, checkPositiveNumber } = require('../internal/options.js'); +const { createMemoryIdempotencyStore } = require('./memory-store.js'); + +const STORE_METHODS = ['reserve', 'complete', 'release']; + +/** + * Resolves the options of the idempotency decorator. + * + * @param config The options. + * @param transportId The id of the wrapped transport, the default namespace. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createIdempotencyConfig = (config = {}, transportId) => { + const store = config.store ?? createMemoryIdempotencyStore(); + for (const method of STORE_METHODS) { + if (typeof store?.[method] !== 'function') { + throw new TypeError(`store must have a ${method} method.`); + } + } + const namespace = config.namespace ?? transportId; + if (typeof namespace !== 'string') { + throw new TypeError(`namespace must be a string: ${String(namespace)}`); + } + return { + store, + ttlMs: checkPositiveNumber(config.ttlMs ?? 24 * 60 * 60 * 1000, 'ttlMs'), + key: checkOptionalFunction(config.key, 'key'), + namespace, + }; +}; + +module.exports = { createIdempotencyConfig }; diff --git a/src/idempotency/fingerprint.js b/src/idempotency/fingerprint.js new file mode 100644 index 0000000..97d6839 --- /dev/null +++ b/src/idempotency/fingerprint.js @@ -0,0 +1,70 @@ +// Not ported: the fingerprint of a message under an idempotency key, so that a +// key used again for another message is refused rather than answered with the +// receipt of the first. + +const { createHash } = require('node:crypto'); + +const addAddresses = (hash, label, addresses) => { + hash.update(`${label}\u0000`); + for (const address of addresses) { + hash.update(`${address.name ?? ''}\u0001${address.address}\u0000`); + } +}; + +const addValue = (hash, label, value) => { + hash.update(`${label}\u0000${JSON.stringify(value) ?? ''}\u0000`); +}; + +const addMessage = (hash, message) => { + addAddresses(hash, 'from', [message.sender]); + addAddresses(hash, 'to', message.recipients); + addAddresses(hash, 'cc', message.ccRecipients); + addAddresses(hash, 'bcc', message.bccRecipients); + addAddresses(hash, 'reply-to', message.replyRecipients); + addValue(hash, 'subject', message.subject); + addValue(hash, 'text', message.content.text); + addValue(hash, 'html', message.content.html); + addValue(hash, 'priority', message.priority); + addValue(hash, 'tags', message.tags); + // Headers iterate in the order of their lowercased names. + for (const [name, value] of message.headers) addValue(hash, `header ${name}`, value); + for (const attachment of message.attachments) { + addValue(hash, 'attachment', [ + attachment.filename, + attachment.contentType, + attachment.inline, + attachment.contentId, + ]); + } + addValue(hash, 'in-reply-to', message.inReplyTo); + addValue(hash, 'references', message.references); + addValue(hash, 'calendar', message.calendar); + addValue(hash, 'scheduled-at', message.scheduledAt?.toISOString()); + addValue(hash, 'tracking', message.tracking); + if (message.metadata !== undefined) { + const keys = Object.keys(message.metadata).toSorted(); + for (const key of keys) addValue(hash, `metadata ${key}`, message.metadata[key]); + } +}; + +/** + * The SHA-256 of what a message says: its addresses, subject, body, header + * fields, the names and types of its attachments (not their bytes), its + * threading, schedule, tracking and metadata; for a raw message, its + * envelope. Its Message-ID and Date are left out, as an attempt may set them. + * + * @param message A message or a raw message. + * @returns The fingerprint, in hexadecimal. + * @internal + */ +const fingerprintOf = (message) => { + const hash = createHash('sha256'); + if (message.envelope !== undefined && message.recipients === undefined) { + addValue(hash, 'raw', [message.envelope.from, message.envelope.to]); + } else { + addMessage(hash, message); + } + return hash.digest('hex'); +}; + +module.exports = { fingerprintOf }; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js new file mode 100644 index 0000000..b34c050 --- /dev/null +++ b/src/idempotency/idempotency-transport.js @@ -0,0 +1,223 @@ +// Ported from unemail v0.7.0 (src/middleware/idempotency.ts): a decorator that +// answers a message whose idempotency key was sent before with the receipt of +// that send. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the key is reserved before the send, so a second send +// under a key still in flight is not sent, where unemail looked the key up +// before and wrote it after, and two concurrent sends both went; a key used +// again for another message is refused, where unemail answered with the +// first's receipt; a delivery of unknown outcome keeps the key, where unemail +// remembered successes only and a retry sent again; and a store that fails +// fails the message it was asked about alone. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { warnThrown } = require('../internal/callback.js'); +const { refuseRaw } = require('../internal/capabilities.js'); +const { gatedSendMany } = require('../internal/gate.js'); +const { createIdempotencyConfig } = require('./config.js'); +const { fingerprintOf } = require('./fingerprint.js'); + +const STORE_FAILED = 'MAILER_IDEMPOTENCY_STORE_FAILED'; + +const isAbortOf = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +const disposeOf = async (transport) => { + const target = Object(transport); + if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); + else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); +}; + +/** + * A transport that sends a message with an idempotency key once: a send under + * a key already used answers with the receipt of the first, or as of unknown + * delivery while the first is in flight. + */ +class IdempotencyTransport { + id; + config; + wrappedTransport; + + /** + * @param transport The transport to guard. + * @param config `store`, `ttlMs`, `key` and `namespace`. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport, config) { + this.wrappedTransport = transport; + this.id = transport.id; + this.config = createIdempotencyConfig(config, transport.id); + } + + /** + * Sends one message, unless its key was used before. + * + * @returns The receipt of the send, that of the first send under the key, + * or a failed one: `idempotency.in_flight` (of unknown delivery), + * `idempotency.key_reused` or `idempotency.store_failed`. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + const entry = await this.#reserve(message); + if (entry.receipt !== undefined) return entry.receipt; + return this.#sendReserved(entry, options, (sendOptions) => + this.wrappedTransport.send(message, sendOptions), + ); + } + + /** + * Sends a raw message, unless the key `key` gives it was used before; a raw + * message has no `idempotencyKey`, so without `key` it goes as it is. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const entry = await this.#reserve(message); + if (entry.receipt !== undefined) return entry.receipt; + return this.#sendReserved(entry, options, (sendOptions) => + this.wrappedTransport.sendRaw(message, sendOptions), + ); + } + + /** + * Sends messages through the `sendMany()` of the wrapped transport, one + * whose key was used before, in this batch too, answered in place. + */ + sendMany(messages, options) { + return gatedSendMany(this.wrappedTransport, messages, options, { + admit: async (message) => { + const entry = await this.#reserve(message); + return entry.receipt === undefined ? { message, entry } : { receipt: entry.receipt }; + }, + settle: async (receipt, admitted) => { + await this.#settle(admitted.entry, receipt); + return receipt; + }, + fail: (error, admitted) => this.#thrown(admitted.entry, error, options?.signal), + }); + } + + /** Verifies the wrapped transport. */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** Disposes of the wrapped transport when it supports disposal. */ + async [Symbol.asyncDispose]() { + await disposeOf(this.wrappedTransport); + } + + async #sendReserved(entry, options, send) { + if (options?.signal?.aborted) { + // Aborted before the send began: nothing went. + await this.#release(entry); + options.signal.throwIfAborted(); + } + let receipt; + try { + receipt = await send(options); + } catch (error) { + await this.#thrown(entry, error, options?.signal); + throw error; + } + await this.#settle(entry, receipt); + return receipt; + } + + // Reserves the key of a message: `{ key, fingerprint }` to send under it, + // `{}` for a message without one, or `{ receipt }` to answer with. + async #reserve(message) { + const key = this.config.key === undefined ? message.idempotencyKey : this.config.key(message); + if (key === undefined || key === null || key === '') return {}; + const storeKey = `${this.config.namespace}:${key}`; + const fingerprint = fingerprintOf(message); + let record; + try { + record = await this.config.store.reserve(storeKey, fingerprint, this.config.ttlMs); + } catch (error) { + return { receipt: this.#storeFailure(error) }; + } + if (record === undefined || record === null) return { key: storeKey, fingerprint }; + return { receipt: this.#replyOf(record, fingerprint, key) }; + } + + #replyOf(record, fingerprint, key) { + if (record.fingerprint !== fingerprint) { + return createFailedReceipt( + `The idempotency key ${key} was used for another message; this one was not sent.`, + { + provider: this.id, + category: 'validation', + retryable: false, + code: 'idempotency.key_reused', + }, + ); + } + if (record.state === 'done' && record.receipt !== undefined) return record.receipt; + return createFailedReceipt( + `A message with the idempotency key ${key} is being sent; this one was not sent again.`, + { provider: this.id, deliveryUnknown: true, code: 'idempotency.in_flight' }, + ); + } + + #storeFailure(error) { + return createFailedReceipt( + `The idempotency store failed: ${error instanceof Error ? error.message : String(error)}`, + { + provider: this.id, + category: 'service-unavailable', + retryable: true, + code: 'idempotency.store_failed', + }, + ); + } + + // A sent or possibly sent message keeps its key with its receipt; one that + // failed to go frees it for another attempt. + async #settle(entry, receipt) { + if (entry.key === undefined) return; + try { + if (receipt.successful || receipt.deliveryUnknown === true) { + await this.config.store.complete( + entry.key, + { fingerprint: entry.fingerprint, receipt }, + this.config.ttlMs, + ); + } else { + await this.config.store.release(entry.key); + } + } catch (error) { + // The receipt stands; the key stays reserved until it expires. + warnThrown('The idempotency store', error, STORE_FAILED); + } + } + + // A send the caller aborted may have gone, so its key is kept as of unknown + // delivery; one that threw otherwise, a bug before any request most likely, + // frees it. + async #thrown(entry, error, signal) { + if (entry.key === undefined) return; + if (isAbortOf(error, signal)) { + const receipt = createFailedReceipt( + 'The send was aborted after it began; whether the message went is not known.', + { provider: this.id, deliveryUnknown: true, code: 'idempotency.aborted' }, + ); + await this.#settle(entry, receipt); + return; + } + await this.#release(entry); + } + + async #release(entry) { + if (entry.key === undefined) return; + try { + await this.config.store.release(entry.key); + } catch (error) { + warnThrown('The idempotency store', error, STORE_FAILED); + } + } +} + +module.exports = { IdempotencyTransport }; diff --git a/src/idempotency/index.d.ts b/src/idempotency/index.d.ts new file mode 100644 index 0000000..0548bc8 --- /dev/null +++ b/src/idempotency/index.d.ts @@ -0,0 +1,161 @@ +// Types ported from unemail v0.7.0 (src/middleware) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +/// + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * What a store holds for a key: `pending` while its send is in flight, + * `done` with the receipt of a send that went or may have gone. + */ +export interface IdempotencyRecord { + readonly state: 'pending' | 'done'; + /** The fingerprint of the message sent under the key. */ + readonly fingerprint: string; + /** The receipt of the send, when it is `done`. */ + readonly receipt?: Receipt; +} + +/** + * Where idempotency keys are kept. A store shared by several processes, such + * as one on Redis or MongoDB, must make `reserve` atomic: of two calls for a + * free key, one reserves it and the other gets the record. + */ +export interface IdempotencyStore { + /** + * Reserves a key for `ttlMs`: `undefined` (or `null`) when it was free and + * is now `pending`, or the record that holds it. + */ + reserve( + key: string, + fingerprint: string, + ttlMs: number, + ): IdempotencyRecord | undefined | null | Promise; + /** Keeps a key `done` with the receipt of its send, for `ttlMs`. */ + complete( + key: string, + record: { readonly fingerprint: string; readonly receipt: Receipt }, + ttlMs: number, + ): void | Promise; + /** Frees a key whose send failed and may be made again. */ + release(key: string): void | Promise; +} + +/** + * The options of {@link IdempotencyTransport}. + */ +export interface IdempotencyConfig { + /** + * Where the keys are kept. The default is an in-process store, which + * deduplicates only within this process. + */ + readonly store?: IdempotencyStore; + /** + * How long a key is kept, in milliseconds. + * @default 86400000 + */ + readonly ttlMs?: number; + /** + * The key of a message; its `idempotencyKey` without it. A message + * without a key goes as it is. + */ + readonly key?: (message: Message | RawMessage) => string | undefined; + /** + * The prefix of the keys in the store, so that the same key sent through + * two transports is two sends. + * @default the id of the wrapped transport + */ + readonly namespace?: string; +} + +/** + * The options of {@link IdempotencyTransport}, with their defaults. + */ +export interface ResolvedIdempotencyConfig { + readonly store: IdempotencyStore; + readonly ttlMs: number; + readonly key?: (message: Message | RawMessage) => string | undefined; + readonly namespace: string; +} + +/** + * The options of {@link createMemoryIdempotencyStore}. + */ +export interface MemoryIdempotencyStoreOptions { + /** + * The most keys kept, the oldest dropped first. + * @default 100000 + */ + readonly maxEntries?: number; + /** The clock, in milliseconds. */ + readonly now?: () => number; +} + +/** + * Makes a store of idempotency keys in this process: shared by the transports + * given it, not by other processes, and lost on a restart. + * + * @throws {TypeError} If an option is invalid. + */ +export declare function createMemoryIdempotencyStore( + options?: MemoryIdempotencyStoreOptions, +): IdempotencyStore; + +/** + * A transport that sends a message with an idempotency key once. The key is + * reserved before the send; a later send under it answers with the receipt of + * the first, or, while the first is in flight, with a receipt of unknown + * delivery (code `idempotency.in_flight`); the same key for another message + * is refused (`idempotency.key_reused`). A send that failed frees its key, one + * of unknown delivery keeps it; a store that fails fails the message it was + * asked about alone (`idempotency.store_failed`, retryable). + * + * @example + * ```js + * const { IdempotencyTransport } = require('@alexify/mailer/idempotency'); + * const once = new IdempotencyTransport(transport, { store: redisStore }); + * await once.send(createMessage({ ...fields, idempotencyKey: `welcome:${userId}` })); + * ``` + */ +export declare class IdempotencyTransport + implements RawTransport, VerifiableTransport, AsyncDisposable +{ + readonly id: TProviderId; + /** The resolved options. */ + readonly config: ResolvedIdempotencyConfig; + /** + * @param transport The transport to guard. + * @param config The options. + * @throws {TypeError} If an option is invalid. + */ + constructor(transport: Transport, config?: IdempotencyConfig); + /** Sends one message, unless its key was used before. */ + send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends a raw message, unless the key `key` gives it was used before. A + * wrapped transport without `sendRaw()` gives a failed receipt of the + * category `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Sends messages through the `sendMany()` of the wrapped transport, one whose + * key was used before, in the batch too, answered in place. + */ + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the wrapped transport. */ + verify(options?: TransportOptions): Promise; + /** Disposes of the wrapped transport when it supports disposal. */ + [Symbol.asyncDispose](): Promise; +} diff --git a/src/idempotency/index.js b/src/idempotency/index.js new file mode 100644 index 0000000..48cbe75 --- /dev/null +++ b/src/idempotency/index.js @@ -0,0 +1,8 @@ +// Ported from unemail v0.7.0 (src/middleware/idempotency.ts): the entry of the +// idempotency decorator and its in-process store. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { IdempotencyTransport } = require('./idempotency-transport.js'); +const { createMemoryIdempotencyStore } = require('./memory-store.js'); + +module.exports = { IdempotencyTransport, createMemoryIdempotencyStore }; diff --git a/src/idempotency/memory-store.js b/src/idempotency/memory-store.js new file mode 100644 index 0000000..dfe78e0 --- /dev/null +++ b/src/idempotency/memory-store.js @@ -0,0 +1,64 @@ +// Ported from unemail v0.7.0 (src/middleware/idempotency.ts): the in-process +// store of idempotency keys, `memoryIdempotencyStore`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a key is reserved before the send, so a second send +// under it finds it taken, where unemail looked a key up and wrote it only +// after the send; a record keeps the fingerprint of the message; and the +// clock is injected. + +const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); + +/** + * Makes a store of idempotency keys in this process: shared by the + * transports given it, not by other processes, and lost on a restart. It + * holds at most `maxEntries` keys, dropping the oldest first. + * + * @param options `maxEntries` (100000) and `now`. + * @returns The store. + * @throws {TypeError} If an option is invalid. + */ +const createMemoryIdempotencyStore = (options = {}) => { + const maxEntries = checkPositiveInteger(options.maxEntries ?? 100000, 'maxEntries'); + const now = checkOptionalFunction(options.now, 'now') ?? Date.now; + const entries = new Map(); + + // A Map iterates in the order of insertion, and a record is written anew + // when it changes, so the first ones are the oldest. + const trim = () => { + const at = now(); + for (const [key, entry] of entries) { + if (entries.size <= maxEntries && entry.expiresAt > at) break; + entries.delete(key); + } + }; + + const write = (key, entry) => { + entries.delete(key); + entries.set(key, entry); + if (entries.size > maxEntries) trim(); + }; + + return { + reserve(key, fingerprint, ttlMs) { + const entry = entries.get(key); + if (entry !== undefined && entry.expiresAt > now()) { + return { state: entry.state, fingerprint: entry.fingerprint, receipt: entry.receipt }; + } + write(key, { state: 'pending', fingerprint, receipt: undefined, expiresAt: now() + ttlMs }); + return undefined; + }, + complete(key, record, ttlMs) { + write(key, { + state: 'done', + fingerprint: record.fingerprint, + receipt: record.receipt, + expiresAt: now() + ttlMs, + }); + }, + release(key) { + entries.delete(key); + }, + }; +}; + +module.exports = { createMemoryIdempotencyStore }; diff --git a/tests/types/idempotency.test-d.ts b/tests/types/idempotency.test-d.ts new file mode 100644 index 0000000..1d27dfb --- /dev/null +++ b/tests/types/idempotency.test-d.ts @@ -0,0 +1,44 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type Transport, + type VerifiableTransport, +} from '../../index.js'; +import { + IdempotencyTransport, + createMemoryIdempotencyStore, + type IdempotencyRecord, + type IdempotencyStore, +} from '../../src/idempotency/index.js'; + +declare const inner: Transport<'resend'>; +const store: IdempotencyStore = createMemoryIdempotencyStore({ maxEntries: 10, now: Date.now }); +const custom: IdempotencyStore = { + reserve: async (_key, fingerprint): Promise => + fingerprint === '' ? undefined : { state: 'pending', fingerprint }, + complete: async () => {}, + release: () => {}, +}; +const once = new IdempotencyTransport(inner, { + store: custom, + ttlMs: 60_000, + key: (message) => ('idempotencyKey' in message ? message.idempotencyKey : undefined), + namespace: 'tenant-a', +}); +new IdempotencyTransport(inner, { store }); +new IdempotencyTransport(inner); +expectType<'resend'>(once.id); +expectAssignable>(once); +expectAssignable>(once); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, + idempotencyKey: 'k', +}); +expectType>>(once.send(message)); +expectType>>(once.sendMany([message])); +expectError(new IdempotencyTransport(inner, { store: {} })); diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js new file mode 100644 index 0000000..92f45ae --- /dev/null +++ b/tests/unit/idempotency.test.js @@ -0,0 +1,302 @@ +// Not ported: tests of ./idempotency, ported from unemail's idempotency +// middleware, against a fake transport. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { + IdempotencyTransport, + createMemoryIdempotencyStore, +} = require('../../src/idempotency/index.js'); +const { collect, failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const message = (fields = {}) => + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Welcome', + content: { text: 'x' }, + ...fields, + }); + +const deferred = () => { + let resolve; + const promise = new Promise((_resolve) => { + resolve = _resolve; + }); + return { promise, resolve }; +}; + +test('Idempotency: a key is sent once, then answered with its receipt', async () => { + const inner = fakeTransport(); + const once = new IdempotencyTransport(inner); + const first = await once.send(message({ idempotencyKey: 'welcome:1' })); + const second = await once.send(message({ idempotencyKey: 'welcome:1' })); + assert.strictEqual(first.messageId, 'm0'); + assert.deepStrictEqual(second, first); + assert.strictEqual(inner.calls.length, 1); + // Without a key, a message goes every time. + await once.send(message()); + await once.send(message()); + assert.strictEqual(inner.calls.length, 3); +}); + +test('Idempotency: a send under a key in flight is not sent again', async () => { + const gate = deferred(); + const inner = fakeTransport(() => gate.promise); + const once = new IdempotencyTransport(inner); + const first = once.send(message({ idempotencyKey: 'k' })); + // Changed in the port: unemail looked the key up before the send and wrote + // it after, so this one went too. + const second = await once.send(message({ idempotencyKey: 'k' })); + assert.strictEqual(second.successful, false); + assert.strictEqual(second.deliveryUnknown, true); + assert.strictEqual(second.errors[0].code, 'idempotency.in_flight'); + gate.resolve(sent('only')); + assert.strictEqual((await first).messageId, 'only'); + assert.strictEqual(inner.calls.length, 1); +}); + +test('Idempotency: a key used for another message is refused', async () => { + const inner = fakeTransport(); + const once = new IdempotencyTransport(inner); + await once.send(message({ idempotencyKey: 'k' })); + // Changed in the port: unemail answered with the receipt of the first. + const other = await once.send(message({ idempotencyKey: 'k', subject: 'Another' })); + assert.strictEqual(other.errors[0].code, 'idempotency.key_reused'); + assert.strictEqual(other.errors[0].category, 'validation'); + assert.strictEqual(other.retryable, false); + // A pinned Message-ID or Date is not part of the message's fingerprint. + const again = await once.send( + message({ idempotencyKey: 'k', messageId: '', date: new Date(0) }), + ); + assert.strictEqual(again.messageId, 'm0'); + for (const fields of [ + { to: 'c@example.com' }, + { headers: { 'X-Campaign': 'b' } }, + { metadata: { order: '2' } }, + { + attachments: { + filename: 'r.txt', + contentType: 'text/plain', + inline: false, + contentId: '', + content: new Uint8Array(1), + }, + }, + { scheduledAt: new Date(1) }, + ]) { + const refused = await once.send(message({ idempotencyKey: 'k', ...fields })); + assert.strictEqual(refused.errors[0].code, 'idempotency.key_reused', JSON.stringify(fields)); + } + assert.strictEqual(inner.calls.length, 1); +}); + +test('Idempotency: a failure frees the key, an unknown delivery keeps it', async () => { + const answers = [ + failed('validation', { retryable: false }), + sent('second'), + failed('timeout', { deliveryUnknown: true, retryable: false }), + ]; + const inner = fakeTransport((_m, index) => answers[index]); + const once = new IdempotencyTransport(inner); + assert.strictEqual((await once.send(message({ idempotencyKey: 'a' }))).successful, false); + assert.strictEqual((await once.send(message({ idempotencyKey: 'a' }))).messageId, 'second'); + const unknown = await once.send(message({ idempotencyKey: 'b' })); + // Changed in the port: unemail remembered successes only, so a retry went again. + const replay = await once.send(message({ idempotencyKey: 'b' })); + assert.deepStrictEqual(replay, unknown); + assert.strictEqual(inner.calls.length, 3); +}); + +test('Idempotency: an abort keeps the key as of unknown delivery, another throw frees it', async () => { + const answers = []; + const started = deferred(); + const inner = fakeTransport((_m, index, options) => { + if (answers[index] !== undefined) return answers[index]; + started.resolve(); + return new Promise((_resolve, reject) => { + options.signal.addEventListener('abort', () => reject(options.signal.reason)); + }); + }); + const once = new IdempotencyTransport(inner); + const controller = new AbortController(); + const aborted = once.send(message({ idempotencyKey: 'a' }), { signal: controller.signal }); + await started.promise; + controller.abort(new Error('left')); + await assert.rejects(aborted, /left/); + const after = await once.send(message({ idempotencyKey: 'a' })); + assert.strictEqual(after.deliveryUnknown, true); + assert.strictEqual(after.errors[0].code, 'idempotency.aborted'); + answers[1] = { throw: new TypeError('bug') }; + answers[2] = sent('fixed'); + await assert.rejects(once.send(message({ idempotencyKey: 'b' })), /bug/); + assert.strictEqual((await once.send(message({ idempotencyKey: 'b' }))).messageId, 'fixed'); + await assert.rejects( + once.send(message(), { signal: AbortSignal.abort(new Error('already')) }), + /already/, + ); + // Aborted after the key is reserved and before the send begins: the key is + // free again, as nothing went. + const early = new AbortController(); + const slow = new IdempotencyTransport(inner, { + store: { + ...createMemoryIdempotencyStore(), + reserve: async () => { + early.abort(new Error('changed my mind')); + return undefined; + }, + release: () => {}, + }, + }); + await assert.rejects( + slow.send(message({ idempotencyKey: 'c' }), { signal: early.signal }), + /changed my mind/, + ); + assert.strictEqual(inner.calls.length, 3); +}); + +test('Idempotency: a store that fails fails its message alone, or becomes a warning', async (t) => { + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const memory = createMemoryIdempotencyStore(); + let failing = 'reserve'; + const store = { + reserve: (...args) => { + if (failing === 'reserve') throw new Error('store down'); + return memory.reserve(...args); + }, + complete: async (...args) => { + if (failing === 'complete') throw new Error('cannot write'); + return memory.complete(...args); + }, + release: (key) => { + // oxlint-disable-next-line no-throw-literal -- a store may throw anything + if (failing === 'release') throw 'cannot delete'; + return memory.release(key); + }, + }; + const answers = [ + sent('one'), + failed('validation', { retryable: false }), + { throw: new Error('bug') }, + ]; + const inner = fakeTransport((_m, index) => answers[index]); + const once = new IdempotencyTransport(inner, { store }); + const down = await once.send(message({ idempotencyKey: 'a' })); + assert.strictEqual(down.errors[0].code, 'idempotency.store_failed'); + assert.strictEqual(down.retryable, true); + assert.strictEqual(inner.calls.length, 0); + failing = 'complete'; + assert.strictEqual((await once.send(message({ idempotencyKey: 'a' }))).messageId, 'one'); + failing = 'release'; + assert.strictEqual((await once.send(message({ idempotencyKey: 'b' }))).successful, false); + await assert.rejects(once.send(message({ idempotencyKey: 'c' })), /bug/); + assert.deepStrictEqual(warnings, [ + ['The idempotency store threw: cannot write', 'MAILER_IDEMPOTENCY_STORE_FAILED'], + ['The idempotency store threw: cannot delete', 'MAILER_IDEMPOTENCY_STORE_FAILED'], + ['The idempotency store threw: cannot delete', 'MAILER_IDEMPOTENCY_STORE_FAILED'], + ]); +}); + +test('Idempotency: sendMany answers repeated keys in place, the batch too', async () => { + const inner = fakeTransport(undefined, { readAhead: 1 }); + const once = new IdempotencyTransport(inner); + await once.send(message({ idempotencyKey: 'before' })); + const receipts = await collect( + once.sendMany([ + message({ idempotencyKey: 'x' }), + message({ idempotencyKey: 'before' }), + message({ idempotencyKey: 'x' }), + message(), + ]), + ); + // The second x comes once the first is done, and gets its receipt. + assert.deepStrictEqual( + receipts.map((receipt) => receipt.messageId ?? receipt.errors[0].code), + ['m1', 'm0', 'm1', 'm2'], + ); + // Read while the first is in flight, a second y is not sent. + const twice = await collect( + once.sendMany([message({ idempotencyKey: 'y' }), message({ idempotencyKey: 'y' })]), + ); + assert.deepStrictEqual( + twice.map((receipt) => receipt.messageId ?? receipt.errors[0].code), + ['m3', 'idempotency.in_flight'], + ); + assert.strictEqual((await once.send(message({ idempotencyKey: 'x' }))).messageId, 'm1'); + const exploding = fakeTransport(undefined, { + overrides: { + // oxlint-disable-next-line require-yield + async *sendMany(messages) { + await messages[Symbol.asyncIterator]().next(); + throw new Error('batch exploded'); + }, + }, + }); + const guarded = new IdempotencyTransport(exploding); + await assert.rejects( + collect(guarded.sendMany([message({ idempotencyKey: 'y' })])), + /batch exploded/, + ); + // The key of the message that threw is free again. + assert.strictEqual((await guarded.send(message({ idempotencyKey: 'y' }))).messageId, 'm0'); +}); + +test('Idempotency: a key function, a namespace, sendRaw, verify and dispose', async () => { + const store = createMemoryIdempotencyStore(); + const inner = fakeTransport(); + const keyOf = (m) => m.envelope?.to[0] ?? m.metadata?.order; + const a = new IdempotencyTransport(inner, { store, key: keyOf, namespace: 'tenant-a' }); + const b = new IdempotencyTransport(inner, { store, key: keyOf, namespace: 'tenant-b' }); + await a.send(message({ metadata: { order: '1' } })); + await a.send(message({ metadata: { order: '1' } })); + await b.send(message({ metadata: { order: '1' } })); + assert.strictEqual(inner.calls.length, 2); + const raw = { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }; + await a.sendRaw(raw); + await a.sendRaw(raw); + assert.strictEqual(inner.calls.length, 3); + const otherRaw = { envelope: { from: 'c@example.com', to: ['b@example.com'] }, content: 'x' }; + assert.strictEqual((await a.sendRaw(otherRaw)).errors[0].code, 'idempotency.key_reused'); + await a.verify(); + assert.strictEqual(inner.verified, 1); + await a[Symbol.asyncDispose](); + assert.strictEqual(inner.disposed, 1); + const plain = { id: 'plain', send: async () => sent('p') }; + const bare = new IdempotencyTransport(plain); + assert.strictEqual((await bare.sendRaw(raw)).errors[0].category, 'configuration'); + await bare.verify(); + await bare[Symbol.asyncDispose](); + let disposed = 0; + await new IdempotencyTransport({ ...plain, [Symbol.dispose]: () => disposed++ })[ + Symbol.asyncDispose + ](); + assert.strictEqual(disposed, 1); + assert.throws(() => new IdempotencyTransport(inner, { store: {} }), /store must have a reserve/); + assert.throws(() => new IdempotencyTransport(inner, { ttlMs: 0 }), /ttlMs/); + assert.throws(() => new IdempotencyTransport(inner, { namespace: 1 }), /namespace/); + assert.strictEqual(new IdempotencyTransport(inner).config.namespace, 'fake'); +}); + +test('Idempotency: the memory store forgets a key after its time, and the oldest first', () => { + let time = 0; + const store = createMemoryIdempotencyStore({ maxEntries: 2, now: () => time }); + assert.strictEqual(store.reserve('a', 'fa', 100), undefined); + assert.deepStrictEqual(store.reserve('a', 'fa', 100), { + state: 'pending', + fingerprint: 'fa', + receipt: undefined, + }); + store.complete('a', { fingerprint: 'fa', receipt: sent('ra') }, 100); + assert.strictEqual(store.reserve('a', 'fa', 100).receipt.messageId, 'ra'); + time = 100; + assert.strictEqual(store.reserve('a', 'fa', 100), undefined); + store.reserve('b', 'fb', 1000); + store.reserve('c', 'fc', 1000); + // Three keys for two places: the oldest, a, goes. + assert.strictEqual(store.reserve('a', 'fa', 100), undefined); + assert.notStrictEqual(store.reserve('c', 'fc', 1000), undefined); + store.release('c'); + assert.strictEqual(store.reserve('c', 'fc', 1000), undefined); + assert.throws(() => createMemoryIdempotencyStore({ maxEntries: 0 }), /maxEntries/); +}); From 1491af594ea0f1b8d9c066df0a6571c7b7beb5f7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:46:26 +0300 Subject: [PATCH 038/143] feat(hooks): a decorator that calls hooks around each send Ported from sently's mailer hooks as a decorator of one transport: `beforeSend` (and `beforeSendRaw`) may replace a message, or refuse it by throwing, its receipt then `rejected` and unsent; `afterSend` hears every receipt, `onRetry` every retry and `onError` every thrown error. The context is made for each send and `onRetry` goes in the options of that send, where sently set it on the shared transport, so concurrent sends heard each other's retries (S4); a hook that throws after the send becomes a process warning (`MAILER_HOOK_FAILED`) and the receipt stands. `sendMany` goes through the wrapped `sendMany`. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/hooks.md | 43 +++++++ package.json | 4 + scripts/size-budgets.json | 3 +- src/hooks/hooks-transport.js | 206 ++++++++++++++++++++++++++++++++ src/hooks/index.d.ts | 121 +++++++++++++++++++ src/hooks/index.js | 6 + tests/types/hooks.test-d.ts | 40 +++++++ tests/unit/hooks.test.js | 222 +++++++++++++++++++++++++++++++++++ 11 files changed, 652 insertions(+), 1 deletion(-) create mode 100644 docs/transports/hooks.md create mode 100644 src/hooks/hooks-transport.js create mode 100644 src/hooks/index.d.ts create mode 100644 src/hooks/index.js create mode 100644 tests/types/hooks.test-d.ts create mode 100644 tests/unit/hooks.test.js diff --git a/CLAUDE.md b/CLAUDE.md index f44ea27..2310c32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -569,6 +569,12 @@ fails on upstream's behavior): (`reserve`, `complete`, `release`), answers a send in flight as of unknown delivery, refuses a key whose fingerprint differs, keeps a key whose delivery is unknown, and fails only the message whose store call failed. +- S4. sently (`src/mailer.ts`): `onRetry` and `onFallback` were set on the shared transport for + each send and unset when it ended, so of two concurrent sends one told its retries to the + other's hooks, and the first to end unset the hooks of the other; a hook that threw was written + with `console.warn` outside production and dropped in it. `./hooks` makes the context of each + send and passes `onRetry` in the options of that send, and a hook that throws after the send + becomes a process warning (`MAILER_HOOK_FAILED`). ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 06ae2ca..5f99e24 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -109,6 +109,7 @@ const guide = [ { text: 'Circuit breaker', link: '/transports/breaker' }, { text: 'Rate limit', link: '/transports/ratelimit' }, { text: 'Idempotency', link: '/transports/idempotency' }, + { text: 'Hooks', link: '/transports/hooks' }, ], }, ]; diff --git a/docs/api/exports.md b/docs/api/exports.md index d578a1e..50a807d 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -72,6 +72,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/breaker` | `CircuitBreakerTransport` | [Circuit breaker](/transports/breaker) | | `@alexify/mailer/ratelimit` | `RateLimitTransport` | [Rate limit](/transports/ratelimit) | | `@alexify/mailer/idempotency` | `IdempotencyTransport`, `createMemoryIdempotencyStore` | [Idempotency](/transports/idempotency) | +| `@alexify/mailer/hooks` | `HooksTransport` | [Hooks](/transports/hooks) | The types of each subpath (configurations, receipts with provider details, the structural `TracerProvider`, `MeterProvider` and `Logger` of the decorators) are in diff --git a/docs/transports/hooks.md b/docs/transports/hooks.md new file mode 100644 index 0000000..a826fd4 --- /dev/null +++ b/docs/transports/hooks.md @@ -0,0 +1,43 @@ +# Hooks + +`HooksTransport` calls functions of the application around each send of a +transport: before it, after it, on a retry and on a thrown error. It comes from +the hooks of sently. + +## Basic usage + +```js +const { HooksTransport } = require('@alexify/mailer/hooks'); + +const transport = new HooksTransport(ses, { + beforeSend: async (message) => { + if (await suppression.has(message.recipients[0].address)) { + throw new Error('on the suppression list'); + } + }, + afterSend: async (receipt, message, context) => { + await audit.record({ receipt, subject: message.subject, at: context.startedAt }); + }, + onRetry: (event, context) => metrics.increment('email.retry', { op: context.operation }), +}); +``` + +## The hooks + +| Hook | Called | When it throws | +| --- | --- | --- | +| `beforeSend(message, context)` | before a message is sent; it may return a message to send in its place | the message is not sent; its receipt is `rejected` (code `hooks.refused`), not retryable | +| `beforeSendRaw(raw, context)` | before a raw message is sent, the same way | the same | +| `afterSend(receipt, message, context)` | after every send, its receipt successful or not | a process warning (`MAILER_HOOK_FAILED`); the receipt stands | +| `onRetry(event, context)` | before each retry of the wrapped transport, or of a `./retry` inside it | a process warning | +| `onError(error, message, context)` | when the wrapped transport throws, before the error goes on | a process warning; the error goes on | + +A hook may be async; it is awaited. The context, `{ transportId, operation, +startedAt }`, is made for each send, and `onRetry` goes in the options of that +send, so concurrent sends never hear each other's retries; the caller's own +`onRetry` still hears them all. `startedAt` comes from the clock `now` +(`Date.now` by default). + +`sendMany()` goes through the `sendMany()` of the wrapped transport, each +message between its hooks, one `beforeSend` refuses answered in place; its +`onRetry` hears the retries with the context of the batch. diff --git a/package.json b/package.json index c4b9f14..df57d99 100644 --- a/package.json +++ b/package.json @@ -147,6 +147,10 @@ "./idempotency": { "types": "./src/idempotency/index.d.ts", "default": "./src/idempotency/index.js" + }, + "./hooks": { + "types": "./src/hooks/index.d.ts", + "default": "./src/hooks/index.js" } }, "directories": { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 1cac1ce..bfb85aa 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -30,5 +30,6 @@ "./logger": 2.5, "./breaker": 2, "./ratelimit": 2.5, - "./idempotency": 3 + "./idempotency": 3, + "./hooks": 2 } diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js new file mode 100644 index 0000000..4d3fdcf --- /dev/null +++ b/src/hooks/hooks-transport.js @@ -0,0 +1,206 @@ +// Ported from sently v1.2.2 (src/mailer.ts): hooks around each send of a +// transport — before it, after it, on a retry and on a thrown error. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Also ported from sently v1.2.2 (src/core/hooks.ts): a hook that throws does +// not break the send. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the context is made for each send, and `onRetry` goes +// in the options of that send, where sently set it on the shared transport, +// so concurrent sends told their retries to each other's hook and one ending +// unset the hook of another; a hook that throws after the send becomes a +// process warning, where sently wrote it with `console.warn` outside +// production and dropped it in production; and `beforeSend` may replace the +// message or refuse it. + +const { createFailedReceipt } = require('../core/receipt.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { warnThrown } = require('../internal/callback.js'); +const { refuseRaw } = require('../internal/capabilities.js'); +const { gatedSendMany } = require('../internal/gate.js'); +const { checkOptionalFunction } = require('../internal/options.js'); + +const HOOK_FAILED = 'MAILER_HOOK_FAILED'; +const HOOKS = ['beforeSend', 'beforeSendRaw', 'afterSend', 'onRetry', 'onError']; + +const disposeOf = async (transport) => { + const target = Object(transport); + if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); + else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); +}; + +/** + * A transport that calls the hooks of the application around each send of + * the one it wraps. + */ +class HooksTransport { + id; + config; + wrappedTransport; + + /** + * @param transport The transport to wrap. + * @param hooks `beforeSend`, `beforeSendRaw`, `afterSend`, `onRetry`, + * `onError`, and `now`. + * @throws {TypeError} If a hook is not a function. + */ + constructor(transport, hooks = {}) { + this.wrappedTransport = transport; + this.id = transport.id; + const config = {}; + for (const name of HOOKS) config[name] = checkOptionalFunction(hooks[name], name); + config.now = checkOptionalFunction(hooks.now, 'now') ?? Date.now; + this.config = config; + } + + /** + * Sends one message between its hooks. + * + * @returns The receipt of the wrapped transport, or a `rejected` one when + * `beforeSend` throws. + * @throws What the wrapped transport throws, after `onError`. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + const context = this.#contextOf('send'); + const before = await this.#before('beforeSend', message, context); + if (before.receipt !== undefined) return before.receipt; + return this.#around(before.message, context, options, (sendOptions) => + this.wrappedTransport.send(before.message, sendOptions), + ); + } + + /** + * Sends a raw message between `beforeSendRaw` and `afterSend`. A wrapped + * transport without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const context = this.#contextOf('sendRaw'); + const before = await this.#before('beforeSendRaw', message, context); + if (before.receipt !== undefined) return before.receipt; + return this.#around(before.message, context, options, (sendOptions) => + this.wrappedTransport.sendRaw(before.message, sendOptions), + ); + } + + /** + * Sends messages through the `sendMany()` of the wrapped transport, each + * between its hooks; one `beforeSend` refuses is answered in place. + */ + sendMany(messages, options) { + const batch = this.#contextOf('sendMany'); + return gatedSendMany(this.wrappedTransport, messages, this.#optionsOf(options, batch), { + admit: async (message) => { + const context = this.#contextOf('sendMany'); + const before = await this.#before('beforeSend', message, context); + return before.receipt === undefined + ? { message: before.message, context } + : { receipt: before.receipt }; + }, + settle: async (receipt, admitted) => { + await this.#after(receipt, admitted.message, admitted.context); + return receipt; + }, + fail: (error, admitted) => this.#error(error, admitted.message, admitted.context), + }); + } + + /** Verifies the wrapped transport. */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + } + + /** Disposes of the wrapped transport when it supports disposal. */ + async [Symbol.asyncDispose]() { + await disposeOf(this.wrappedTransport); + } + + #contextOf(operation) { + return { transportId: this.id, operation, startedAt: this.config.now() }; + } + + // The options of one send, its retries told to `onRetry` with its context. + #optionsOf(options, context) { + const hook = this.config.onRetry; + if (hook === undefined) return options; + const onRetry = options?.onRetry; + return { + ...options, + onRetry: (event) => { + try { + hook(event, context); + } catch (error) { + warnThrown('The onRetry hook of HooksTransport', error, HOOK_FAILED); + } + onRetry?.(event); + }, + }; + } + + // Runs a before hook: `{ message }` to send, or `{ receipt }` of a refusal. + async #before(name, message, context) { + const hook = this.config[name]; + if (hook === undefined) return { message }; + let replaced; + try { + replaced = await hook(message, context); + } catch (error) { + return { + receipt: this.#refusal( + `The ${name} hook refused the message: ${error instanceof Error ? error.message : String(error)}`, + ), + }; + } + if (replaced === undefined) return { message }; + if (replaced === null || typeof replaced !== 'object') { + return { + receipt: this.#refusal(`The ${name} hook returned neither a message nor nothing.`), + }; + } + return { message: replaced }; + } + + #refusal(text) { + return createFailedReceipt(text, { + provider: this.id, + category: 'rejected', + retryable: false, + code: 'hooks.refused', + }); + } + + async #around(message, context, options, send) { + let receipt; + try { + receipt = await send(this.#optionsOf(options, context)); + } catch (error) { + await this.#error(error, message, context); + throw error; + } + await this.#after(receipt, message, context); + return receipt; + } + + async #after(receipt, message, context) { + if (this.config.afterSend === undefined) return; + try { + await this.config.afterSend(receipt, message, context); + } catch (error) { + warnThrown('The afterSend hook of HooksTransport', error, HOOK_FAILED); + } + } + + async #error(error, message, context) { + if (this.config.onError === undefined) return; + try { + await this.config.onError(error, message, context); + } catch (hookError) { + warnThrown('The onError hook of HooksTransport', hookError, HOOK_FAILED); + } + } +} + +module.exports = { HooksTransport }; diff --git a/src/hooks/index.d.ts b/src/hooks/index.d.ts new file mode 100644 index 0000000..d940966 --- /dev/null +++ b/src/hooks/index.d.ts @@ -0,0 +1,121 @@ +// Types ported from sently v1.2.2 (src) +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors + +/// + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + RetryEvent, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * The context of one send, made anew for each. + */ +export interface HookContext { + /** The id of the wrapped transport. */ + readonly transportId: string; + /** The method of the send. */ + readonly operation: 'send' | 'sendMany' | 'sendRaw'; + /** When the send began, by the clock `now`, in milliseconds. */ + readonly startedAt: number; +} + +/** + * The hooks of {@link HooksTransport}. A hook may be async; it is awaited. + */ +export interface Hooks { + /** + * Before a message is sent: it may return a message to send in its place; + * when it throws, the message is not sent and its receipt is `rejected` + * (code `hooks.refused`), not retryable. + */ + readonly beforeSend?: ( + message: Message, + context: HookContext, + ) => Message | void | Promise; + /** Before a raw message is sent, as `beforeSend`. */ + readonly beforeSendRaw?: ( + message: RawMessage, + context: HookContext, + ) => RawMessage | void | Promise; + /** + * After a send, with its receipt, successful or not; one that throws + * becomes a process warning (`MAILER_HOOK_FAILED`) and the receipt stands. + */ + readonly afterSend?: ( + receipt: Receipt, + message: Message | RawMessage, + context: HookContext, + ) => void | Promise; + /** + * Before a retry of the wrapped transport, or of a `./retry` within it, with + * the context of its send (that of the batch for `sendMany()`). + */ + readonly onRetry?: (event: RetryEvent, context: HookContext) => void; + /** When the wrapped transport throws, before the error goes on. */ + readonly onError?: ( + error: unknown, + message: Message | RawMessage, + context: HookContext, + ) => void | Promise; + /** The clock of `startedAt`, in milliseconds. */ + readonly now?: () => number; +} + +/** + * A transport that calls the hooks of the application around each send of + * the one it wraps: `beforeSend` may replace or refuse a message, `afterSend` + * hears every receipt, `onRetry` every retry and `onError` every thrown + * error. The context is that of each send, so concurrent sends never hear + * each other's. + * + * @example + * ```js + * const { HooksTransport } = require('@alexify/mailer/hooks'); + * const hooked = new HooksTransport(transport, { + * beforeSend: async (message) => { + * if (await suppressed(message.recipients)) throw new Error('suppressed'); + * }, + * afterSend: (receipt, message, context) => audit(receipt, context), + * }); + * ``` + */ +export declare class HooksTransport + implements RawTransport, VerifiableTransport, AsyncDisposable +{ + readonly id: TProviderId; + /** The hooks, checked. */ + readonly config: Hooks & { readonly now: () => number }; + /** + * @param transport The transport to wrap. + * @param hooks The hooks. + * @throws {TypeError} If a hook is not a function. + */ + constructor(transport: Transport, hooks?: Hooks); + /** Sends one message between its hooks. */ + send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends a raw message between `beforeSendRaw` and `afterSend`. A wrapped + * transport without `sendRaw()` gives a failed receipt of the category + * `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Sends messages through the `sendMany()` of the wrapped transport, each + * between its hooks; one `beforeSend` refuses is answered in place. + */ + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the wrapped transport. */ + verify(options?: TransportOptions): Promise; + /** Disposes of the wrapped transport when it supports disposal. */ + [Symbol.asyncDispose](): Promise; +} diff --git a/src/hooks/index.js b/src/hooks/index.js new file mode 100644 index 0000000..b02ee8c --- /dev/null +++ b/src/hooks/index.js @@ -0,0 +1,6 @@ +// Ported from sently v1.2.2 (src/mailer.ts): the entry of the hooks decorator. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors + +const { HooksTransport } = require('./hooks-transport.js'); + +module.exports = { HooksTransport }; diff --git a/tests/types/hooks.test-d.ts b/tests/types/hooks.test-d.ts new file mode 100644 index 0000000..583edfe --- /dev/null +++ b/tests/types/hooks.test-d.ts @@ -0,0 +1,40 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type Transport, + type VerifiableTransport, +} from '../../index.js'; +import { HooksTransport, type HookContext } from '../../src/hooks/index.js'; + +declare const inner: Transport<'postmark'>; +const hooked = new HooksTransport(inner, { + beforeSend: async (message, context) => { + expectType(context); + return message.subject === '' ? undefined : message; + }, + beforeSendRaw: (raw) => raw, + afterSend: (receipt, _message, context) => { + expectType(receipt); + expectType<'send' | 'sendMany' | 'sendRaw'>(context.operation); + }, + onRetry: (event) => { + expectType(event.attempt); + }, + onError: async () => {}, + now: () => 0, +}); +expectType<'postmark'>(hooked.id); +expectAssignable>(hooked); +expectAssignable>(hooked); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(hooked.send(message)); +expectType>>(hooked.sendMany([message])); +new HooksTransport(inner); +expectError(new HooksTransport(inner, { afterSend: 'log' })); diff --git a/tests/unit/hooks.test.js b/tests/unit/hooks.test.js new file mode 100644 index 0000000..41aa2e9 --- /dev/null +++ b/tests/unit/hooks.test.js @@ -0,0 +1,222 @@ +// Not ported: tests of ./hooks, ported from sently's mailer hooks, against a +// fake transport. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { HooksTransport } = require('../../src/hooks/index.js'); +const { collect, failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const message = (subject = 'Hello') => + createMessage({ from: 'a@example.com', to: 'b@example.com', subject, content: { text: 'x' } }); + +test('Hooks: beforeSend may replace the message, afterSend hears the receipt', async () => { + const inner = fakeTransport(); + const heard = []; + let time = 100; + const hooked = new HooksTransport(inner, { + beforeSend: (m) => (m.subject === 'swap' ? message('swapped') : undefined), + afterSend: async (receipt, m, context) => { + heard.push([receipt.messageId, m.subject, context.operation, context.startedAt]); + }, + now: () => time++, + }); + await hooked.send(message('keep')); + await hooked.send(message('swap')); + assert.deepStrictEqual( + inner.calls.map((call) => call.message.subject), + ['keep', 'swapped'], + ); + assert.deepStrictEqual(heard, [ + ['m0', 'keep', 'send', 100], + ['m1', 'swapped', 'send', 101], + ]); +}); + +test('Hooks: a beforeSend that throws refuses the message, unsent', async () => { + const inner = fakeTransport(); + const hooked = new HooksTransport(inner, { + beforeSend: (m) => { + if (m.subject === 'suppressed') throw new Error('on the suppression list'); + if (m.subject === 'odd') return 'not a message'; + return undefined; + }, + }); + const refused = await hooked.send(message('suppressed')); + assert.strictEqual(refused.successful, false); + assert.strictEqual(refused.retryable, false); + assert.strictEqual(refused.errors[0].category, 'rejected'); + assert.strictEqual(refused.errors[0].code, 'hooks.refused'); + assert.strictEqual( + refused.errorMessages[0], + 'The beforeSend hook refused the message: on the suppression list', + ); + const odd = await hooked.send(message('odd')); + assert.match(odd.errorMessages[0], /returned neither a message nor nothing/); + assert.strictEqual(inner.calls.length, 0); + const literal = new HooksTransport(inner, { + beforeSend: () => { + // oxlint-disable-next-line no-throw-literal -- a hook may throw anything + throw 'no'; + }, + }); + assert.match((await literal.send(message())).errorMessages[0], /refused the message: no$/); +}); + +test('Hooks: each send hears its own retries, the caller still hears them all', async () => { + // Changed in the port: sently set onRetry on the shared transport, so of two + // concurrent sends, one heard the other's retries. + const inner = fakeTransport(async (m, _index, options) => { + await new Promise((resolve) => setImmediate(resolve)); + options?.onRetry?.({ attempt: 1, delayMilliseconds: 10, error: new Error(m.subject) }); + return sent(m.subject); + }); + const heard = []; + const callerHeard = []; + const hooked = new HooksTransport(inner, { + beforeSend: () => undefined, + onRetry: (event, context) => { + heard.push([event.error.message, context.operation]); + if (event.error.message === 'b') throw new Error('hook broke'); + }, + }); + const warnings = []; + const original = process.emitWarning; + process.emitWarning = (text, options) => warnings.push([text, options.code]); + try { + await Promise.all([ + hooked.send(message('a'), { onRetry: (event) => callerHeard.push(event.error.message) }), + hooked.send(message('b'), { onRetry: (event) => callerHeard.push(event.error.message) }), + ]); + } finally { + process.emitWarning = original; + } + assert.deepStrictEqual(heard.sort(), [ + ['a', 'send'], + ['b', 'send'], + ]); + assert.deepStrictEqual(callerHeard.sort(), ['a', 'b']); + assert.deepStrictEqual(warnings, [ + ['The onRetry hook of HooksTransport threw: hook broke', 'MAILER_HOOK_FAILED'], + ]); + // Without an onRetry hook, the options go as they are. + const plain = new HooksTransport(inner); + const options = { signal: new AbortController().signal }; + await plain.send(message('c'), options); + assert.strictEqual(inner.calls.at(-1).options, options); +}); + +test('Hooks: a hook that throws after the send is a warning, the receipt stands', async (t) => { + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const inner = fakeTransport((m) => + m.subject === 'boom' ? { throw: new Error('socket') } : sent('ok'), + ); + const onError = []; + const hooked = new HooksTransport(inner, { + afterSend: () => { + throw new Error('audit down'); + }, + onError: async (error, m, context) => { + onError.push([error.message, m.subject, context.operation]); + throw new Error('alert down'); + }, + }); + assert.strictEqual((await hooked.send(message())).messageId, 'ok'); + await assert.rejects(hooked.send(message('boom')), /socket/); + assert.deepStrictEqual(onError, [['socket', 'boom', 'send']]); + assert.deepStrictEqual(warnings, [ + ['The afterSend hook of HooksTransport threw: audit down', 'MAILER_HOOK_FAILED'], + ['The onError hook of HooksTransport threw: alert down', 'MAILER_HOOK_FAILED'], + ]); +}); + +test('Hooks: sendMany calls the hooks of each message, through the wrapped sendMany', async () => { + const inner = fakeTransport( + (m, index) => (m.subject === 'bad' ? failed('validation') : sent(`m${index}`)), + { + readAhead: 1, + }, + ); + const after = []; + const retries = []; + const hooked = new HooksTransport(inner, { + beforeSend: (m) => { + if (m.subject === 'refuse') throw new Error('no'); + }, + afterSend: (receipt, m, context) => + after.push([m.subject, receipt.successful, context.operation]), + onRetry: (_event, context) => retries.push(context.operation), + }); + const receipts = await collect( + hooked.sendMany([message('one'), message('refuse'), message('bad'), message('two')]), + ); + assert.deepStrictEqual( + receipts.map( + (receipt) => receipt.successful || receipt.errors[0].code || receipt.errors[0].category, + ), + [true, 'hooks.refused', 'validation', true], + ); + assert.deepStrictEqual(after, [ + ['one', true, 'sendMany'], + ['bad', false, 'sendMany'], + ['two', true, 'sendMany'], + ]); + assert.ok(inner.calls.every((call) => typeof call.options.onRetry === 'function')); + inner.calls[0].options.onRetry({ attempt: 1, delayMilliseconds: 1, error: new Error('x') }); + assert.deepStrictEqual(retries, ['sendMany']); + const errors = []; + const exploding = new HooksTransport( + fakeTransport(undefined, { + overrides: { + // oxlint-disable-next-line require-yield + async *sendMany(messages) { + await messages[Symbol.asyncIterator]().next(); + throw new Error('batch exploded'); + }, + }, + }), + { onError: (error, m) => errors.push([error.message, m.subject]) }, + ); + await assert.rejects(collect(exploding.sendMany([message('x')])), /batch exploded/); + assert.deepStrictEqual(errors, [['batch exploded', 'x']]); +}); + +test('Hooks: sendRaw, verify, dispose and the options', async () => { + const inner = fakeTransport(); + const raws = []; + const hooked = new HooksTransport(inner, { + beforeSendRaw: (raw) => ({ ...raw, envelope: { ...raw.envelope, to: ['c@example.com'] } }), + afterSend: (_receipt, raw, context) => raws.push([raw.envelope.to[0], context.operation]), + }); + const raw = { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }; + await hooked.sendRaw(raw); + assert.deepStrictEqual(inner.calls[0].message.envelope.to, ['c@example.com']); + assert.deepStrictEqual(raws, [['c@example.com', 'sendRaw']]); + const refusing = new HooksTransport(inner, { + beforeSendRaw: () => { + throw new Error('no raw'); + }, + }); + assert.match((await refusing.sendRaw(raw)).errorMessages[0], /beforeSendRaw hook refused/); + await hooked.verify(); + assert.strictEqual(inner.verified, 1); + await hooked[Symbol.asyncDispose](); + assert.strictEqual(inner.disposed, 1); + const plain = { id: 'plain', send: async () => sent('p') }; + const bare = new HooksTransport(plain); + assert.strictEqual((await bare.sendRaw(raw)).errors[0].category, 'configuration'); + await bare.verify(); + await bare[Symbol.asyncDispose](); + let disposed = 0; + await new HooksTransport({ ...plain, [Symbol.dispose]: () => disposed++ })[Symbol.asyncDispose](); + assert.strictEqual(disposed, 1); + assert.strictEqual(new HooksTransport(inner).id, 'fake'); + assert.throws( + () => new HooksTransport(inner, { afterSend: 'log' }), + /afterSend must be a function/, + ); + await assert.rejects( + bare.send(message(), { signal: AbortSignal.abort(new Error('already')) }), + /already/, + ); +}); From 0fc2dbe7cafb3f71e9922186a431a2404b891bce Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:48:53 +0300 Subject: [PATCH 039/143] feat(tee): a decorator that mirrors each message to other transports Ported from unemail's tee driver as a decorator of a primary transport, whose receipt is the one returned. Each mirror's send has a timeout of its own and by default is not awaited, where unemail awaited every leg with no bound, so a slow or hung mirror held the primary's receipt (U13); the copies in flight are bounded by `maxPending`, and `awaitMirrors` waits for them, bounded, following the caller's signal. `onMirror` hears every copy's outcome; without it a failed or skipped copy becomes a process warning. `sendMany` goes through the primary's `sendMany`, `sendRaw` to the mirrors that send raw, and disposal disposes of every transport once. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/transports/tee.md | 47 +++++++++ package.json | 4 + scripts/size-budgets.json | 3 +- src/tee/config.js | 41 ++++++++ src/tee/index.d.ts | 124 ++++++++++++++++++++++ src/tee/index.js | 6 ++ src/tee/tee-transport.js | 183 +++++++++++++++++++++++++++++++++ tests/types/tee.test-d.ts | 34 +++++++ tests/unit/tee.test.js | 204 +++++++++++++++++++++++++++++++++++++ 12 files changed, 651 insertions(+), 1 deletion(-) create mode 100644 docs/transports/tee.md create mode 100644 src/tee/config.js create mode 100644 src/tee/index.d.ts create mode 100644 src/tee/index.js create mode 100644 src/tee/tee-transport.js create mode 100644 tests/types/tee.test-d.ts create mode 100644 tests/unit/tee.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 2310c32..30336cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -575,6 +575,10 @@ fails on upstream's behavior): with `console.warn` outside production and dropped in it. `./hooks` makes the context of each send and passes `onRetry` in the options of that send, and a hook that throws after the send becomes a process warning (`MAILER_HOOK_FAILED`). +- U13. unemail (`src/drivers/tee.ts`): every leg was awaited with no bound, so the primary's + result waited for the slowest mirror, and a mirror that hung held the send for as long as it + hung. `./tee` gives each mirror a timeout of its own and by default does not await it, bounds + the copies in flight, and awaits them, bounded, only with `awaitMirrors`. ## Repository layout diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 5f99e24..b1ad368 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -110,6 +110,7 @@ const guide = [ { text: 'Rate limit', link: '/transports/ratelimit' }, { text: 'Idempotency', link: '/transports/idempotency' }, { text: 'Hooks', link: '/transports/hooks' }, + { text: 'Tee', link: '/transports/tee' }, ], }, ]; diff --git a/docs/api/exports.md b/docs/api/exports.md index 50a807d..87a5c01 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -73,6 +73,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/ratelimit` | `RateLimitTransport` | [Rate limit](/transports/ratelimit) | | `@alexify/mailer/idempotency` | `IdempotencyTransport`, `createMemoryIdempotencyStore` | [Idempotency](/transports/idempotency) | | `@alexify/mailer/hooks` | `HooksTransport` | [Hooks](/transports/hooks) | +| `@alexify/mailer/tee` | `TeeTransport` | [Tee](/transports/tee) | The types of each subpath (configurations, receipts with provider details, the structural `TracerProvider`, `MeterProvider` and `Logger` of the decorators) are in diff --git a/docs/transports/tee.md b/docs/transports/tee.md new file mode 100644 index 0000000..dc2acac --- /dev/null +++ b/docs/transports/tee.md @@ -0,0 +1,47 @@ +# Tee + +`TeeTransport` sends each message through a primary transport and mirrors it +to others: to shadow a new provider against the one in use, or to copy real mail +into a local catcher while developing. The primary's receipt is the one +returned, whatever the mirrors do. It comes from the tee driver of unemail. + +## Basic usage + +```js +const { TeeTransport } = require('@alexify/mailer/tee'); + +const transport = new TeeTransport(ses, { + mirrors: [postmark], + onMirror: ({ mirror, receipt, error, skipped }) => + metrics.record('shadow', { mirror, ok: receipt?.successful === true, error, skipped }), +}); +``` + +## Mirrors never hold the primary + +Each mirror's send has a timeout of its own (`timeout`, 10 seconds), and by +default it is not awaited: the primary's receipt comes back as soon as the +primary answers, and the copies go on alone, not stopped by the caller's +signal. At most `maxPending` copies are in flight; one over that is skipped and +reported. On a runtime that freezes after the response, such as a serverless +function, set `awaitMirrors`: the copies are then awaited, bounded by the +timeout, and follow the caller's signal. + +`onMirror` hears the outcome of every copy: its `receipt`, the `error` its send +threw, or why it was `skipped`. Without it, a failed or skipped copy becomes a +process warning (`MAILER_TEE_MIRROR_FAILED`). + +## Options + +| Option | Default | Description | +| --- | --- | --- | +| `mirrors` | (required) | The transports each message is mirrored to. | +| `timeout` | `10000` | The time a mirror's send may take, in milliseconds. | +| `awaitMirrors` | `false` | Waits for the copies before returning. | +| `maxPending` | `100` | The most copies in flight while not awaited. | +| `onMirror` | | Hears every outcome of a mirror. | + +`sendMany()` goes through the `sendMany()` of the primary, each message mirrored +as it is handed to it. `sendRaw()` mirrors to the mirrors that send raw +messages and reports the others as skipped. `verify()` checks the primary +alone, and disposal disposes of the primary and every mirror once. diff --git a/package.json b/package.json index df57d99..dcd4600 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,10 @@ "./hooks": { "types": "./src/hooks/index.d.ts", "default": "./src/hooks/index.js" + }, + "./tee": { + "types": "./src/tee/index.d.ts", + "default": "./src/tee/index.js" } }, "directories": { diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index bfb85aa..c07ae9b 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -31,5 +31,6 @@ "./breaker": 2, "./ratelimit": 2.5, "./idempotency": 3, - "./hooks": 2 + "./hooks": 2, + "./tee": 2 } diff --git a/src/tee/config.js b/src/tee/config.js new file mode 100644 index 0000000..7d9c086 --- /dev/null +++ b/src/tee/config.js @@ -0,0 +1,41 @@ +// Ported from unemail v0.7.0 (src/drivers/tee.ts): the options of the tee. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { + checkOptionalFunction, + checkPositiveInteger, + checkPositiveNumber, +} = require('../internal/options.js'); + +/** + * Resolves the options of the tee. + * + * @param config The options. + * @returns The options with their defaults. + * @throws {TypeError} If an option is invalid. + * @internal + */ +const createTeeConfig = (config = {}) => { + const mirrors = config.mirrors; + if (!Array.isArray(mirrors) || mirrors.length === 0) { + throw new TypeError('mirrors must be a non-empty array of transports.'); + } + for (const mirror of mirrors) { + if (typeof mirror?.send !== 'function') { + throw new TypeError('Every mirror must be a transport with a send method.'); + } + } + const awaitMirrors = config.awaitMirrors ?? false; + if (typeof awaitMirrors !== 'boolean') { + throw new TypeError('awaitMirrors must be a boolean.'); + } + return { + mirrors: [...mirrors], + timeout: checkPositiveNumber(config.timeout ?? 10000, 'timeout'), + awaitMirrors, + maxPending: checkPositiveInteger(config.maxPending ?? 100, 'maxPending'), + onMirror: checkOptionalFunction(config.onMirror, 'onMirror'), + }; +}; + +module.exports = { createTeeConfig }; diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts new file mode 100644 index 0000000..5c793c0 --- /dev/null +++ b/src/tee/index.d.ts @@ -0,0 +1,124 @@ +// Types ported from unemail v0.7.0 (src/drivers) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +/// + +import type { + Message, + RawMessage, + RawTransport, + Receipt, + Transport, + TransportOptions, + VerifiableTransport, +} from '../../index.js'; + +/** + * What became of the copy of a message sent to a mirror: its receipt, the + * error its send threw, or why it was skipped. + */ +export interface MirrorOutcome { + /** The id of the mirror. */ + readonly mirror: string; + /** The message mirrored. */ + readonly message: Message | RawMessage; + readonly receipt?: Receipt; + readonly error?: unknown; + /** Why the copy was not sent. */ + readonly skipped?: string; +} + +/** + * The options of {@link TeeTransport}. + */ +export interface TeeConfig { + /** The transports each message is mirrored to. */ + readonly mirrors: readonly Transport[]; + /** + * The time a mirror's send may take, in milliseconds. + * @default 10000 + */ + readonly timeout?: number; + /** + * Waits for the mirrors before returning the primary's receipt, as a + * runtime that freezes after the response needs; they follow the caller's + * signal then. By default their sends go on alone. + * @default false + */ + readonly awaitMirrors?: boolean; + /** + * The most mirrored sends in flight while not awaited; a copy over it is + * skipped. + * @default 100 + */ + readonly maxPending?: number; + /** + * Hears every outcome of a mirror. Without it, a failure or a skipped copy + * becomes a process warning (`MAILER_TEE_MIRROR_FAILED`); one that throws + * becomes one too (`MAILER_TEE_LISTENER_FAILED`). + */ + readonly onMirror?: (outcome: MirrorOutcome) => void; +} + +/** + * The options of {@link TeeTransport}, with their defaults. + */ +export interface ResolvedTeeConfig { + readonly mirrors: readonly Transport[]; + readonly timeout: number; + readonly awaitMirrors: boolean; + readonly maxPending: number; + readonly onMirror?: (outcome: MirrorOutcome) => void; +} + +/** + * A transport that sends each message through a primary transport and + * mirrors it to others, to shadow a new provider or copy mail into a local + * catcher. The primary's receipt is the one returned, whatever the mirrors + * do: each has a timeout of its own and, by default, its send is not awaited. + * + * @example + * ```js + * const { TeeTransport } = require('@alexify/mailer/tee'); + * const shadowed = new TeeTransport(ses, { + * mirrors: [postmark], + * onMirror: (outcome) => metrics.record(outcome), + * }); + * ``` + */ +export declare class TeeTransport + implements RawTransport, VerifiableTransport, AsyncDisposable +{ + readonly id: TProviderId; + /** The resolved options. */ + readonly config: ResolvedTeeConfig; + /** + * @param primary The transport whose receipt is returned. + * @param config The options. + * @throws {TypeError} If an option is invalid. + */ + constructor(primary: Transport, config: TeeConfig); + /** Sends a message through the primary and mirrors it. */ + send(message: Message, options?: TransportOptions): Promise>; + /** + * Sends a raw message through the primary, mirrored to the mirrors that send + * raw messages. A primary without `sendRaw()` gives a failed receipt of the + * category `configuration`. + */ + sendRaw(message: RawMessage, options?: TransportOptions): Promise>; + /** + * Sends messages through the `sendMany()` of the primary, each mirrored as it + * is handed to it. + */ + sendMany( + messages: Iterable | AsyncIterable, + options?: TransportOptions, + ): AsyncIterable>; + /** Verifies the primary; the mirrors are not checked. */ + verify(options?: TransportOptions): Promise; + /** + * Disposes of the primary and the mirrors, each once, whatever fails; the + * failures reject together as an `AggregateError`. + */ + [Symbol.asyncDispose](): Promise; +} diff --git a/src/tee/index.js b/src/tee/index.js new file mode 100644 index 0000000..06ca614 --- /dev/null +++ b/src/tee/index.js @@ -0,0 +1,6 @@ +// Ported from unemail v0.7.0 (src/drivers/tee.ts): the entry of the tee. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +const { TeeTransport } = require('./tee-transport.js'); + +module.exports = { TeeTransport }; diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js new file mode 100644 index 0000000..9f12c69 --- /dev/null +++ b/src/tee/tee-transport.js @@ -0,0 +1,183 @@ +// Ported from unemail v0.7.0 (src/drivers/tee.ts): a transport that sends each +// message through a primary transport and mirrors it to others, the primary's +// receipt the one returned. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a mirror has a timeout of its own and, by default, its +// send is not awaited, where unemail awaited every leg with no bound, so a +// slow or hung mirror held the primary's receipt; with `awaitMirrors` they +// are awaited, bounded by the timeout; and the detached sends in flight are +// bounded too. + +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { warnThrown } = require('../internal/callback.js'); +const { refuseRaw } = require('../internal/capabilities.js'); +const { gatedSendMany } = require('../internal/gate.js'); +const { createTeeConfig } = require('./config.js'); + +const disposeOf = async (transport) => { + const target = Object(transport); + if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); + else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); +}; + +const describe = (outcome) => { + if (outcome.skipped !== undefined) return outcome.skipped; + if (outcome.error !== undefined) { + return outcome.error instanceof Error ? outcome.error.message : String(outcome.error); + } + return outcome.receipt.errorMessages[0]; +}; + +/** + * A transport that sends each message through `primary` and mirrors it to + * `mirrors`, which never change the primary's receipt. + */ +class TeeTransport { + id; + config; + primary; + #pending = 0; + + /** + * @param primary The transport whose receipt is returned. + * @param config `mirrors`, `timeout`, `awaitMirrors`, `maxPending` and + * `onMirror`. + * @throws {TypeError} If an option is invalid. + */ + constructor(primary, config) { + this.primary = primary; + this.id = primary.id; + this.config = createTeeConfig(config); + } + + /** + * Sends a message through the primary and mirrors it. + * + * @returns The receipt of the primary. + * @throws What the primary throws. + */ + async send(message, options) { + options?.signal?.throwIfAborted(); + const mirrored = this.#mirror(message, options?.signal, 'send'); + const receipt = await this.primary.send(message, options); + await mirrored; + return receipt; + } + + /** + * Sends a raw message through the primary, mirrored to the mirrors that + * send raw messages. A primary without `sendRaw()` gives a failed receipt + * of the category `configuration`. + */ + async sendRaw(message, options) { + options?.signal?.throwIfAborted(); + if (!isRawTransport(this.primary)) return refuseRaw(this.primary); + const mirrored = this.#mirror(message, options?.signal, 'sendRaw'); + const receipt = await this.primary.sendRaw(message, options); + await mirrored; + return receipt; + } + + /** + * Sends messages through the `sendMany()` of the primary, each mirrored as + * it is handed to it. + */ + sendMany(messages, options) { + return gatedSendMany(this.primary, messages, options, { + admit: (message) => ({ message, mirrored: this.#mirror(message, options?.signal, 'send') }), + settle: async (receipt, admitted) => { + await admitted.mirrored; + return receipt; + }, + }); + } + + /** Verifies the primary; the mirrors are not checked. */ + async verify(options) { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(this.primary)) await this.primary.verify(options); + } + + /** + * Disposes of the primary and the mirrors, each once, all of them whatever + * fails; the failures reject together. + */ + async [Symbol.asyncDispose]() { + const seen = new Set(); + const errors = []; + const disposeOnce = async (transport) => { + if (seen.has(transport)) return; + seen.add(transport); + try { + await disposeOf(transport); + } catch (error) { + errors.push(error); + } + }; + await disposeOnce(this.primary); + for (const mirror of this.config.mirrors) await disposeOnce(mirror); + if (errors.length > 0) throw new AggregateError(errors, 'Disposing of the tee failed.'); + } + + // Starts the sends of the mirrors: awaited by the caller with + // `awaitMirrors`, otherwise a promise already settled. + #mirror(message, signal, operation) { + const sends = []; + for (const mirror of this.config.mirrors) { + sends.push(this.#mirrorOne(mirror, message, signal, operation)); + } + return this.config.awaitMirrors ? Promise.all(sends) : Promise.resolve(); + } + + async #mirrorOne(mirror, message, signal, operation) { + if (operation === 'sendRaw' && !isRawTransport(mirror)) { + this.#report({ mirror: mirror.id, message, skipped: 'it does not send raw messages' }); + return; + } + const detached = !this.config.awaitMirrors; + if (detached && this.#pending >= this.config.maxPending) { + this.#report({ + mirror: mirror.id, + message, + skipped: `${this.config.maxPending} mirrored sends are in flight`, + }); + return; + } + // A detached mirror outlives the caller's signal; an awaited one follows it. + const timeout = AbortSignal.timeout(this.config.timeout); + const mirrorSignal = + detached || signal === undefined ? timeout : AbortSignal.any([signal, timeout]); + if (detached) this.#pending++; + try { + const receipt = await (operation === 'sendRaw' + ? mirror.sendRaw(message, { signal: mirrorSignal }) + : mirror.send(message, { signal: mirrorSignal })); + this.#report({ mirror: mirror.id, message, receipt }); + } catch (error) { + this.#report({ mirror: mirror.id, message, error }); + } finally { + if (detached) this.#pending--; + } + } + + // Tells `onMirror` of an outcome; without it, a failure becomes a process + // warning rather than vanishing. + #report(outcome) { + const listener = this.config.onMirror; + if (listener === undefined) { + if (outcome.receipt?.successful !== true) { + process.emitWarning(`The mirror ${outcome.mirror} failed: ${describe(outcome)}`, { + code: 'MAILER_TEE_MIRROR_FAILED', + }); + } + return; + } + try { + listener(outcome); + } catch (error) { + warnThrown('The onMirror of TeeTransport', error, 'MAILER_TEE_LISTENER_FAILED'); + } + } +} + +module.exports = { TeeTransport }; diff --git a/tests/types/tee.test-d.ts b/tests/types/tee.test-d.ts new file mode 100644 index 0000000..6abaea3 --- /dev/null +++ b/tests/types/tee.test-d.ts @@ -0,0 +1,34 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { + createMessage, + type RawTransport, + type Receipt, + type Transport, + type VerifiableTransport, +} from '../../index.js'; +import { TeeTransport, type MirrorOutcome } from '../../src/tee/index.js'; + +declare const primary: Transport<'ses'>; +declare const mirror: Transport<'mock'>; +const tee = new TeeTransport(primary, { + mirrors: [mirror], + timeout: 5000, + awaitMirrors: true, + maxPending: 10, + onMirror: (outcome) => { + expectType(outcome); + expectType(outcome.mirror); + }, +}); +expectType<'ses'>(tee.id); +expectAssignable>(tee); +expectAssignable>(tee); +const message = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'x', + content: { text: 'x' }, +}); +expectType>>(tee.send(message)); +expectType>>(tee.sendMany([message])); +expectError(new TeeTransport(primary, {})); diff --git a/tests/unit/tee.test.js b/tests/unit/tee.test.js new file mode 100644 index 0000000..6d473d3 --- /dev/null +++ b/tests/unit/tee.test.js @@ -0,0 +1,204 @@ +// Not ported: tests of ./tee, ported from unemail's tee driver, against fake +// transports. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { TeeTransport } = require('../../src/tee/index.js'); +const { collect, failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const message = (subject = 'Hello') => + createMessage({ from: 'a@example.com', to: 'b@example.com', subject, content: { text: 'x' } }); + +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +// Waits until a condition holds, for a timer that a busy machine may delay. +const waitFor = async (condition) => { + const deadline = Date.now() + 5000; + while (!condition()) { + if (Date.now() > deadline) throw new Error('timed out waiting'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +}; + +// A mirror that answers only when its signal aborts. +const hanging = () => + fakeTransport( + (_m, _index, options) => + new Promise((_resolve, reject) => { + options.signal.addEventListener('abort', () => reject(options.signal.reason)); + }), + ); + +test('Tee: the primary answers at once; a hung mirror is cut by its own timeout', async () => { + const primary = fakeTransport((_m, index) => sent(`p${index}`)); + const mirror = hanging(); + const outcomes = []; + const tee = new TeeTransport(primary, { + mirrors: [mirror], + timeout: 20, + onMirror: (outcome) => outcomes.push(outcome), + }); + // Changed in the port: unemail awaited every leg with no bound. + const receipt = await tee.send(message()); + assert.strictEqual(receipt.messageId, 'p0'); + assert.strictEqual(outcomes.length, 0); + assert.strictEqual(mirror.calls.length, 1); + await waitFor(() => outcomes.length === 1); + assert.strictEqual(outcomes[0].mirror, 'fake'); + assert.strictEqual(outcomes[0].error.name, 'TimeoutError'); +}); + +test('Tee: awaitMirrors waits for them, bounded, and they follow the caller', async () => { + const primary = fakeTransport(() => sent('p')); + const good = fakeTransport(() => sent('copy')); + const bad = fakeTransport(() => failed('network')); + const outcomes = []; + const tee = new TeeTransport(primary, { + mirrors: [good, bad, hanging()], + awaitMirrors: true, + timeout: 20, + onMirror: (outcome) => + outcomes.push( + outcome.receipt?.messageId ?? outcome.receipt?.errors[0].category ?? outcome.error.name, + ), + }); + assert.strictEqual((await tee.send(message())).messageId, 'p'); + assert.deepStrictEqual(outcomes.sort(), ['TimeoutError', 'copy', 'network'].sort()); + const controller = new AbortController(); + const caller = new TeeTransport(primary, { + mirrors: [hanging()], + awaitMirrors: true, + onMirror: (outcome) => outcomes.push(outcome.error.message), + }); + const pending = caller.send(message(), { signal: controller.signal }); + await settle(); + controller.abort(new Error('caller left')); + await pending; + assert.strictEqual(outcomes.at(-1), 'caller left'); +}); + +test('Tee: a mirror never changes the primary, which alone throws', async (t) => { + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const primary = fakeTransport((m) => + m.subject === 'boom' ? { throw: new Error('primary down') } : sent('p'), + ); + const thrower = fakeTransport(() => ({ throw: 'mirror down' })); + const failing = fakeTransport(() => failed('timeout')); + const tee = new TeeTransport(primary, { mirrors: [thrower, failing], awaitMirrors: true }); + assert.strictEqual((await tee.send(message())).messageId, 'p'); + await assert.rejects(tee.send(message('boom')), /primary down/); + assert.deepStrictEqual(warnings.slice(0, 2), [ + ['The mirror fake failed: mirror down', 'MAILER_TEE_MIRROR_FAILED'], + ['The mirror fake failed: timeout failure', 'MAILER_TEE_MIRROR_FAILED'], + ]); + const listening = new TeeTransport(primary, { + mirrors: [failing], + awaitMirrors: true, + onMirror: () => { + throw new Error('listener broke'); + }, + }); + await listening.send(message()); + assert.deepStrictEqual(warnings.at(-1), [ + 'The onMirror of TeeTransport threw: listener broke', + 'MAILER_TEE_LISTENER_FAILED', + ]); +}); + +test('Tee: detached copies in flight are bounded', async () => { + const primary = fakeTransport(() => sent('p')); + const slow = hanging(); + const outcomes = []; + const tee = new TeeTransport(primary, { + mirrors: [slow], + maxPending: 2, + timeout: 20, + onMirror: (outcome) => outcomes.push(outcome.skipped ?? outcome.error.name), + }); + for (let i = 0; i < 3; i++) await tee.send(message()); + assert.strictEqual(slow.calls.length, 2); + assert.deepStrictEqual(outcomes, ['2 mirrored sends are in flight']); + await waitFor(() => outcomes.length === 3); + await tee.send(message()); + assert.strictEqual(slow.calls.length, 3); +}); + +test('Tee: sendMany mirrors each message handed to the primary', async () => { + const primary = fakeTransport(undefined, { readAhead: 1 }); + const mirror = fakeTransport(() => sent('copy')); + const tee = new TeeTransport(primary, { mirrors: [mirror], awaitMirrors: true }); + const receipts = await collect(tee.sendMany([message('a'), message('b'), message('c')])); + assert.deepStrictEqual( + receipts.map((receipt) => receipt.messageId), + ['m0', 'm1', 'm2'], + ); + assert.deepStrictEqual( + mirror.calls.map((call) => call.message.subject), + ['a', 'b', 'c'], + ); + assert.ok(primary.calls.every((call) => call.operation === 'sendMany')); +}); + +test('Tee: sendRaw goes to the mirrors that send raw, verify and dispose', async (t) => { + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const primary = fakeTransport(); + const rawMirror = fakeTransport(() => sent('copy')); + const plainMirror = { id: 'plain', send: async () => sent('x') }; + const tee = new TeeTransport(primary, { + mirrors: [rawMirror, plainMirror, rawMirror], + awaitMirrors: true, + }); + const raw = { envelope: { from: 'a@example.com', to: ['b@example.com'] }, content: 'x' }; + assert.strictEqual((await tee.sendRaw(raw)).messageId, 'm0'); + assert.strictEqual(rawMirror.calls.filter((call) => call.operation === 'sendRaw').length, 2); + assert.deepStrictEqual(warnings, [ + ['The mirror plain failed: it does not send raw messages', 'MAILER_TEE_MIRROR_FAILED'], + ]); + await tee.verify(); + assert.strictEqual(primary.verified, 1); + await tee[Symbol.asyncDispose](); + // Each transport once, the repeated mirror too. + assert.strictEqual(primary.disposed, 1); + assert.strictEqual(rawMirror.disposed, 1); + const bare = new TeeTransport(plainMirror, { mirrors: [rawMirror] }); + assert.strictEqual((await bare.sendRaw(raw)).errors[0].category, 'configuration'); + await bare.verify(); + const broken = { + id: 'broken', + send: async () => sent('b'), + [Symbol.dispose]: () => { + throw new Error('cannot close'); + }, + }; + await assert.rejects( + new TeeTransport(broken, { mirrors: [rawMirror] })[Symbol.asyncDispose](), + (error) => error instanceof AggregateError && error.errors[0].message === 'cannot close', + ); + await assert.rejects( + bare.send(message(), { signal: AbortSignal.abort(new Error('already')) }), + /already/, + ); +}); + +test('Tee: the options are checked', () => { + const primary = fakeTransport(); + assert.throws(() => new TeeTransport(primary, {}), /mirrors must be a non-empty array/); + assert.throws(() => new TeeTransport(primary, { mirrors: [] }), /non-empty/); + assert.throws(() => new TeeTransport(primary, { mirrors: [{}] }), /with a send method/); + assert.throws(() => new TeeTransport(primary, { mirrors: [primary], timeout: 0 }), /timeout/); + assert.throws( + () => new TeeTransport(primary, { mirrors: [primary], awaitMirrors: 'yes' }), + /awaitMirrors/, + ); + assert.throws( + () => new TeeTransport(primary, { mirrors: [primary], maxPending: 0 }), + /maxPending/, + ); + const tee = new TeeTransport(primary, { mirrors: [primary] }); + assert.strictEqual(tee.id, 'fake'); + assert.strictEqual(tee.config.timeout, 10000); + assert.strictEqual(tee.config.awaitMirrors, false); + assert.strictEqual(tee.config.maxPending, 100); +}); From 152852e3dbabcd62b98c1dfbe2c83f64dccfada2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:49:42 +0300 Subject: [PATCH 040/143] test: the decorators of 1.1 composed with a pool An open circuit takes its provider out of a pool, which fails over to the next without calling it; idempotency outside hooks and a pool reserves a message once for all its attempts and answers a repeat with the first receipt. Co-Authored-By: Claude Opus 5.5 --- tests/unit/decorators.test.js | 63 +++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/unit/decorators.test.js diff --git a/tests/unit/decorators.test.js b/tests/unit/decorators.test.js new file mode 100644 index 0000000..35c735b --- /dev/null +++ b/tests/unit/decorators.test.js @@ -0,0 +1,63 @@ +// Not ported: the decorators of 1.1 composed with ./pool, as the docs suggest. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('../../index.js'); +const { CircuitBreakerTransport } = require('../../src/breaker/index.js'); +const { HooksTransport } = require('../../src/hooks/index.js'); +const { IdempotencyTransport } = require('../../src/idempotency/index.js'); +const { PoolTransport } = require('../../src/pool/index.js'); +const { failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); + +const message = (fields = {}) => + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Hello', + content: { text: 'x' }, + ...fields, + }); + +test('Decorators: an open circuit takes its provider out of a pool', async () => { + const down = fakeTransport(() => failed('network')); + down.id = 'down'; + const up = fakeTransport((_m, index) => sent(`up${index}`)); + up.id = 'up'; + const pool = new PoolTransport({ + transports: [ + { transport: new CircuitBreakerTransport(down, { failureThreshold: 2 }), priority: 2 }, + { transport: up, priority: 1 }, + ], + strategy: 'priority', + }); + for (let i = 0; i < 4; i++) assert.strictEqual((await pool.send(message())).successful, true); + // Two failures opened the circuit; the next sends went straight to the other. + assert.strictEqual(down.calls.length, 2); + assert.strictEqual(up.calls.length, 4); +}); + +test('Decorators: idempotency outside a pool reserves a message once for all its attempts', async () => { + const flaky = fakeTransport(() => failed('server-error')); + flaky.id = 'flaky'; + const steady = fakeTransport((_m, index) => sent(`s${index}`)); + steady.id = 'steady'; + const heard = []; + const transport = new IdempotencyTransport( + new HooksTransport( + new PoolTransport({ + transports: [ + { transport: flaky, priority: 2 }, + { transport: steady, priority: 1 }, + ], + strategy: 'priority', + }), + { afterSend: (receipt) => heard.push(receipt.messageId) }, + ), + ); + const first = await transport.send(message({ idempotencyKey: 'order-1' })); + const again = await transport.send(message({ idempotencyKey: 'order-1' })); + assert.strictEqual(first.messageId, 's0'); + assert.deepStrictEqual(again, first); + assert.strictEqual(flaky.calls.length, 1); + assert.strictEqual(steady.calls.length, 1); + assert.deepStrictEqual(heard, ['s0']); +}); From 0dc23b76f9ee4120845907758b4a03a9d26a56f3 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 22:57:31 +0300 Subject: [PATCH 041/143] feat(render): render templates into the content of a message A new subpath `./render`, ported from unemail's renderers and sently's React plugin. `renderContent(renderer, input)` gives `{ html, text }` for `createMessage`, the text derived from the HTML when the renderer gives none; `createReactEmailRenderer({ render })` renders React Email components with the `render` of `@react-email/render` the application passes in, its text from React Email's plain-text pass; and `htmlToText` reads HTML in one pass, linear in its length. unemail's `htmlToText` threw on `�`, left comments and the doctype in the text (React Email writes `` into every body), read an `href` in double quotes only, and was quadratic on unclosed ` --- CLAUDE.md | 7 + docs/.vitepress/config.mts | 1 + docs/api/exports.md | 1 + docs/messages/render.md | 73 +++++++++ package.json | 7 + pnpm-lock.yaml | 193 +++++++++++++++++++++-- scripts/size-budgets.json | 1 + src/render/entities.js | 78 +++++++++ src/render/html-to-text.js | 296 +++++++++++++++++++++++++++++++++++ src/render/index.d.ts | 106 +++++++++++++ src/render/index.js | 7 + src/render/react-email.js | 42 +++++ src/render/renderer.js | 45 ++++++ tests/types/render.test-d.ts | 27 ++++ tests/unit/render.test.js | 199 +++++++++++++++++++++++ 15 files changed, 1073 insertions(+), 10 deletions(-) create mode 100644 docs/messages/render.md create mode 100644 src/render/entities.js create mode 100644 src/render/html-to-text.js create mode 100644 src/render/index.d.ts create mode 100644 src/render/index.js create mode 100644 src/render/react-email.js create mode 100644 src/render/renderer.js create mode 100644 tests/types/render.test-d.ts create mode 100644 tests/unit/render.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 30336cf..2745f4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -579,6 +579,13 @@ fails on upstream's behavior): result waited for the slowest mirror, and a mirror that hung held the send for as long as it hung. `./tee` gives each mirror a timeout of its own and by default does not await it, bounds the copies in flight, and awaits them, bounded, only with `awaitMirrors`. +- U14. unemail (`src/render/_html.ts`, `src/render/react.ts`): `htmlToText` threw a RangeError on + a numeric reference past U+10FFFF (`�`), left comments and the doctype in the text + (React Email writes `` into every body), read an `href` in double quotes only, and its + lazy patterns rescanned the rest of the input for each unclosed ` ({ html: marked.parse(source) }) }; + +const content = await renderContent(markdown, '# Hello\n\nYour order has shipped.'); +// { html: '

Hello

\n

Your order has shipped.

\n', +// text: 'Hello\n\nYour order has shipped.' } +``` + +When a renderer gives HTML without text, `renderContent` derives the text with +`htmlToText`; `{ autoText: false }` leaves it without one. + +## htmlToText + +`htmlToText(html)` gives the plain text an HTML body reads as, for the text +alternative a message should carry: + +- the text of the elements, its whitespace collapsed as a browser shows it, + but in `pre`; +- a line for a block such as `div` or a table cell, a blank line for a + paragraph, a heading or a list; `- ` or `1. ` before a list item; +- the address of a link after its label, or alone when the label is empty or + the address itself; +- the character references decoded, an invalid number as U+FFFD. + +Scripts, styles, the `head`, comments, the doctype and elements marked +`data-skip-in-text="true"` (React Email's preview text) are left out. It is not +a DOM parser: it handles what a mail body holds, in one pass, in time linear in +its length. Set `text` yourself when fidelity matters. + +```js +htmlToText('

Hi there

  • one
'); +// 'Hi there (https://example.com/start)\n\n- one' +``` diff --git a/package.json b/package.json index dcd4600..09358a2 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,10 @@ "types": "./src/attachments/index.d.ts", "default": "./src/attachments/index.js" }, + "./render": { + "types": "./src/render/index.d.ts", + "default": "./src/render/index.js" + }, "./smtp": { "types": "./src/smtp/index.d.ts", "default": "./src/smtp/index.js" @@ -224,6 +228,7 @@ "@opentelemetry/sdk-metrics": "^1.30.1", "@opentelemetry/sdk-trace-base": "^1.30.1", "@opentelemetry/semantic-conventions": "^1.43.0", + "@react-email/render": "^2.1.1", "@vercel/analytics": "^2.0.1", "@vercel/speed-insights": "^2.0.0", "c8": "^12.0.0", @@ -232,6 +237,8 @@ "oxlint": "^1.75.0", "pino": "^10.4.0", "postal-mime": "^3.0.1", + "react": "^18.3.1", + "react-dom": "^18.3.1", "tsd": "^0.33.0", "typescript": "^7.0.2", "vitepress": "^1.6.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f521b7c..49b1080 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,12 +26,15 @@ importers: '@opentelemetry/semantic-conventions': specifier: ^1.43.0 version: 1.43.0 + '@react-email/render': + specifier: ^2.1.1 + version: 2.1.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@vercel/analytics': specifier: ^2.0.1 - version: 2.0.1(vue@3.5.44(typescript@7.0.2)) + version: 2.0.1(react@18.3.1)(vue@3.5.44(typescript@7.0.2)) '@vercel/speed-insights': specifier: ^2.0.0 - version: 2.0.0(vue@3.5.44(typescript@7.0.2)) + version: 2.0.0(react@18.3.1)(vue@3.5.44(typescript@7.0.2)) c8: specifier: ^12.0.0 version: 12.0.0 @@ -50,6 +53,12 @@ importers: postal-mime: specifier: ^3.0.1 version: 3.0.1 + react: + specifier: ^18.3.1 + version: 18.3.1 + react-dom: + specifier: ^18.3.1 + version: 18.3.1(react@18.3.1) tsd: specifier: ^0.33.0 version: 0.33.0 @@ -58,7 +67,7 @@ importers: version: 7.0.2 vitepress: specifier: ^1.6.4 - version: 1.6.4(@algolia/client-search@5.60.0)(postcss@8.5.30)(search-insights@2.17.3)(typescript@7.0.2) + version: 1.6.4(@algolia/client-search@5.60.0)(postcss@8.5.30)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3)(typescript@7.0.2) packages: @@ -812,6 +821,13 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@react-email/render@2.1.1': + resolution: {integrity: sha512-DvVx4k6XjJ9cYBigd37+7SK2E89emUH32Lcn6COsMickQchXdvXa+ZA04BF7475Kg+8ikaIpus+zQYUwVVyjBg==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^18.0 || ^19.0 || ^19.0.0-rc + '@rollup/rollup-android-arm-eabi@4.64.6': resolution: {integrity: sha512-FkO19Fowv/OQeJZPVfdetJaI3/VpbZmBsnmAlR/rQ+b3QwfiGQiOpVegpSRH331kelvHTVw9hXYG3DwjAaRzyQ==} cpu: [arm] @@ -950,6 +966,9 @@ packages: cpu: [x64] os: [win32] + '@selderee/plugin-htmlparser2@0.11.0': + resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==} + '@shikijs/core@2.5.0': resolution: {integrity: sha512-uu/8RExTKtavlpH7XqnVYBrfBkUc20ngXiX9NSrBhOVZYv/7XQRKUyhtkeflY5QsxC0GbJThCerruZfsUaSldg==} @@ -1413,6 +1432,10 @@ packages: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -1428,6 +1451,19 @@ packages: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + emoji-regex-xs@1.0.0: resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} @@ -1437,6 +1473,10 @@ packages: emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + entities@7.0.1: resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} engines: {node: '>=0.12'} @@ -1557,9 +1597,19 @@ packages: html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + html-to-text@9.0.5: + resolution: {integrity: sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==} + engines: {node: '>=14'} + html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + html5parser@3.0.0: + resolution: {integrity: sha512-iNpSopa+4YHX50UOk825tBy7MghmXHo/ZpLskBYN0kAr1xhH8GlIMk5bLRXcZlfP3AnLUcSuFMu8C4MdOUxA8A==} + + htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -1636,6 +1686,9 @@ packages: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + leac@0.6.0: + resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} + lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -1651,6 +1704,10 @@ packages: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + lru-cache@11.5.3: resolution: {integrity: sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==} engines: {node: 20 || >=22} @@ -1798,6 +1855,9 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + parseley@0.12.1: + resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -1817,6 +1877,9 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} + peberminta@0.9.0: + resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==} + perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} @@ -1856,6 +1919,11 @@ packages: preact-render-to-string: optional: true + prettier@3.9.10: + resolution: {integrity: sha512-skGHTH1Yb9QvaTC4hWnRTzEYtCpApF0UMBPz+t/0ZPzP8ihefo/btKd+QA7pUE0fEwA1SbS/NmhyEcstdXOIdA==} + engines: {node: '>=14'} + hasBin: true + pretty-format@29.7.0: resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -1876,9 +1944,18 @@ packages: resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==} engines: {node: '>=8'} + react-dom@18.3.1: + resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} + peerDependencies: + react: ^18.3.1 + react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + react@18.3.1: + resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} + engines: {node: '>=0.10.0'} + read-pkg-up@7.0.1: resolution: {integrity: sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==} engines: {node: '>=8'} @@ -1927,9 +2004,15 @@ packages: resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} engines: {node: '>=10'} + scheduler@0.23.2: + resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} + search-insights@2.17.3: resolution: {integrity: sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==} + selderee@0.11.0: + resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==} + semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -2338,9 +2421,9 @@ snapshots: '@docsearch/css@3.8.2': {} - '@docsearch/js@3.8.2(@algolia/client-search@5.60.0)(search-insights@2.17.3)': + '@docsearch/js@3.8.2(@algolia/client-search@5.60.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3)': dependencies: - '@docsearch/react': 3.8.2(@algolia/client-search@5.60.0)(search-insights@2.17.3) + '@docsearch/react': 3.8.2(@algolia/client-search@5.60.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3) preact: 10.29.8 transitivePeerDependencies: - '@algolia/client-search' @@ -2350,13 +2433,15 @@ snapshots: - react-dom - search-insights - '@docsearch/react@3.8.2(@algolia/client-search@5.60.0)(search-insights@2.17.3)': + '@docsearch/react@3.8.2(@algolia/client-search@5.60.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3)': dependencies: '@algolia/autocomplete-core': 1.17.7(@algolia/client-search@5.60.0)(algoliasearch@5.60.0)(search-insights@2.17.3) '@algolia/autocomplete-preset-algolia': 1.17.7(@algolia/client-search@5.60.0)(algoliasearch@5.60.0) '@docsearch/css': 3.8.2 algoliasearch: 5.60.0 optionalDependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) search-insights: 2.17.3 transitivePeerDependencies: - '@algolia/client-search' @@ -2694,6 +2779,15 @@ snapshots: '@pinojs/redact@0.4.0': {} + '@react-email/render@2.1.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + entities: 4.5.0 + html-to-text: 9.0.5 + html5parser: 3.0.0 + prettier: 3.9.10 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + '@rollup/rollup-android-arm-eabi@4.64.6': optional: true @@ -2769,6 +2863,11 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.64.6': optional: true + '@selderee/plugin-htmlparser2@0.11.0': + dependencies: + domhandler: 5.0.3 + selderee: 0.11.0 + '@shikijs/core@2.5.0': dependencies: '@shikijs/engine-javascript': 2.5.0 @@ -2911,12 +3010,14 @@ snapshots: '@ungap/structured-clone@1.4.0': {} - '@vercel/analytics@2.0.1(vue@3.5.44(typescript@7.0.2))': + '@vercel/analytics@2.0.1(react@18.3.1)(vue@3.5.44(typescript@7.0.2))': optionalDependencies: + react: 18.3.1 vue: 3.5.44(typescript@7.0.2) - '@vercel/speed-insights@2.0.0(vue@3.5.44(typescript@7.0.2))': + '@vercel/speed-insights@2.0.0(react@18.3.1)(vue@3.5.44(typescript@7.0.2))': optionalDependencies: + react: 18.3.1 vue: 3.5.44(typescript@7.0.2) '@vitejs/plugin-vue@5.2.4(vite@5.4.21)(vue@3.5.44(typescript@7.0.2))': @@ -3140,6 +3241,8 @@ snapshots: decamelize@1.2.0: {} + deepmerge@4.3.1: {} + dequal@2.0.3: {} devlop@1.1.0: @@ -3152,12 +3255,32 @@ snapshots: dependencies: path-type: 4.0.0 + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + emoji-regex-xs@1.0.0: {} emoji-regex@10.6.0: {} emoji-regex@8.0.0: {} + entities@4.5.0: {} + entities@7.0.1: {} error-ex@1.3.4: @@ -3337,8 +3460,25 @@ snapshots: html-escaper@2.0.2: {} + html-to-text@9.0.5: + dependencies: + '@selderee/plugin-htmlparser2': 0.11.0 + deepmerge: 4.3.1 + dom-serializer: 2.0.0 + htmlparser2: 8.0.2 + selderee: 0.11.0 + html-void-elements@3.0.0: {} + html5parser@3.0.0: {} + + htmlparser2@8.0.2: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 4.5.0 + ignore@5.3.2: {} indent-string@4.0.0: {} @@ -3395,6 +3535,8 @@ snapshots: kind-of@6.0.3: {} + leac@0.6.0: {} + lines-and-columns@1.2.4: {} locate-path@5.0.0: @@ -3410,6 +3552,10 @@ snapshots: chalk: 4.1.2 is-unicode-supported: 0.1.0 + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + lru-cache@11.5.3: {} lru-cache@6.0.0: @@ -3594,6 +3740,11 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + parseley@0.12.1: + dependencies: + leac: 0.6.0 + peberminta: 0.9.0 + path-exists@4.0.0: {} path-key@3.1.1: {} @@ -3607,6 +3758,8 @@ snapshots: path-type@4.0.0: {} + peberminta@0.9.0: {} + perfect-debounce@1.0.0: {} picocolors@1.1.1: {} @@ -3647,6 +3800,8 @@ snapshots: preact@10.29.8: {} + prettier@3.9.10: {} + pretty-format@29.7.0: dependencies: '@jest/schemas': 29.6.3 @@ -3663,8 +3818,18 @@ snapshots: quick-lru@4.0.1: {} + react-dom@18.3.1(react@18.3.1): + dependencies: + loose-envify: 1.4.0 + react: 18.3.1 + scheduler: 0.23.2 + react-is@18.3.1: {} + react@18.3.1: + dependencies: + loose-envify: 1.4.0 + read-pkg-up@7.0.1: dependencies: find-up: 4.1.0 @@ -3744,8 +3909,16 @@ snapshots: safe-stable-stringify@2.5.0: {} + scheduler@0.23.2: + dependencies: + loose-envify: 1.4.0 + search-insights@2.17.3: {} + selderee@0.11.0: + dependencies: + parseley: 0.12.1 + semver@5.7.2: {} semver@7.8.5: {} @@ -3961,10 +4134,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - vitepress@1.6.4(@algolia/client-search@5.60.0)(postcss@8.5.30)(search-insights@2.17.3)(typescript@7.0.2): + vitepress@1.6.4(@algolia/client-search@5.60.0)(postcss@8.5.30)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3)(typescript@7.0.2): dependencies: '@docsearch/css': 3.8.2 - '@docsearch/js': 3.8.2(@algolia/client-search@5.60.0)(search-insights@2.17.3) + '@docsearch/js': 3.8.2(@algolia/client-search@5.60.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(search-insights@2.17.3) '@iconify-json/simple-icons': 1.2.100 '@shikijs/core': 2.5.0 '@shikijs/transformers': 2.5.0 diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index c07ae9b..ac93a4c 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -2,6 +2,7 @@ ".": 10, "./mime": 7.5, "./attachments": 4, + "./render": 3, "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, diff --git a/src/render/entities.js b/src/render/entities.js new file mode 100644 index 0000000..becfc2e --- /dev/null +++ b/src/render/entities.js @@ -0,0 +1,78 @@ +// Ported from unemail v0.7.0 (src/render/_html.ts): the character references +// of HTML, `decodeEntities`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: a numeric reference past U+10FFFF, to a surrogate or to +// NUL is U+FFFD, as HTML reads it, where unemail threw a RangeError; the named +// references of mail (dashes, quotes, the ellipsis, the euro…) are decoded, +// where unemail knew six; and all of them in one pass. + +const NAMED = new Map([ + ['amp', '&'], + ['lt', '<'], + ['gt', '>'], + ['quot', '"'], + ['apos', "'"], + ['nbsp', ' '], + ['ensp', ' '], + ['emsp', ' '], + ['thinsp', ' '], + ['shy', ''], + ['zwnj', ''], + ['zwj', ''], + ['copy', '©'], + ['reg', '®'], + ['trade', '™'], + ['hellip', '…'], + ['mdash', '—'], + ['ndash', '–'], + ['lsquo', '‘'], + ['rsquo', '’'], + ['sbquo', '‚'], + ['ldquo', '“'], + ['rdquo', '”'], + ['bdquo', '„'], + ['laquo', '«'], + ['raquo', '»'], + ['bull', '•'], + ['middot', '·'], + ['euro', '€'], + ['pound', '£'], + ['yen', '¥'], + ['cent', '¢'], + ['sect', '§'], + ['para', '¶'], + ['deg', '°'], + ['plusmn', '±'], + ['times', '×'], + ['divide', '÷'], + ['larr', '←'], + ['rarr', '→'], + ['uarr', '↑'], + ['darr', '↓'], +]); + +const REFERENCE = /&(#[0-9]{1,10}|#[xX][0-9a-fA-F]{1,8}|[a-zA-Z][a-zA-Z0-9]{0,31});/g; + +const codePointOf = (code) => { + if (code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '�'; + return String.fromCodePoint(code); +}; + +const decodeReference = (reference, body) => { + if (body[0] !== '#') return NAMED.get(body) ?? reference; + const hex = body[1] === 'x' || body[1] === 'X'; + return codePointOf(Number.parseInt(body.slice(hex ? 2 : 1), hex ? 16 : 10)); +}; + +/** + * Decodes the character references of a text, each once, so that `&lt;` + * is `<`; an unknown name stays as it is. + * + * @param text The text. + * @returns The decoded text. + * @internal + */ +const decodeEntities = (text) => + text.includes('&') ? text.replace(REFERENCE, decodeReference) : text; + +module.exports = { decodeEntities }; diff --git a/src/render/html-to-text.js b/src/render/html-to-text.js new file mode 100644 index 0000000..011b4ac --- /dev/null +++ b/src/render/html-to-text.js @@ -0,0 +1,296 @@ +// Ported from unemail v0.7.0 (src/render/_html.ts): the plain text of an HTML +// body, `htmlToText`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: one pass over the HTML, linear in its length, where +// unemail's lazy patterns rescanned the rest of the input for each unclosed +// `` +// into every body), and so are `head`, `title` and what is marked +// `data-skip-in-text`, such as the preview text of React Email; an `href` in +// single quotes or none is read; the whitespace of the source collapses, as a +// browser shows it, but in `pre`; and a list item gets its bullet or number. + +const { decodeEntities } = require('./entities.js'); + +// Their content is not text. +const SKIPPED = new Set(['script', 'style', 'head', 'title', 'template']); +// A paragraph: a blank line before and after. +const PARAGRAPHS = new Set([ + 'p', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'blockquote', + 'pre', + 'table', + 'ul', + 'ol', + 'dl', + 'hr', + 'figure', +]); +// A line of its own. +const LINES = new Set([ + 'div', + 'section', + 'article', + 'header', + 'footer', + 'nav', + 'main', + 'aside', + 'address', + 'center', + 'tr', + 'td', + 'th', + 'thead', + 'tbody', + 'tfoot', + 'dt', + 'dd', + 'figcaption', + 'body', +]); +const VOID = new Set([ + 'area', + 'base', + 'br', + 'col', + 'embed', + 'hr', + 'img', + 'input', + 'link', + 'meta', + 'source', + 'track', + 'wbr', +]); +const WHITESPACE = /[ \t\n\r\f]+/g; +const ATTRIBUTE = /([^\s"'=<>/]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g; + +const isNameCharacter = (code) => + (code >= 97 && code <= 122) || + (code >= 65 && code <= 90) || + (code >= 48 && code <= 57) || + code === 45; + +// The attributes of a tag, by lowercased name. +const attributesOf = (text) => { + const attributes = new Map(); + for (const match of text.matchAll(ATTRIBUTE)) { + const value = match[2] ?? match[3] ?? match[4] ?? ''; + attributes.set(match[1].toLowerCase(), decodeEntities(value)); + } + return attributes; +}; + +// The end of a tag: the `>` outside the quotes of its attributes. +const tagEndOf = (html, from) => { + for (let i = from; i < html.length; i++) { + const c = html[i]; + if (c === '>') return i; + if (c === '"' || c === "'") { + const close = html.indexOf(c, i + 1); + if (close < 0) return -1; + i = close; + } + } + return -1; +}; + +const isUsableHref = (href) => { + const lower = href.trim().toLowerCase(); + return lower !== '' && !lower.startsWith('#') && !lower.startsWith('javascript:'); +}; + +/** + * The plain text an HTML body reads as, for the text alternative of a + * message. + * + * @internal + */ +class TextWriter { + out = ''; + #breaks = 0; + #prefix = ''; + #pre = 0; + #lists = []; + #links = []; + + // A break before the next text: 1 for a line, 2 for a paragraph. + breakLine(count) { + if (this.#breaks < count) this.#breaks = count; + } + + lineBreak() { + this.#breaks = Math.min(this.#breaks + 1, 2); + if (this.out === '') this.#breaks = 0; + } + + text(raw) { + let text = this.#pre > 0 ? raw.replaceAll('\r\n', '\n') : raw.replace(WHITESPACE, ' '); + text = decodeEntities(text); + if (this.#pre === 0 && (this.out === '' || this.#breaks > 0 || this.out.endsWith('\n'))) { + text = text.trimStart(); + } + if (text === '') return; + if (this.#breaks > 0 && this.out !== '') { + this.#trimEnd(); + this.out += this.#breaks === 2 ? '\n\n' : '\n'; + } + this.#breaks = 0; + this.out += this.#prefix + text; + this.#prefix = ''; + } + + open(name, attributes) { + if (name === 'pre') this.#pre++; + if (name === 'ul' || name === 'ol') this.#lists.push({ ordered: name === 'ol', count: 0 }); + if (name === 'li') { + this.breakLine(1); + const list = this.#lists.at(-1); + if (list?.ordered) this.#prefix = `${++list.count}. `; + else this.#prefix = '- '; + } + if (name === 'a') { + const href = attributes?.get('href'); + this.#links.push({ href, start: this.out.length }); + } + } + + close(name) { + if (name === 'pre' && this.#pre > 0) this.#pre--; + if ((name === 'ul' || name === 'ol') && this.#lists.length > 0) this.#lists.pop(); + if (name === 'a') this.#closeLink(); + } + + // Keeps where a link goes: its address after its label, or in place of an + // empty one, once. + #closeLink() { + const link = this.#links.pop(); + if (link === undefined || link.href === undefined || !isUsableHref(link.href)) return; + const href = link.href.trim(); + const address = href.toLowerCase().startsWith('mailto:') ? href.slice(7) : href; + // A label much longer than the address differs from it; one this long + // is not read, so that nested links stay linear. + if (this.out.length - link.start > href.length + 8) { + this.out += ` (${href})`; + return; + } + const label = this.out.slice(link.start).trim(); + if (label === '') this.text(address); + else if (label !== href && label !== address) this.out += ` (${href})`; + } + + #trimEnd() { + let end = this.out.length; + while (end > 0 && (this.out[end - 1] === ' ' || this.out[end - 1] === '\t')) end--; + if (end < this.out.length) this.out = this.out.slice(0, end); + } + + finish() { + this.#trimEnd(); + return this.out; + } +} + +/** + * The plain text an HTML body reads as: the text of its elements, a line or a + * blank line for its blocks, list items with a bullet or a number, and the + * address of each link after its label. Scripts, styles, the head, comments + * and what is marked `data-skip-in-text` are left out. It handles what a + * mail body holds, in one pass; set `text` yourself when fidelity matters. + * + * @param html The HTML. + * @returns The text. + */ +const htmlToText = (html) => { + if (typeof html !== 'string') throw new TypeError('htmlToText takes a string.'); + const lower = html.toLowerCase(); + const writer = new TextWriter(); + // The element skipped as `data-skip-in-text`, and its depth. + let skip; + let i = 0; + while (i < html.length) { + const lt = html.indexOf('<', i); + const textEnd = lt < 0 ? html.length : lt; + if (textEnd > i && skip === undefined) writer.text(html.slice(i, textEnd)); + if (lt < 0) break; + i = readTag(html, lower, lt, writer, (tag) => { + if (skip !== undefined) { + if (tag.name === skip.name && tag.closing) skip.depth--; + else if (tag.name === skip.name && !tag.selfClosing) skip.depth++; + if (skip.depth === 0) skip = undefined; + return; + } + if (!tag.closing && !tag.selfClosing && tag.skipInText) { + skip = { name: tag.name, depth: 1 }; + return; + } + applyTag(writer, tag); + }); + } + return writer.finish(); +}; + +const applyTag = (writer, tag) => { + if (tag.name === 'br') { + writer.lineBreak(); + return; + } + if (PARAGRAPHS.has(tag.name)) writer.breakLine(2); + else if (LINES.has(tag.name)) writer.breakLine(1); + if (tag.closing) writer.close(tag.name); + else writer.open(tag.name, tag.attributes); +}; + +// Reads the markup at `<`, hands an element tag to `onTag`, and returns where +// the text goes on. +const readTag = (html, lower, at, writer, onTag) => { + if (html.startsWith('', at + 4); + return end < 0 ? html.length : end + 3; + } + const next = html[at + 1]; + if (next === '!' || next === '?') { + const end = html.indexOf('>', at); + return end < 0 ? html.length : end + 1; + } + const closing = next === '/'; + const nameStart = at + (closing ? 2 : 1); + let nameEnd = nameStart; + while (nameEnd < html.length && isNameCharacter(html.charCodeAt(nameEnd))) nameEnd++; + if (nameEnd === nameStart) { + // A `<` that opens no tag is text. + writer.text('<'); + return at + 1; + } + const end = tagEndOf(html, nameEnd); + if (end < 0) return html.length; + const name = lower.slice(nameStart, nameEnd); + if (!closing && SKIPPED.has(name)) { + const close = lower.indexOf(`', close); + return closeEnd < 0 ? html.length : closeEnd + 1; + } + const inner = html.slice(nameEnd, end); + const selfClosing = VOID.has(name) || inner.endsWith('/'); + const needsAttributes = name === 'a' || inner.includes('data-skip-in-text'); + const attributes = !closing && needsAttributes ? attributesOf(inner) : undefined; + onTag({ + name, + closing, + selfClosing, + attributes, + skipInText: attributes?.get('data-skip-in-text') === 'true', + }); + return end + 1; +}; + +module.exports = { htmlToText }; diff --git a/src/render/index.d.ts b/src/render/index.d.ts new file mode 100644 index 0000000..fd94320 --- /dev/null +++ b/src/render/index.d.ts @@ -0,0 +1,106 @@ +// Types ported from unemail v0.7.0 (src/render) +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind + +import type { MessageContent } from '../../index.js'; + +/** + * What a renderer gives: the HTML of a message, its text, or both. + */ +export interface RenderOutput { + readonly html?: string; + readonly text?: string; +} + +/** + * Turns the input of a template into the content of a message: React Email + * components, MJML, a template language of the application's own. + */ +export interface Renderer { + render(input: TInput): RenderOutput | Promise; +} + +/** + * The options of {@link renderContent}. + */ +export interface RenderContentOptions { + /** + * Derives the text from the HTML with {@link htmlToText} when the renderer + * gives none. + * @default true + */ + readonly autoText?: boolean; +} + +/** + * Renders the content of a message for `createMessage`: `{ html, text }`, the + * text derived from the HTML when the renderer gives none. + * + * @example + * ```js + * const message = createMessage({ + * from, to, subject: 'Welcome', + * content: await renderContent(reactEmail, Welcome({ name: 'Ada' })), + * }); + * ``` + * @throws {TypeError} If the renderer is not one, or gives no string. + */ +export declare function renderContent( + renderer: Renderer, + input: TInput, + options?: RenderContentOptions, +): Promise; + +/** + * The options `render` of `@react-email/render` takes. + */ +export interface ReactEmailRenderOptions { + readonly plainText?: boolean; + readonly pretty?: boolean; + readonly [option: string]: unknown; +} + +/** + * The options of {@link createReactEmailRenderer}. + */ +export interface ReactEmailRendererOptions { + /** `require('@react-email/render').render`, or a function like it. */ + readonly render: ( + element: TElement, + options?: ReactEmailRenderOptions, + ) => string | Promise; + /** + * Takes the text from React Email's plain-text pass rather than deriving it + * from the HTML. + * @default true + */ + readonly plainText?: boolean; + /** Passed to `render`, such as `{ pretty: true }`. */ + readonly renderOptions?: ReactEmailRenderOptions; +} + +/** + * Makes a renderer of React Email components with the `render` the + * application passes in, so the package depends on neither React nor React + * Email. + * + * @example + * ```js + * const { render } = require('@react-email/render'); + * const reactEmail = createReactEmailRenderer({ render }); + * ``` + * @throws {TypeError} If `render` is not a function. + */ +export declare function createReactEmailRenderer( + options: ReactEmailRendererOptions, +): Renderer; + +/** + * The plain text an HTML body reads as: the text of its elements, a line or a + * blank line for its blocks, list items with a bullet or a number, and the + * address of each link after its label. Scripts, styles, the head, comments + * and what is marked `data-skip-in-text` are left out. It handles what a + * mail body holds, in one pass, in time linear in its length. + * + * @throws {TypeError} If `html` is not a string. + */ +export declare function htmlToText(html: string): string; diff --git a/src/render/index.js b/src/render/index.js new file mode 100644 index 0000000..efa2a19 --- /dev/null +++ b/src/render/index.js @@ -0,0 +1,7 @@ +// Not ported: the entry of ./render, the content of a message from a template. + +const { htmlToText } = require('./html-to-text.js'); +const { createReactEmailRenderer } = require('./react-email.js'); +const { renderContent } = require('./renderer.js'); + +module.exports = { createReactEmailRenderer, htmlToText, renderContent }; diff --git a/src/render/react-email.js b/src/render/react-email.js new file mode 100644 index 0000000..829986f --- /dev/null +++ b/src/render/react-email.js @@ -0,0 +1,42 @@ +// Ported from unemail v0.7.0 (src/render/react.ts): a renderer of React Email +// components, `reactRenderer`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Also ported from sently v1.2.2 (src/plugins/react.ts): the text of the +// element rendered by React Email's own plain-text pass. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: `render` is passed in, never imported, so the package +// has no dependency on React Email; and a plain-text pass that fails fails the +// render, where unemail let it fall back to the derived text unseen. + +/** + * Makes a renderer of React Email components with the `render` of + * `@react-email/render` the application passes in. + * + * @param options `render`; `plainText` (true), the text from React Email's + * plain-text pass rather than derived from the HTML; and `renderOptions`, + * passed to `render`, such as `{ pretty: true }`. + * @returns The renderer, whose input is a React element. + * @throws {TypeError} If `render` is not a function. + */ +const createReactEmailRenderer = (options = {}) => { + const { render, renderOptions } = options; + if (typeof render !== 'function') { + throw new TypeError( + 'createReactEmailRenderer needs render, the render function of @react-email/render.', + ); + } + const plainText = options.plainText ?? true; + if (typeof plainText !== 'boolean') throw new TypeError('plainText must be a boolean.'); + return { + async render(element) { + if (element === undefined || element === null) { + throw new TypeError('The React Email renderer needs an element.'); + } + const html = await render(element, renderOptions); + if (!plainText) return { html }; + return { html, text: await render(element, { ...renderOptions, plainText: true }) }; + }, + }; +}; + +module.exports = { createReactEmailRenderer }; diff --git a/src/render/renderer.js b/src/render/renderer.js new file mode 100644 index 0000000..489e632 --- /dev/null +++ b/src/render/renderer.js @@ -0,0 +1,45 @@ +// Ported from unemail v0.7.0 (src/render/index.ts): a renderer turns the input +// of a template into the content of a message, its text derived from its HTML +// when it gives none, `withRender`. +// https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind +// Changed in the port: the content is rendered before `createMessage`, which +// takes it, rather than by a middleware of the send; and what the renderer +// gives is checked. + +const { htmlToText } = require('./html-to-text.js'); + +/** + * Renders the content of a message: `{ html, text }` for `createMessage`, the + * text derived from the HTML with `htmlToText` when the renderer gives none. + * + * @param renderer An object with `render(input)`, sync or async, which gives + * `{ html?, text? }`. + * @param input What the renderer takes, such as a React element. + * @param options `autoText` (true): derive the text when there is none. + * @returns The content. + * @throws {TypeError} If the renderer is not one, or gives no string. + */ +const renderContent = async (renderer, input, options = {}) => { + if (typeof renderer?.render !== 'function') { + throw new TypeError('renderer must have a render method.'); + } + const autoText = options.autoText ?? true; + if (typeof autoText !== 'boolean') throw new TypeError('autoText must be a boolean.'); + const output = await renderer.render(input); + const html = output?.html; + const text = output?.text; + if ( + (html !== undefined && typeof html !== 'string') || + (text !== undefined && typeof text !== 'string') + ) { + throw new TypeError('A renderer gives html and text as strings.'); + } + if (html === undefined) { + if (text === undefined) throw new TypeError('The renderer gave neither html nor text.'); + return { text }; + } + const plain = text ?? (autoText ? htmlToText(html) : undefined); + return plain === undefined ? { html } : { html, text: plain }; +}; + +module.exports = { renderContent }; diff --git a/tests/types/render.test-d.ts b/tests/types/render.test-d.ts new file mode 100644 index 0000000..18beccc --- /dev/null +++ b/tests/types/render.test-d.ts @@ -0,0 +1,27 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import { createMessage, type MessageContent } from '../../index.js'; +import { + createReactEmailRenderer, + htmlToText, + renderContent, + type Renderer, +} from '../../src/render/index.js'; + +interface Element { + readonly type: string; +} +declare const render: (element: Element, options?: { plainText?: boolean }) => Promise; +const reactEmail = createReactEmailRenderer({ + render, + plainText: true, + renderOptions: { pretty: false }, +}); +expectAssignable>(reactEmail); +const content = await renderContent(reactEmail, { type: 'div' }); +expectType(content); +createMessage({ from: 'a@example.com', to: 'b@example.com', subject: 'x', content }); +const upper: Renderer = { render: (source) => ({ html: source.toUpperCase() }) }; +expectType>(renderContent(upper, 'x', { autoText: false })); +expectError(renderContent(upper, 42)); +expectType(htmlToText('

x

')); +expectError(createReactEmailRenderer({})); diff --git a/tests/unit/render.test.js b/tests/unit/render.test.js new file mode 100644 index 0000000..23e97cf --- /dev/null +++ b/tests/unit/render.test.js @@ -0,0 +1,199 @@ +// Not ported: tests of ./render, ported from unemail's render tests, with the +// React Email adapter run on @react-email/render, a devDependency. +const { test } = require('node:test'); +const assert = require('node:assert'); +const React = require('react'); +const { render } = require('@react-email/render'); +const { createMessage } = require('../../index.js'); +const { + createReactEmailRenderer, + htmlToText, + renderContent, +} = require('../../src/render/index.js'); + +const h = React.createElement; + +test('Render: htmlToText keeps the cases of unemail', () => { + assert.strictEqual(htmlToText('

one

two

'), 'one\n\ntwo'); + assert.strictEqual(htmlToText('a
b'), 'a\nb'); + assert.strictEqual(htmlToText('Acme'), 'Acme (https://acme.com)'); + assert.strictEqual( + htmlToText('https://acme.com'), + 'https://acme.com', + ); + assert.strictEqual( + htmlToText('

safe

'), + 'safe', + ); + assert.strictEqual(htmlToText('

a & b

'), 'a & b'); + assert.strictEqual(htmlToText('

&lt;

'), '<'); + assert.strictEqual(htmlToText('

€ €

'), '€ €'); + assert.strictEqual(htmlToText('
x
'), 'x'); +}); + +test('Render: comments, the doctype and the head are not text', () => { + // Changed in the port: unemail left `` and `` in the text. + assert.strictEqual( + htmlToText( + '' + + 'Subject' + + '

Hello

', + ), + 'Hello', + ); + assert.strictEqual(htmlToText('

x

+ +Webhook test certificate +======================== + +`sns-cert.pem` and `sns-key.pem` stand for the certificate SNS signs its +messages with, so that the tests of `./webhooks` sign and verify SNS messages +without AWS. They were made with + + openssl req -x509 -newkey rsa:2048 -nodes -days 36500 \ + -subj "/CN=sns.amazonaws.com" -keyout sns-key.pem -out sns-cert.pem + +The private key is intentionally committed and must never be used outside the +tests. diff --git a/tests/fixtures/webhooks/sns-cert.pem b/tests/fixtures/webhooks/sns-cert.pem new file mode 100644 index 0000000..b0baed5 --- /dev/null +++ b/tests/fixtures/webhooks/sns-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGzCCAgOgAwIBAgIUFeF1IswUt0mb9/UJDdcp9tG5JJIwDQYJKoZIhvcNAQEL +BQAwHDEaMBgGA1UEAwwRc25zLmFtYXpvbmF3cy5jb20wIBcNMjYxMDE3MjAwNzU0 +WhgPMjEyNjA5MjMyMDA3NTRaMBwxGjAYBgNVBAMMEXNucy5hbWF6b25hd3MuY29t +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAohCKt9ewykT5rVSH6vkF +VNVE867NqqTbN00KT+p1VEJ7ZViuDgEdrg/BgM3NzGXymQiB8rrw/8Cf5CpAMHCr +shMeQxkmBTIOu8vd5fniwjxHlsDkojlOs42R2HhmNTGEa/grWyKfhYAOSRZMadq7 +pkLVa7WbsIHcdTdSBcmT5gRiOPnLXBj5hgZ1xCl1vQii/X+uLhj6ItKJHAkfuoI+ +Ify+4Anj7xyVqOchHP5lNqbAHKg8aOVOEnYLbghkB+nBf/lJIaTjQUuilgPk+LzA +eziPMR+XXLBNy8v4n8kr5IAD/ht/hOb2agSP7r4unF2q6Vxs7/z+8cMQt4TparZy +pQIDAQABo1MwUTAdBgNVHQ4EFgQUGEZLQKHE+iFm0z7AMb9ewhjwRZMwHwYDVR0j +BBgwFoAUGEZLQKHE+iFm0z7AMb9ewhjwRZMwDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAQEAW0GDwCGgzINSR2BXdZnx8iQlhzfL71Tlpy1hy/4BTdd7 +esKCs2zlm1MMnSu/jxlbKdA88z2YFqb9ISYXeazo6oRAK3/wRYQ4E56PAIwOq48R +z0OBXidE5ASPoQgwp+W6s92dJpcP3l7K9SStM1xdtJd111o5M0Cn4IX2oWxKla7G +xIAVOO3pcUW/NLS4wMHhFeiYjFewUUUH4WIC0MLVU6fmVHmnEOGzVx3eWVQPwGYL +0ZpyHD1kVD9CYH+23KnhsF4RTyB/ZE/52Xq6dY/4TNsfwCCljmtPAgGOVJr+KGG4 +fsXTydBWrl06FEEBbDx9Jvqpv5ciEDCqljLpc0A/eA== +-----END CERTIFICATE----- diff --git a/tests/fixtures/webhooks/sns-key.pem b/tests/fixtures/webhooks/sns-key.pem new file mode 100644 index 0000000..4a13030 --- /dev/null +++ b/tests/fixtures/webhooks/sns-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiEIq317DKRPmt +VIfq+QVU1UTzrs2qpNs3TQpP6nVUQntlWK4OAR2uD8GAzc3MZfKZCIHyuvD/wJ/k +KkAwcKuyEx5DGSYFMg67y93l+eLCPEeWwOSiOU6zjZHYeGY1MYRr+CtbIp+FgA5J +Fkxp2rumQtVrtZuwgdx1N1IFyZPmBGI4+ctcGPmGBnXEKXW9CKL9f64uGPoi0okc +CR+6gj4h/L7gCePvHJWo5yEc/mU2psAcqDxo5U4SdgtuCGQH6cF/+UkhpONBS6KW +A+T4vMB7OI8xH5dcsE3Ly/ifySvkgAP+G3+E5vZqBI/uvi6cXarpXGzv/P7xwxC3 +hOlqtnKlAgMBAAECggEAGNK2z0+dbRs2hlrcZ1xqzs0h1BSyEiamddMk4WEUmpTa +SLIrasq3TJTDkKKKmKRZjct0oq9elZXgB7cvAMcT4raJ8UDAsZVda1y1FBzRI5Yk +aE4LAba3EXUoNy4YiEzu2bN36ewk30e3fcYJa0H3xxC/7CZeKUwiw94hf/BvE7vZ +YCIkKbsiPw4SsD2xHx2osaLX009WKFXKYDYr0VyHVG2/GYL+sNlvTfUfIB7yxRUm +M33Qt1ojtx8H657vcJ14O5xgUtH24dDT4JJT6DFZ4H9+PyFqL2w9itKBIvKewm7J ++v8h19AQWNHppvZKqdFjAPmbUECI0JLEXDJJNDkKiwKBgQDUEtOIN+zZPdX3yQVP +gNu0a7s+x5cdJMkA7dcsoSeGCL7PJKlcYx1nDFhY9lSzN3d+abFGo4Q7FKsHcX/W +weGdZTW+CaVhZXHP5rqMIuImFBnp0s5hsSN331QsPgQzVjMRNmB+mx/om7w+HFgL +PG9YtohcNlDQokpGjgKlhVLoqwKBgQDDof+cE5v8UYPnthiGDbCTx+iWDv2h/M9n +78AqGmvAAoAHtwFU1I2NEDxHUoeikSFBxrhnu+D+a8v3c8iGE4atJ2BaNn8FCZA0 +YjzcGNjau54bNozaMjo0IhlP4iSub22peEHOZefYpgC/QcqzU3NK0sP5a2EIBiDa +nL9WkFOx7wKBgQCmABKKJtSDe86XhUdd4wyW3lkUHCEiE+AulP2AFqkcVo4dEKJt +CbxDJDunbzH/kqQ7VDoKhAYGRPhW/DXIAb07ebSmi2Bjq4d56C4PcWK50bQy/erE +N2b+9tYpR2X8VTXl/D1t/LjuV7DyoTcdnHG/7CkynOWHWfFv5ssVixXJvwKBgHKj +h6nqQ8oD6ovpxU0o0ISI7N6iQjaectuxPqZtfR6zWFss/na7Z4d2MByLXDgDW72z +uXPwQNRWTtve2W7YGl91Z5z1ulE1XW+U8Zz4Ynsq3meL+9YRANJz+Ok618vwOA7L +DaTpSodGH8Hzx9h12mXKRFTwe5e8H5x7phQP1zu5AoGAKVf9WtxyQ+BznCzFySTt +6N0qI6WnBrXf3qk3AmEDA3/GoDYOZHSA4P/5YyMzl3FqUaL20R+yM/ZKtQsp4jua +bq8atn4YpPFDNPIoFzf7ZnmPmaCiSTfJZ03baN8nJlBElIaRfGKUBLXEMhZOyL0D +8s477O7BK/Eu7q+c8nDfk4Q= +-----END PRIVATE KEY----- diff --git a/tests/helpers/webhooks/sns.js b/tests/helpers/webhooks/sns.js new file mode 100644 index 0000000..301214e --- /dev/null +++ b/tests/helpers/webhooks/sns.js @@ -0,0 +1,62 @@ +// Not ported: SNS messages signed with the test certificate, as AWS signs them +// (https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html). + +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const { sign } = require('node:crypto'); + +const FIXTURES = path.join(__dirname, '..', '..', 'fixtures', 'webhooks'); +const certificate = readFileSync(path.join(FIXTURES, 'sns-cert.pem'), 'utf8'); +const privateKey = readFileSync(path.join(FIXTURES, 'sns-key.pem'), 'utf8'); + +const TOPIC = 'arn:aws:sns:eu-west-1:123456789012:ses-events'; +const CERT_URL = 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-abc.pem'; + +// The name and value of each field, one per line, in the order AWS lists. +const NOTIFICATION = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']; +const SUBSCRIPTION = [ + 'Message', + 'MessageId', + 'SubscribeURL', + 'Timestamp', + 'Token', + 'TopicArn', + 'Type', +]; + +const signSns = (message, version = '2') => { + const fields = message.Type === 'Notification' ? NOTIFICATION : SUBSCRIPTION; + let text = ''; + for (const field of fields) { + if (message[field] !== undefined) text += `${field}\n${message[field]}\n`; + } + const algorithm = version === '1' ? 'RSA-SHA1' : 'RSA-SHA256'; + return { + ...message, + SignatureVersion: version, + SigningCertURL: message.SigningCertURL ?? CERT_URL, + Signature: sign(algorithm, Buffer.from(text, 'utf8'), privateKey).toString('base64'), + }; +}; + +const notification = (record, fields = {}) => + signSns({ + Type: 'Notification', + MessageId: fields.MessageId ?? 'sns-1', + TopicArn: TOPIC, + Message: typeof record === 'string' ? record : JSON.stringify(record), + Timestamp: fields.Timestamp ?? '2027-01-01T12:00:00.000Z', + ...fields, + }); + +// A fetch that serves the test certificate, and counts its requests. +const certificateFetch = () => { + const requests = []; + const fetch = async (url, init) => { + requests.push({ url: String(url), init }); + return new Response(certificate, { status: 200 }); + }; + return { fetch, requests }; +}; + +module.exports = { CERT_URL, TOPIC, certificate, certificateFetch, notification, signSns }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts new file mode 100644 index 0000000..f3a6e7e --- /dev/null +++ b/tests/types/webhooks.test-d.ts @@ -0,0 +1,29 @@ +import { expectAssignable, expectError, expectType } from 'tsd'; +import type { DeliveryEvent } from '../../index.js'; +import { + WebhookVerificationError, + createMemoryReplayStore, + createSesWebhookVerifier, + type ReplayStore, + type SesWebhookResult, + type WebhookVerificationReason, +} from '../../src/webhooks/index.js'; + +const store: ReplayStore = createMemoryReplayStore({ maxEntries: 100, now: Date.now }); +const ses = createSesWebhookVerifier({ + topicArns: ['arn:aws:sns:eu-west-1:1:ses'], + fetch: globalThis.fetch, + toleranceMs: 60_000, + replayStore: store, + maxBodyBytes: 1024, +}); +createSesWebhookVerifier({ topicArns: ['arn'], replayStore: null }); +const result = await ses.verify({ body: '{}', headers: new Headers() }); +expectType(result); +expectAssignable[]>(result.events); +expectType(result.confirmation?.subscribeUrl); +await ses.verify({ body: new Uint8Array(), headers: { 'x-amz-sns-message-type': 'Notification' } }); +expectError(ses.verify({ body: {} })); +expectError(createSesWebhookVerifier({})); +const error = new WebhookVerificationError('replay', 'seen'); +expectType(error.reason); diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js new file mode 100644 index 0000000..7d3a615 --- /dev/null +++ b/tests/unit/webhooks-request.test.js @@ -0,0 +1,54 @@ +// Not ported: tests of what the webhook verifiers share. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { + createMemoryReplayStore, + dateOf, + equalBytes, + headerOf, +} = require('../../src/webhooks/request.js'); + +test('Webhooks: a header field is read from Headers or from Node headers, in any case', () => { + const headers = new Headers({ 'Svix-Id': 'msg_1' }); + assert.strictEqual(headerOf({ headers }, 'svix-id'), 'msg_1'); + assert.strictEqual(headerOf({ headers }, 'svix-timestamp'), undefined); + assert.strictEqual(headerOf({ headers: { 'svix-id': 'msg_2' } }, 'Svix-Id'), 'msg_2'); + assert.strictEqual(headerOf({ headers: { 'X-Twilio': ['a', 'b'] } }, 'x-twilio'), 'a'); + assert.strictEqual(headerOf({ headers: { other: 'x' } }, 'svix-id'), undefined); + assert.strictEqual(headerOf({}, 'svix-id'), undefined); + assert.strictEqual(headerOf(undefined, 'svix-id'), undefined); +}); + +test('Webhooks: secrets compare in constant time, a length apart is unequal', () => { + assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abc')), true); + assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abd')), false); + assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abcd')), false); +}); + +test('Webhooks: dates come from seconds, milliseconds or ISO text', () => { + assert.strictEqual(dateOf(1_700_000_000).getTime(), 1_700_000_000_000); + assert.strictEqual(dateOf(1_700_000_000_123).getTime(), 1_700_000_000_123); + assert.strictEqual(dateOf('1700000000.5').getTime(), 1_700_000_000_500); + assert.strictEqual(dateOf('2026-10-17T12:00:00Z').toISOString(), '2026-10-17T12:00:00.000Z'); + assert.strictEqual(dateOf('yesterday'), undefined); + assert.strictEqual(dateOf(''), undefined); + assert.strictEqual(dateOf(undefined), undefined); + assert.strictEqual(dateOf(Number.NaN), undefined); +}); + +test('Webhooks: the memory replay store forgets an id after its time, the oldest first', () => { + let time = 0; + const store = createMemoryReplayStore({ maxEntries: 2, now: () => time }); + assert.strictEqual(store.add('a', 100), true); + assert.strictEqual(store.add('a', 100), false); + time = 100; + assert.strictEqual(store.add('a', 100), true); + assert.strictEqual(store.add('b', 1000), true); + assert.strictEqual(store.add('c', 1000), true); + // Three ids for two places: a, the oldest, goes. + assert.strictEqual(store.add('a', 100), true); + assert.strictEqual(store.add('c', 1000), false); + assert.throws(() => createMemoryReplayStore({ maxEntries: 0 }), /maxEntries/); + assert.throws(() => createMemoryReplayStore({ now: 1 }), /now must be a function/); + assert.strictEqual(createMemoryReplayStore().add('x', 1), true); +}); diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js new file mode 100644 index 0000000..5016566 --- /dev/null +++ b/tests/unit/webhooks-ses.test.js @@ -0,0 +1,353 @@ +// Not ported: tests of the SES verifier of ./webhooks, ported from sently's SES +// webhook, with SNS messages signed by a test certificate. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { + createSesWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); +const { + CERT_URL, + TOPIC, + certificateFetch, + notification, + signSns, +} = require('../helpers/webhooks/sns.js'); + +const NOW = Date.parse('2027-01-01T12:00:30Z'); +const verifier = (options = {}) => { + const served = certificateFetch(); + const made = createSesWebhookVerifier({ + topicArns: [TOPIC], + fetch: served.fetch, + now: () => NOW, + ...options, + }); + return { ...made, requests: served.requests }; +}; +const request = (message) => ({ body: JSON.stringify(message), headers: {} }); +const mail = (fields = {}) => ({ + timestamp: '2026-10-17T11:59:00.000Z', + messageId: 'ses-message-1', + destination: ['bob@example.com', 'carol@example.com'], + tags: { + category: ['orders', 'eu'], + orderId: ['42'], + priority: ['high'], + 'ses:configuration-set': ['events'], + }, + ...fields, +}); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; + +test('Webhooks SES: a bounce is an event per recipient, hard or soft', async () => { + const { verify, requests } = verifier(); + const record = { + eventType: 'Bounce', + mail: mail(), + bounce: { + bounceType: 'Permanent', + bounceSubType: 'General', + timestamp: '2026-10-17T12:00:00.000Z', + bouncedRecipients: [ + { + emailAddress: 'bob@example.com', + status: '5.1.1', + diagnosticCode: 'smtp; 550 5.1.1 unknown', + }, + { emailAddress: 'carol@example.com' }, + ], + }, + }; + const { events } = await verify(request(notification(record))); + // Changed in the port: sently made one event, of the first recipient. + assert.strictEqual(events.length, 2); + assert.deepStrictEqual( + { ...events[0], metadata: { ...events[0].metadata } }, + { + provider: 'ses', + type: 'bounced', + id: 'sns-1:0', + messageId: 'ses-message-1', + recipient: 'bob@example.com', + occurredAt: new Date('2026-10-17T12:00:00.000Z'), + bounce: { kind: 'hard', code: '5.1.1', diagnostic: 'smtp; 550 5.1.1 unknown' }, + reason: 'General', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + raw: record, + }, + ); + assert.deepStrictEqual(events[1].bounce, { kind: 'hard' }); + assert.strictEqual(events[1].id, 'sns-1:1'); + assert.deepStrictEqual(requests[0].url, CERT_URL); + assert.strictEqual(requests[0].init.redirect, 'error'); + const soft = await verify( + request( + notification( + { ...record, bounce: { ...record.bounce, bounceType: 'Transient' } }, + { MessageId: 'sns-2' }, + ), + ), + ); + assert.strictEqual(soft.events[0].bounce.kind, 'soft'); + const unclear = await verify( + request( + notification( + { ...record, bounce: { ...record.bounce, bounceType: 'Undetermined' } }, + { MessageId: 'sns-3' }, + ), + ), + ); + assert.strictEqual(unclear.events[0].bounce.kind, 'undetermined'); + // The certificate was read once. + assert.strictEqual(requests.length, 1); +}); + +test('Webhooks SES: every type of SES event', async () => { + const { verify } = verifier(); + const cases = [ + [{ eventType: 'Send', mail: mail() }, ['accepted', 'accepted']], + [ + { + notificationType: 'Delivery', + mail: mail(), + delivery: { recipients: ['bob@example.com'], timestamp: '2026-10-17T12:00:01Z' }, + }, + ['delivered'], + ], + [ + { + eventType: 'Complaint', + mail: mail(), + complaint: { + complaintFeedbackType: 'abuse', + complainedRecipients: [{ emailAddress: 'bob@example.com' }], + }, + }, + ['complained'], + ], + [ + { + eventType: 'DeliveryDelay', + mail: mail(), + deliveryDelay: { + delayType: 'MailboxFull', + delayedRecipients: [{ emailAddress: 'bob@example.com' }], + }, + }, + ['deferred'], + ], + // Changed in the port: sently counted these as bounces. + [ + { eventType: 'Reject', mail: mail(), reject: { reason: 'Bad content' } }, + ['rejected', 'rejected'], + ], + [ + { + eventType: 'Rendering Failure', + mail: mail(), + failure: { errorMessage: 'Missing variable' }, + }, + ['rejected', 'rejected'], + ], + [ + { + eventType: 'Open', + mail: mail({ destination: ['bob@example.com'] }), + open: { timestamp: '2026-10-17T12:00:02Z' }, + }, + ['opened'], + ], + [{ eventType: 'Click', mail: mail(), click: { link: 'https://example.com/a' } }, ['clicked']], + [ + { + eventType: 'Subscription', + mail: mail(), + subscription: { newTopicPreferences: { unsubscribeAll: true } }, + }, + ['unsubscribed'], + ], + [{ eventType: 'Subscription', mail: mail(), subscription: {} }, ['unknown']], + [{ eventType: 'Something new', mail: mail({ tags: undefined }) }, ['unknown']], + [{ eventType: 'Bounce', mail: mail({ messageId: undefined }) }, ['bounced']], + ]; + for (let i = 0; i < cases.length; i++) { + const [record, types] = cases[i]; + const { events } = await verify(request(notification(record, { MessageId: `sns-${i}` }))); + assert.deepStrictEqual( + events.map((event) => event.type), + types, + record.eventType ?? record.notificationType, + ); + } + const click = await verify(request(notification(cases[7][0], { MessageId: 'click' }))); + assert.strictEqual(click.events[0].url, 'https://example.com/a'); + // A click names no recipient when the message had two. + assert.strictEqual(click.events[0].recipient, undefined); + const reject = await verify(request(notification(cases[4][0], { MessageId: 'reject' }))); + assert.strictEqual(reject.events[1].reason, 'Bad content'); + const delay = await verify(request(notification(cases[3][0], { MessageId: 'delay' }))); + assert.strictEqual(delay.events[0].reason, 'MailboxFull'); + const plain = await verify( + request(notification('not json', { MessageId: 'text', Subject: 'Test' })), + ); + assert.strictEqual(plain.events[0].type, 'unknown'); +}); + +test('Webhooks SES: a subscription is verified and handed back, not confirmed', async () => { + const { verify, requests } = verifier(); + const message = signSns({ + Type: 'SubscriptionConfirmation', + MessageId: 'sub-1', + Token: 'token-1', + TopicArn: TOPIC, + Message: 'You have chosen to subscribe', + SubscribeURL: 'https://sns.eu-west-1.amazonaws.com/?Action=ConfirmSubscription&Token=token-1', + Timestamp: '2027-01-01T12:00:00.000Z', + }); + const result = await verify(request(message)); + assert.deepStrictEqual(result, { + events: [], + confirmation: { + type: 'SubscriptionConfirmation', + topicArn: TOPIC, + subscribeUrl: message.SubscribeURL, + token: 'token-1', + message: 'You have chosen to subscribe', + }, + }); + // Only the certificate was fetched: the subscription is the caller's to confirm. + assert.strictEqual(requests.length, 1); + const elsewhere = signSns({ + ...message, + MessageId: 'sub-2', + SubscribeURL: 'https://evil.example/confirm', + }); + await rejects(verify(request(elsewhere)), 'payload'); +}); + +test('Webhooks SES: a forged, foreign, old or repeated message is refused', async () => { + // Changed in the port: sently verified nothing. + const { verify } = verifier(); + const good = notification({ eventType: 'Send', mail: mail() }); + await rejects(verify(request({ ...good, Message: '{"eventType":"Delivery"}' })), 'signature'); + await rejects(verify(request({ ...good, TopicArn: 'arn:aws:sns:eu-west-1:999:other' })), 'topic'); + await rejects( + verify(request({ ...good, SigningCertURL: 'https://evil.example/cert.pem' })), + 'certificate', + ); + await rejects( + verify(request({ ...good, SigningCertURL: 'http://sns.eu-west-1.amazonaws.com/cert.pem' })), + 'certificate', + ); + await rejects(verify(request({ ...good, SigningCertURL: 'not a url' })), 'certificate'); + await rejects(verify(request({ ...good, SignatureVersion: '3' })), 'signature'); + await rejects( + verify( + request( + notification( + { eventType: 'Send', mail: mail() }, + { MessageId: 'old', Timestamp: '2027-01-01T10:00:00.000Z' }, + ), + ), + ), + 'timestamp', + ); + assert.strictEqual((await verify(request(good))).events.length, 2); + await rejects(verify(request(good)), 'replay'); + const sha1 = signSns( + { ...notification({ eventType: 'Send', mail: mail() }, { MessageId: 'v1' }) }, + '1', + ); + assert.strictEqual((await verify(request(sha1))).events.length, 2); + await rejects(verify(request({ ...good, Type: 'Other' })), 'payload'); + await rejects(verify({ body: '[]', headers: {} }), 'payload'); + await rejects(verify({ body: 'not json' }), 'payload'); + await rejects(verify({ body: Buffer.from([0xff, 0xfe]) }), 'payload'); + await rejects( + verify( + request({ + ...notification({ eventType: 'Send' }), + MessageId: undefined, + Signature: good.Signature, + }), + ), + 'payload', + ); + await assert.rejects(verify({ body: { Type: 'Notification' } }), /raw body/); +}); + +test('Webhooks SES: a certificate that cannot be read or is not valid now is refused', async () => { + const good = notification({ eventType: 'Send', mail: mail() }); + const failing = createSesWebhookVerifier({ + topicArns: [TOPIC], + now: () => NOW, + fetch: async () => { + throw new Error('offline'); + }, + }); + await rejects(failing.verify(request(good)), 'certificate'); + const missing = createSesWebhookVerifier({ + topicArns: [TOPIC], + now: () => NOW, + fetch: async () => new Response('gone', { status: 404 }), + }); + await rejects(missing.verify(request(good)), 'certificate'); + const garbage = createSesWebhookVerifier({ + topicArns: [TOPIC], + now: () => NOW, + fetch: async () => new Response('not a certificate'), + }); + await rejects(garbage.verify(request(good)), 'certificate'); + const late = verifier({ now: () => Date.parse('2200-01-01T00:00:00Z'), toleranceMs: 1e15 }); + await rejects(late.verify(request(good)), 'certificate'); +}); + +test('Webhooks SES: the options are checked, and a replay store may be given or none', async () => { + assert.throws(() => createSesWebhookVerifier({}), /topicArns must list/); + assert.throws(() => createSesWebhookVerifier({ topicArns: [''] }), /topic ARN is a string/); + assert.throws(() => createSesWebhookVerifier({ topicArns: [TOPIC], fetch: 'x' }), /fetch/); + assert.throws( + () => createSesWebhookVerifier({ topicArns: [TOPIC], toleranceMs: 0 }), + /toleranceMs/, + ); + assert.throws( + () => createSesWebhookVerifier({ topicArns: [TOPIC], replayStore: {} }), + /replayStore/, + ); + const seen = []; + const { verify } = verifier({ + replayStore: { add: async (id, ttlMs) => seen.push([id, ttlMs]) > 0 }, + }); + const good = notification({ eventType: 'Send', mail: mail() }); + await verify(request(good)); + assert.deepStrictEqual(seen, [['ses:sns-1', 2 * 60 * 60 * 1000]]); + const open = verifier({ replayStore: null }); + await open.verify(request(good)); + await open.verify(request(good)); + const big = verifier({ maxBodyBytes: 10 }); + await rejects(big.verify(request(good)), 'payload'); + const fetched = []; + const original = globalThis.fetch; + globalThis.fetch = async (url) => { + fetched.push(String(url)); + return new Response(require('../helpers/webhooks/sns.js').certificate); + }; + try { + await createSesWebhookVerifier({ topicArns: [TOPIC], now: () => NOW }).verify({ + body: Buffer.from(JSON.stringify(good)), + headers: new Headers(), + }); + } finally { + globalThis.fetch = original; + } + assert.deepStrictEqual(fetched, [CERT_URL]); +}); From 7130037862195c915a27ebed5ed72f4ad9e33cc2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 23:17:06 +0300 Subject: [PATCH 044/143] feat(webhooks): verified events of Resend, SendGrid, Mailgun and Postmark Ported from sently's webhooks, each verified before anything is read (S6): Resend by its Svix signature (HMAC-SHA256 of id, timestamp and the raw body, checked against the vector of the Svix documentation), a secret that is not one refused when the verifier is made; SendGrid's Signed Event Webhook by ECDSA P-256 over the timestamp and the body, each event of a batch taken once by its id; Mailgun by the HMAC of the timestamp and token of the body, the token taken once; all within five minutes. Postmark signs nothing, so the Basic credentials of its webhook URL are required and compared in constant time. The mappings are fixed where sently's were wrong: an unsubscribe is `unsubscribed`, not a complaint; a drop, a suppression or a failed email is `rejected`, not a bounce; a SendGrid block is a soft bounce; a Postmark bounce is read by its type. Tags and metadata come back as the transports send them. The budget of `./webhooks` is 8 KB for now. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 + docs/api/exports.md | 2 +- docs/guide/webhooks.md | 60 ++++++ scripts/size-budgets.json | 2 +- src/webhooks/index.d.ts | 90 +++++++++ src/webhooks/index.js | 14 +- src/webhooks/mailgun.js | 145 ++++++++++++++ src/webhooks/postmark.js | 161 ++++++++++++++++ src/webhooks/request.js | 26 +++ src/webhooks/resend.js | 168 ++++++++++++++++ src/webhooks/sendgrid.js | 206 ++++++++++++++++++++ src/webhooks/ses.js | 4 +- tests/helpers/webhooks/svix.js | 17 ++ tests/types/webhooks.test-d.ts | 18 ++ tests/unit/webhooks-mailgun.test.js | 274 +++++++++++++++++++++++++++ tests/unit/webhooks-resend.test.js | 149 +++++++++++++++ tests/unit/webhooks-sendgrid.test.js | 149 +++++++++++++++ 17 files changed, 1489 insertions(+), 4 deletions(-) create mode 100644 src/webhooks/mailgun.js create mode 100644 src/webhooks/postmark.js create mode 100644 src/webhooks/resend.js create mode 100644 src/webhooks/sendgrid.js create mode 100644 tests/helpers/webhooks/svix.js create mode 100644 tests/unit/webhooks-mailgun.test.js create mode 100644 tests/unit/webhooks-resend.test.js create mode 100644 tests/unit/webhooks-sendgrid.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 67d13e3..951d13c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -594,6 +594,14 @@ fails on upstream's behavior): `sns..amazonaws.com` URL read through the injected `fetch`, within an hour and once; it makes an event per recipient, says whether a bounce is hard or soft, makes a reject `rejected`, and hands a subscription confirmation to the caller. +- S6. sently (`src/webhooks/{resend,sendgrid,mailgun,postmark}.ts`): `parse` was exported apart + from any check; SendGrid's signature and Postmark's credentials were never checked; Resend's + and Mailgun's signatures were, with no window of time and no replay check, and a Resend secret + that was not base64 threw at each request. An unsubscribe was a spam complaint (SendGrid, + Brevo), a drop or a suppression a bounce (SendGrid, Mailgun), a block a deferral (SendGrid), a + failed Resend email a bounce, and every Postmark bounce `bounced` whatever its type. + `./webhooks` verifies first: Svix for Resend, ECDSA P-256 for SendGrid, HMAC for Mailgun, all + within five minutes and once by id; Basic credentials, required, for Postmark. ## Repository layout diff --git a/docs/api/exports.md b/docs/api/exports.md index 476afe1..d7507d1 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -44,7 +44,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | | `@alexify/mailer/render` | `renderContent`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | -| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | +| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | | `@alexify/mailer/sendmail` | `SendmailTransport` | [Sendmail](/transports/sendmail) | | `@alexify/mailer/ses` | `SesTransport` | [Amazon SES](/transports/ses) | diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index e590410..e7ccee7 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -93,3 +93,63 @@ confirmed for it. A bounce is `hard` when SES says Permanent, `soft` when Transient; a reject or a rendering failure is `rejected`. The `category` tags of the message are its `tags`, as `./ses` sends them, and its other tags its `metadata`. + +## Resend + +```js +const resend = createResendWebhookVerifier({ secret: process.env.RESEND_WEBHOOK_SECRET }); +``` + +Resend signs its webhooks with Svix: the HMAC-SHA256 of `svix-id`, +`svix-timestamp` and the body, with the secret `whsec_…` of the webhook. A +secret that is not one is refused when the verifier is made. An event names +its recipients in `to`, one event each; a bounce is `hard` when Resend says +Permanent, `soft` when Transient; a failed or suppressed email is `rejected`. +`tag1`, `tag2`… of the email's tags are the tags of the message and the others +its metadata, as `./resend` sends them. + +## SendGrid + +```js +const sendgrid = createSendGridWebhookVerifier({ + publicKey: process.env.SENDGRID_WEBHOOK_PUBLIC_KEY, +}); +``` + +Turn on the Signed Event Webhook in SendGrid and pass its verification key +(base64, as SendGrid shows it, or PEM). The signature is ECDSA P-256 over the +timestamp and the body. SendGrid posts events in batches: each event is taken +once by its `sg_event_id`, so a batch sent again keeps only what is new, and is +refused as `replay` when nothing is. A `bounce` is hard, a `blocked` one soft; a +`dropped` message is `rejected`; an unsubscribe is `unsubscribed`. The +categories are the tags of the message and its custom arguments its metadata; +`messageId` is the `X-Message-Id` of the receipt, without the filter SendGrid +adds after it. + +## Mailgun + +```js +const mailgun = createMailgunWebhookVerifier({ + signingKey: process.env.MAILGUN_WEBHOOK_SIGNING_KEY, +}); +``` + +The signature is the HMAC-SHA256 of the timestamp and the token of the body, +with the HTTP webhook signing key; the token is taken once. A permanent +failure is a hard bounce, a temporary one a deferred soft bounce, and a +failure for a suppressed address `rejected`. The user variables are the +metadata, and `messageId` the id of the receipt, in angle brackets. + +## Postmark + +```js +// The webhook URL in Postmark: https://hook:s3cret@example.com/webhooks/postmark +const postmark = createPostmarkWebhookVerifier({ username: 'hook', password: 's3cret' }); +``` + +Postmark signs nothing, so the URL of the webhook carries credentials, which +the verifier requires and compares in constant time. With no time or id +signed, nothing holds a request to a window or takes it once: deduplicate by +the `id` of the events. A bounce is read by its type: `HardBounce` is hard, +`SoftBounce` soft, `Transient` deferred, `SpamComplaint` a complaint, +`ManuallyDeactivated` a rejection. diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 538daac..6fdbb18 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4, "./render": 3, - "./webhooks": 5, + "./webhooks": 8, "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index a61025b..52bd79e 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -151,3 +151,93 @@ export interface SesWebhookResult extends WebhookResult<'ses'> { export declare function createSesWebhookVerifier( options: SesWebhookVerifierOptions, ): WebhookVerifier; + +/** + * The options of {@link createResendWebhookVerifier}. + */ +export interface ResendWebhookVerifierOptions extends WebhookVerifierOptions { + /** The signing secret of the webhook, `whsec_…`. */ + readonly secret: string; +} + +/** + * Makes the verifier of the webhooks of Resend, which Svix signs: the + * HMAC-SHA256 of `svix-id.svix-timestamp.body`, within five minutes, its id + * taken once. `tag1`, `tag2`… of the email's tags are the tags of the + * message, the others its metadata, as `./resend` sends them. + * + * @throws {TypeError} If the secret is not one. + */ +export declare function createResendWebhookVerifier( + options: ResendWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * The options of {@link createSendGridWebhookVerifier}. + */ +export interface SendGridWebhookVerifierOptions extends WebhookVerifierOptions { + /** + * The verification key of the Signed Event Webhook, base64 as SendGrid + * shows it, or PEM. + */ + readonly publicKey: string; +} + +/** + * Makes the verifier of SendGrid's Signed Event Webhook: ECDSA P-256 over + * the timestamp and the body, within five minutes, each event id taken once. + * The categories are the tags of the message, its custom arguments its + * metadata. + * + * @throws {TypeError} If the key is not one. + */ +export declare function createSendGridWebhookVerifier( + options: SendGridWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * The options of {@link createMailgunWebhookVerifier}. + */ +export interface MailgunWebhookVerifierOptions extends WebhookVerifierOptions { + /** The HTTP webhook signing key of the Mailgun domain. */ + readonly signingKey: string; +} + +/** + * Makes the verifier of Mailgun's webhooks: the HMAC-SHA256 of the + * timestamp and token of the body, within five minutes, the token taken + * once. A permanent failure is a hard bounce, a temporary one a deferred + * soft bounce, a suppression `rejected`; the user variables are the + * metadata. + * + * @throws {TypeError} If the key is missing. + */ +export declare function createMailgunWebhookVerifier( + options: MailgunWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * The options of {@link createPostmarkWebhookVerifier}: the credentials of the + * webhook's URL, as Postmark signs nothing. + */ +export interface PostmarkWebhookVerifierOptions { + readonly username: string; + readonly password: string; + /** + * The longest body taken, in bytes. + * @default 5242880 + */ + readonly maxBodyBytes?: number; +} + +/** + * Makes the verifier of Postmark's webhooks, whose URL carries Basic + * credentials (`https://user:password@…`), compared in constant time. + * Postmark signs no time or id, so nothing holds a request to a window or + * takes it once. A bounce is read by its type. + * + * @throws {TypeError} If a credential is missing. + */ +export declare function createPostmarkWebhookVerifier( + options: PostmarkWebhookVerifierOptions, +): WebhookVerifier>; diff --git a/src/webhooks/index.js b/src/webhooks/index.js index 81d527e..22343a2 100644 --- a/src/webhooks/index.js +++ b/src/webhooks/index.js @@ -1,6 +1,18 @@ // Not ported: the entry of ./webhooks, the verifiers of the providers' events. const { WebhookVerificationError, createMemoryReplayStore } = require('./request.js'); +const { createMailgunWebhookVerifier } = require('./mailgun.js'); +const { createPostmarkWebhookVerifier } = require('./postmark.js'); +const { createResendWebhookVerifier } = require('./resend.js'); +const { createSendGridWebhookVerifier } = require('./sendgrid.js'); const { createSesWebhookVerifier } = require('./ses.js'); -module.exports = { WebhookVerificationError, createMemoryReplayStore, createSesWebhookVerifier }; +module.exports = { + WebhookVerificationError, + createMailgunWebhookVerifier, + createMemoryReplayStore, + createPostmarkWebhookVerifier, + createResendWebhookVerifier, + createSendGridWebhookVerifier, + createSesWebhookVerifier, +}; diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js new file mode 100644 index 0000000..903de16 --- /dev/null +++ b/src/webhooks/mailgun.js @@ -0,0 +1,145 @@ +// Ported from sently v1.2.2 (src/webhooks/mailgun.ts): the events of +// Mailgun's webhooks, `MAILGUN_EVENT_MAP`, `parse` and `verifyPayload`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the signature is checked with the time of the request +// held to a window and its token taken once, where sently took any time and +// any number of replays and exported `parse` apart; a message Mailgun dropped +// for a suppression or rejected is `rejected`, not a bounce; an unsubscribe is +// `unsubscribed`; a temporary failure is a deferred soft bounce; and the +// message id is that of the receipt, in angle brackets. + +const { createHmac } = require('node:crypto'); +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + checkTime, + createEvent, + dateOf, + equalBytes, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, +} = require('./request.js'); + +const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; +const TYPES = new Map([ + ['accepted', 'accepted'], + ['delivered', 'delivered'], + ['rejected', 'rejected'], + ['complained', 'complained'], + ['unsubscribed', 'unsubscribed'], + ['opened', 'opened'], + ['clicked', 'clicked'], +]); +const HEX = /^[0-9a-f]+$/i; + +const tagsOf = (list) => { + const tags = []; + if (!Array.isArray(list)) return tags; + for (const tag of list) { + if (typeof tag === 'string') tags.push(tag); + } + return tags; +}; + +const metadataOf = (variables) => { + const metadata = Object.create(null); + if (!isRecord(variables)) return metadata; + for (const name in variables) { + const value = variables[name]; + if (typeof value === 'string') metadata[name] = value; + else if (typeof value === 'number' || typeof value === 'boolean') { + metadata[name] = String(value); + } + } + return metadata; +}; + +// What a failure is: a suppression or a rejection Mailgun made before any +// attempt, a temporary failure it retries, or a bounce. +const failureOf = (data) => { + const status = isRecord(data['delivery-status']) ? data['delivery-status'] : {}; + const reason = stringOf(data.reason); + if (reason?.startsWith('suppress-')) return { type: 'rejected', reason }; + const bounce = { kind: data.severity === 'temporary' ? 'soft' : 'hard' }; + if (status.code !== undefined && status.code !== null) bounce.code = String(status.code); + const diagnostic = stringOf(status.description) ?? stringOf(status.message); + if (diagnostic !== undefined) bounce.diagnostic = diagnostic; + return { type: data.severity === 'temporary' ? 'deferred' : 'bounced', bounce, reason }; +}; + +const messageIdOf = (message) => { + const id = isRecord(message?.headers) ? stringOf(message.headers['message-id']) : undefined; + if (id === undefined) return undefined; + return id.startsWith('<') ? id : `<${id}>`; +}; + +const eventOf = (data, token) => { + const failure = data.event === 'failed' ? failureOf(data) : undefined; + return createEvent({ + provider: 'mailgun', + type: failure?.type ?? TYPES.get(data.event) ?? 'unknown', + id: stringOf(data.id) ?? token, + messageId: messageIdOf(data.message), + recipient: stringOf(data.recipient), + occurredAt: dateOf(data.timestamp), + bounce: failure?.bounce, + reason: failure?.reason ?? (isRecord(data.reject) ? stringOf(data.reject.reason) : undefined), + url: stringOf(data.url), + tags: tagsOf(data.tags), + metadata: metadataOf(data['user-variables']), + raw: data, + }); +}; + +/** + * Makes the verifier of Mailgun's webhooks. + * + * @param options `signingKey`, the HTTP webhook signing key of the domain; + * `now`, `toleranceMs` (five minutes), `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createMailgunWebhookVerifier = (options = {}) => { + if (typeof options.signingKey !== 'string' || options.signingKey === '') { + throw new TypeError('signingKey is the HTTP webhook signing key of Mailgun.'); + } + const signingKey = options.signingKey; + const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); + return { + async verify(request) { + // The signature is a field of the body, read to find it and trusted + // only once it matches. + const payload = parseJsonOf(bodyBytesOf(request, common)); + const signature = isRecord(payload) ? payload.signature : undefined; + const timestamp = stringOf(signature?.timestamp); + const token = stringOf(signature?.token); + const given = stringOf(signature?.signature); + if ( + timestamp === undefined || + token === undefined || + given === undefined || + !HEX.test(given) + ) { + throw new WebhookVerificationError('signature', 'The request has no Mailgun signature.'); + } + const expected = createHmac('sha256', signingKey) + .update(timestamp + token) + .digest(); + if (!equalBytes(expected, Buffer.from(given, 'hex'))) { + throw new WebhookVerificationError('signature', 'The Mailgun signature does not match.'); + } + checkTime(Number(timestamp) * 1000, common); + await checkReplay(common, `mailgun:${token}`); + const data = payload['event-data']; + if (!isRecord(data)) { + throw new WebhookVerificationError('payload', 'The body has no Mailgun event.'); + } + return { events: [eventOf(data, token)] }; + }, + }; +}; + +module.exports = { createMailgunWebhookVerifier }; diff --git a/src/webhooks/postmark.js b/src/webhooks/postmark.js new file mode 100644 index 0000000..709f594 --- /dev/null +++ b/src/webhooks/postmark.js @@ -0,0 +1,161 @@ +// Ported from sently v1.2.2 (src/webhooks/postmark.ts): the events of +// Postmark's webhooks, `POSTMARK_EVENT_MAP` and `parse`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: Postmark signs nothing, so the credentials of the +// webhook's URL (Basic authentication) are required and compared in constant +// time, where sently parsed any request; a bounce is read by its type — a +// hard or a soft bounce, a deferral, a complaint, an unsubscribe or a +// rejection — where sently made every one `bounced`; and an unsubscribe of a +// subscription change is `unsubscribed`. + +const { + WebhookVerificationError, + bodyBytesOf, + createEvent, + dateOf, + equalBytes, + headerOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, +} = require('./request.js'); + +// What a bounce of Postmark is, by its Type. +const BOUNCES = new Map([ + ['HardBounce', { type: 'bounced', kind: 'hard' }], + ['BadEmailAddress', { type: 'bounced', kind: 'hard' }], + ['DMARCPolicy', { type: 'bounced', kind: 'hard' }], + ['SoftBounce', { type: 'bounced', kind: 'soft' }], + ['DnsError', { type: 'bounced', kind: 'soft' }], + ['Blocked', { type: 'bounced', kind: 'soft' }], + ['Transient', { type: 'deferred', kind: 'soft' }], + ['SpamComplaint', { type: 'complained' }], + ['SpamNotification', { type: 'complained' }], + ['Unsubscribe', { type: 'unsubscribed' }], + ['ManuallyDeactivated', { type: 'rejected' }], + ['SMTPApiError', { type: 'rejected' }], + ['TemplateRenderingFailed', { type: 'rejected' }], + ['AutoResponder', { type: 'unknown' }], + ['AddressChange', { type: 'unknown' }], + ['Subscribe', { type: 'unknown' }], + ['ChallengeVerification', { type: 'unknown' }], + ['InboundError', { type: 'unknown' }], +]); +const TYPES = new Map([ + ['Delivery', 'delivered'], + ['SpamComplaint', 'complained'], + ['Open', 'opened'], + ['Click', 'clicked'], +]); + +const credentialsOf = (options) => { + for (const name of ['username', 'password']) { + if (typeof options[name] !== 'string' || options[name] === '') { + throw new TypeError( + `${name} is required: Postmark signs nothing, so its webhook URL carries credentials.`, + ); + } + } + return Buffer.from(`${options.username}:${options.password}`, 'utf8'); +}; + +const checkCredentials = (request, expected) => { + const header = headerOf(request, 'authorization'); + const space = header === undefined ? -1 : header.indexOf(' '); + if (space < 0 || header.slice(0, space).toLowerCase() !== 'basic') { + throw new WebhookVerificationError('credentials', 'The request has no Basic credentials.'); + } + const given = Buffer.from(header.slice(space + 1).trim(), 'base64'); + if (!equalBytes(given, expected)) { + throw new WebhookVerificationError('credentials', 'The credentials do not match.'); + } +}; + +const describeRecord = (record) => { + if (record.RecordType === 'Bounce') { + const known = BOUNCES.get(record.Type) ?? { type: 'bounced', kind: 'undetermined' }; + let bounce; + if (known.kind !== undefined) { + bounce = { kind: known.kind }; + if (record.TypeCode !== undefined) bounce.code = String(record.TypeCode); + const diagnostic = stringOf(record.Details) ?? stringOf(record.Description); + if (diagnostic !== undefined) bounce.diagnostic = diagnostic; + } + return { type: known.type, bounce, reason: stringOf(record.Type), at: record.BouncedAt }; + } + if (record.RecordType === 'SubscriptionChange') { + return { + type: record.SuppressSending === true ? 'unsubscribed' : 'unknown', + reason: stringOf(record.SuppressionReason), + at: record.ChangedAt, + }; + } + return { + type: TYPES.get(record.RecordType) ?? 'unknown', + at: record.DeliveredAt ?? record.ReceivedAt ?? record.BouncedAt, + }; +}; + +const metadataOf = (values) => { + const metadata = Object.create(null); + if (!isRecord(values)) return metadata; + for (const name in values) { + if (typeof values[name] === 'string') metadata[name] = values[name]; + } + return metadata; +}; + +const eventOf = (record) => { + const described = describeRecord(record); + const recipient = stringOf(record.Recipient) ?? stringOf(record.Email); + const messageId = stringOf(record.MessageID); + const occurredAt = dateOf(described.at); + const id = + record.ID === undefined + ? `${record.RecordType}:${messageId}:${recipient}:${occurredAt?.toISOString()}` + : `${record.RecordType}:${record.ID}`; + const tag = stringOf(record.Tag); + return createEvent({ + provider: 'postmark', + type: described.type, + id, + messageId, + recipient, + occurredAt, + bounce: described.bounce, + reason: described.reason, + url: stringOf(record.OriginalLink), + tags: tag === undefined ? [] : [tag], + metadata: metadataOf(record.Metadata), + raw: record, + }); +}; + +/** + * Makes the verifier of Postmark's webhooks, whose URL carries credentials + * (`https://user:password@…`), as Postmark signs nothing. + * + * @param options `username` and `password` of the webhook's URL, and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createPostmarkWebhookVerifier = (options = {}) => { + const expected = credentialsOf(options); + // No time or id to hold a request to: the credentials alone stand for it. + const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkCredentials(request, expected); + const record = parseJsonOf(bytes); + if (!isRecord(record) || typeof record.RecordType !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not a Postmark event.'); + } + return { events: [eventOf(record)] }; + }, + }; +}; + +module.exports = { createPostmarkWebhookVerifier }; diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 7be5991..12fe1a1 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -213,6 +213,31 @@ const isRecord = (value) => typeof value === 'object' && value !== null && !Arra */ const stringOf = (value) => (typeof value === 'string' && value !== '' ? value : undefined); +const NUMBERED_TAG = /^tag(\d+)$/; + +/** + * The tags and metadata of a record of named tags, as the transports send + * them where a provider has named tags only: `tag1`, `tag2`… hold the tags of + * the message, in order, and the other names its metadata. + * + * @internal + */ +const splitNamedTags = (named) => { + const numbered = []; + const metadata = Object.create(null); + if (!isRecord(named)) return { tags: [], metadata }; + for (const name in named) { + const value = named[name]; + if (typeof value !== 'string') continue; + const match = NUMBERED_TAG.exec(name); + if (match === null) metadata[name] = value; + else numbered.push({ index: Number(match[1]), value }); + } + const tags = []; + for (const entry of numbered.toSorted((a, b) => a.index - b.index)) tags.push(entry.value); + return { tags, metadata }; +}; + module.exports = { WebhookVerificationError, bodyBytesOf, @@ -226,5 +251,6 @@ module.exports = { isRecord, parseJsonOf, resolveCommonOptions, + splitNamedTags, stringOf, }; diff --git a/src/webhooks/resend.js b/src/webhooks/resend.js new file mode 100644 index 0000000..5776356 --- /dev/null +++ b/src/webhooks/resend.js @@ -0,0 +1,168 @@ +// Ported from sently v1.2.2 (src/webhooks/resend.ts): the events of Resend, +// signed by Svix, `RESEND_EVENT_MAP`, `parse` and `verifySignature`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the signature is checked before the payload is read, +// over its raw bytes, with the time of the request held to a window and its +// id taken once, where sently exported `parse` apart and took any time and +// any number of replays; a secret that is not base64 is refused when the +// verifier is made, where sently's `atob` threw at each request; an event of +// several recipients is one per recipient; a bounce says whether it is hard +// or soft; and a failed or suppressed email is `rejected`, not `bounced` or +// `unknown`. + +const { createHmac } = require('node:crypto'); +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + checkTime, + createEvent, + dateOf, + equalBytes, + headerOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + splitNamedTags, + stringOf, +} = require('./request.js'); + +const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; +const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; +const TYPES = new Map([ + ['email.sent', 'accepted'], + ['email.scheduled', 'accepted'], + ['email.delivered', 'delivered'], + ['email.delivery_delayed', 'deferred'], + ['email.bounced', 'bounced'], + ['email.failed', 'rejected'], + ['email.suppressed', 'rejected'], + ['email.complained', 'complained'], + ['email.opened', 'opened'], + ['email.clicked', 'clicked'], +]); +const BOUNCE_KINDS = new Map([ + ['Permanent', 'hard'], + ['Transient', 'soft'], +]); + +/** + * The key of a Svix secret, `whsec_` and base64. + * + * @internal + */ +const svixKeyOf = (secret, name) => { + if (typeof secret !== 'string') throw new TypeError(`${name} is the signing secret, whsec_….`); + const encoded = secret.startsWith('whsec_') ? secret.slice(6) : secret; + if (encoded.length === 0 || encoded.length % 4 !== 0 || !BASE64.test(encoded)) { + throw new TypeError(`${name} must be the signing secret, whsec_ and base64.`); + } + return Buffer.from(encoded, 'base64'); +}; + +/** + * Checks the Svix signature of a request: the HMAC-SHA256 of + * `id.timestamp.body` against one of the `v1,` signatures of its header. + * + * @returns The id of the message. + * @internal + */ +const checkSvixSignature = (request, bytes, key, common) => { + const id = headerOf(request, 'svix-id') ?? headerOf(request, 'webhook-id'); + const timestamp = headerOf(request, 'svix-timestamp') ?? headerOf(request, 'webhook-timestamp'); + const header = headerOf(request, 'svix-signature') ?? headerOf(request, 'webhook-signature'); + if (id === undefined || timestamp === undefined || header === undefined) { + throw new WebhookVerificationError('signature', 'The request has no Svix signature.'); + } + const expected = createHmac('sha256', key) + .update(`${id}.${timestamp}.`, 'utf8') + .update(bytes) + .digest(); + let matched = false; + for (const part of header.split(' ')) { + if (!part.startsWith('v1,')) continue; + if (equalBytes(expected, Buffer.from(part.slice(3), 'base64'))) matched = true; + } + if (!matched) { + throw new WebhookVerificationError('signature', 'The Svix signature does not match.'); + } + if (!/^\d+$/.test(timestamp)) { + throw new WebhookVerificationError('timestamp', 'The Svix timestamp is not a number.'); + } + checkTime(Number(timestamp) * 1000, common); + return id; +}; + +const recipientsOf = (to) => { + if (typeof to === 'string') return [to]; + const recipients = []; + if (!Array.isArray(to)) return recipients; + for (const address of to) { + if (typeof address === 'string') recipients.push(address); + } + return recipients; +}; + +const eventsOf = (payload, id) => { + const data = isRecord(payload.data) ? payload.data : {}; + const type = TYPES.get(payload.type) ?? 'unknown'; + const { tags, metadata } = splitNamedTags(data.tags); + let bounce; + if (type === 'bounced' && isRecord(data.bounce)) { + bounce = { kind: BOUNCE_KINDS.get(data.bounce.type) ?? 'undetermined' }; + if (stringOf(data.bounce.message) !== undefined) bounce.diagnostic = data.bounce.message; + } + const detail = data.bounce ?? data.failed ?? data.suppressed ?? data.delivery_delayed; + const reason = isRecord(detail) + ? (stringOf(detail.subType) ?? stringOf(detail.reason) ?? stringOf(detail.message)) + : undefined; + const recipients = recipientsOf(data.to); + const list = recipients.length > 0 ? recipients : [undefined]; + const events = []; + for (let index = 0; index < list.length; index++) { + events.push( + createEvent({ + provider: 'resend', + type, + id: `${id}:${index}`, + messageId: stringOf(data.email_id), + recipient: list[index], + occurredAt: dateOf(payload.created_at ?? data.created_at), + bounce, + reason, + url: isRecord(data.click) ? stringOf(data.click.link) : undefined, + tags, + metadata, + raw: payload, + }), + ); + } + return events; +}; + +/** + * Makes the verifier of the webhooks of Resend, which Svix signs. + * + * @param options `secret`, the signing secret of the webhook (`whsec_…`); + * `now`, `toleranceMs` (five minutes), `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createResendWebhookVerifier = (options = {}) => { + const key = svixKeyOf(options.secret, 'secret'); + const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + const id = checkSvixSignature(request, bytes, key, common); + await checkReplay(common, `resend:${id}`); + const payload = parseJsonOf(bytes); + if (!isRecord(payload)) { + throw new WebhookVerificationError('payload', 'The body is not a Resend event.'); + } + return { events: eventsOf(payload, id) }; + }, + }; +}; + +module.exports = { checkSvixSignature, createResendWebhookVerifier, svixKeyOf }; diff --git a/src/webhooks/sendgrid.js b/src/webhooks/sendgrid.js new file mode 100644 index 0000000..430e449 --- /dev/null +++ b/src/webhooks/sendgrid.js @@ -0,0 +1,206 @@ +// Ported from sently v1.2.2 (src/webhooks/sendgrid.ts): the events of +// SendGrid's Event Webhook, `SENDGRID_EVENT_MAP` and `parse`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: the signature of the Signed Event Webhook (ECDSA P-256 +// over the timestamp and the raw body) is checked, its time held to a window +// and each event id taken once, where sently parsed the payload unverified; +// an unsubscribe is `unsubscribed`, not a spam complaint; a dropped message is +// `rejected`, not a bounce; a block is a soft bounce, not a deferral; and the +// custom arguments come back as the metadata of the event. + +const { createPublicKey, verify: verifySignature } = require('node:crypto'); +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + checkTime, + createEvent, + dateOf, + headerOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, +} = require('./request.js'); + +const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; +const SIGNATURE = 'x-twilio-email-event-webhook-signature'; +const TIMESTAMP = 'x-twilio-email-event-webhook-timestamp'; +const TYPES = new Map([ + ['processed', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + ['bounce', 'bounced'], + ['dropped', 'rejected'], + ['spamreport', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['group_unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], +]); +// The fields SendGrid writes into an event; the others are custom arguments. +const OWN_FIELDS = new Set([ + 'email', + 'timestamp', + 'event', + 'sg_event_id', + 'sg_message_id', + 'smtp-id', + 'category', + 'reason', + 'status', + 'response', + 'attempt', + 'type', + 'bounce_classification', + 'url', + 'url_offset', + 'useragent', + 'ip', + 'tls', + 'cert_err', + 'asm_group_id', + 'sg_machine_open', + 'sg_content_type', + 'sg_template_id', + 'sg_template_name', + 'marketing_campaign_id', + 'marketing_campaign_name', + 'marketing_campaign_version', + 'marketing_campaign_split_id', + 'mc_stats', + 'post_type', + 'pool', + 'send_at', + 'singlesend_id', + 'singlesend_name', + 'template_id', + 'template_version_id', + 'phase_id', + 'newsletter', +]); + +const publicKeyOf = (value) => { + if (typeof value !== 'string' || value.trim() === '') { + throw new TypeError('publicKey is the verification key of the Signed Event Webhook.'); + } + try { + return value.includes('-----BEGIN') + ? createPublicKey(value) + : createPublicKey({ key: Buffer.from(value, 'base64'), format: 'der', type: 'spki' }); + } catch (error) { + throw new TypeError('publicKey is not a public key.', { cause: error }); + } +}; + +const tagsOf = (category) => { + if (typeof category === 'string') return [category]; + const tags = []; + if (!Array.isArray(category)) return tags; + for (const value of category) { + if (typeof value === 'string') tags.push(value); + } + return tags; +}; + +const metadataOf = (record) => { + const metadata = Object.create(null); + for (const name in record) { + if (!OWN_FIELDS.has(name) && typeof record[name] === 'string') metadata[name] = record[name]; + } + return metadata; +}; + +// The id SendGrid gives a message is the X-Message-Id of its answer, the +// receipt's, followed by a dot and the id of a filter. +const messageIdOf = (record) => { + const id = stringOf(record.sg_message_id); + if (id === undefined) return undefined; + const dot = id.indexOf('.'); + return dot > 0 ? id.slice(0, dot) : id; +}; + +const eventOf = (record, index, timestamp) => { + const type = TYPES.get(record.event) ?? 'unknown'; + let bounce; + if (record.event === 'bounce') { + bounce = { kind: record.type === 'blocked' ? 'soft' : 'hard' }; + if (stringOf(record.status) !== undefined) bounce.code = record.status; + if (stringOf(record.reason) !== undefined) bounce.diagnostic = record.reason; + } + return createEvent({ + provider: 'sendgrid', + type, + id: stringOf(record.sg_event_id) ?? `${timestamp}:${index}`, + messageId: messageIdOf(record), + recipient: stringOf(record.email), + occurredAt: dateOf(record.timestamp), + bounce, + reason: + stringOf(record.bounce_classification) ?? + stringOf(record.reason) ?? + stringOf(record.response), + url: stringOf(record.url), + tags: tagsOf(record.category), + metadata: metadataOf(record), + raw: record, + }); +}; + +/** + * Makes the verifier of SendGrid's Signed Event Webhook. + * + * @param options `publicKey`, its verification key (base64 of the key, or + * PEM); `now`, `toleranceMs` (five minutes), `replayStore` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createSendGridWebhookVerifier = (options = {}) => { + const key = publicKeyOf(options.publicKey); + const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + const signature = headerOf(request, SIGNATURE); + const timestamp = headerOf(request, TIMESTAMP); + if (signature === undefined || timestamp === undefined) { + throw new WebhookVerificationError('signature', 'The request has no SendGrid signature.'); + } + const signed = Buffer.concat([Buffer.from(timestamp, 'utf8'), bytes]); + if (!verifySignature('sha256', signed, key, Buffer.from(signature, 'base64'))) { + throw new WebhookVerificationError('signature', 'The SendGrid signature does not match.'); + } + if (!/^\d+$/.test(timestamp)) { + throw new WebhookVerificationError('timestamp', 'The SendGrid timestamp is not a number.'); + } + checkTime(Number(timestamp) * 1000, common); + const payload = parseJsonOf(bytes); + if (!Array.isArray(payload)) { + throw new WebhookVerificationError('payload', 'The body is not a list of SendGrid events.'); + } + // A batch SendGrid sends again keeps the ids of its events: those taken + // before are left out, and a batch of nothing new is a replay. + const events = []; + let replayed = 0; + for (let index = 0; index < payload.length; index++) { + if (!isRecord(payload[index])) continue; + const event = eventOf(payload[index], index, timestamp); + try { + await checkReplay(common, `sendgrid:${event.id}`); + } catch (error) { + if (error.reason !== 'replay') throw error; + replayed++; + continue; + } + events.push(event); + } + if (events.length === 0 && replayed > 0) { + throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + } + return { events }; + }, + }; +}; + +module.exports = { createSendGridWebhookVerifier }; diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index cbcea33..80a165d 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -94,7 +94,9 @@ const tagsOf = (mail) => { const values = mail.tags[name]; if (!Array.isArray(values) || OWN_TAG.test(name) || name === 'priority') continue; if (name === 'category') { - for (const value of values) if (typeof value === 'string') tags.push(value); + for (const value of values) { + if (typeof value === 'string') tags.push(value); + } } else if (typeof values[0] === 'string') { metadata[name] = values[0]; } diff --git a/tests/helpers/webhooks/svix.js b/tests/helpers/webhooks/svix.js new file mode 100644 index 0000000..3b7ea19 --- /dev/null +++ b/tests/helpers/webhooks/svix.js @@ -0,0 +1,17 @@ +// Not ported: requests signed as Svix signs them +// (https://docs.svix.com/receiving/verifying-payloads/how-manual). + +const { createHmac } = require('node:crypto'); + +const SECRET = 'whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'; + +const signSvix = (body, id = 'msg_1', timestamp = '1800000000', secret = SECRET) => { + const key = Buffer.from(secret.slice(6), 'base64'); + const signature = createHmac('sha256', key).update(`${id}.${timestamp}.${body}`).digest('base64'); + return { + body, + headers: { 'svix-id': id, 'svix-timestamp': timestamp, 'svix-signature': `v1,${signature}` }, + }; +}; + +module.exports = { SECRET, signSvix }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index f3a6e7e..31c086d 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -2,7 +2,11 @@ import { expectAssignable, expectError, expectType } from 'tsd'; import type { DeliveryEvent } from '../../index.js'; import { WebhookVerificationError, + createMailgunWebhookVerifier, createMemoryReplayStore, + createPostmarkWebhookVerifier, + createResendWebhookVerifier, + createSendGridWebhookVerifier, createSesWebhookVerifier, type ReplayStore, type SesWebhookResult, @@ -27,3 +31,17 @@ expectError(ses.verify({ body: {} })); expectError(createSesWebhookVerifier({})); const error = new WebhookVerificationError('replay', 'seen'); expectType(error.reason); + +const resend = createResendWebhookVerifier({ secret: 'whsec_abc=', toleranceMs: 1000 }); +expectAssignable[]>((await resend.verify({ body: '' })).events); +const sendgrid = createSendGridWebhookVerifier({ publicKey: 'MFkw…' }); +expectAssignable[]>( + (await sendgrid.verify({ body: '' })).events, +); +createMailgunWebhookVerifier({ signingKey: 'key', replayStore: null }); +const postmark = createPostmarkWebhookVerifier({ username: 'u', password: 'p' }); +expectAssignable[]>( + (await postmark.verify({ body: '' })).events, +); +expectError(createPostmarkWebhookVerifier({ username: 'u' })); +expectError(createResendWebhookVerifier({})); diff --git a/tests/unit/webhooks-mailgun.test.js b/tests/unit/webhooks-mailgun.test.js new file mode 100644 index 0000000..d218103 --- /dev/null +++ b/tests/unit/webhooks-mailgun.test.js @@ -0,0 +1,274 @@ +// Not ported: tests of the Mailgun and Postmark verifiers of ./webhooks, +// ported from sently's Mailgun and Postmark webhooks. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createHmac } = require('node:crypto'); +const { + createMailgunWebhookVerifier, + createPostmarkWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); + +const KEY = 'key-mailgun-signing'; +const NOW = 1_800_000_000_000; +const mailgun = (options = {}) => + createMailgunWebhookVerifier({ signingKey: KEY, now: () => NOW, ...options }); +const signedMailgun = (data, token = 'token-1', timestamp = '1800000000') => ({ + body: JSON.stringify({ + signature: { + timestamp, + token, + signature: createHmac('sha256', KEY) + .update(timestamp + token) + .digest('hex'), + }, + 'event-data': data, + }), + headers: {}, +}); +const eventData = (fields = {}) => ({ + id: 'mg-event-1', + event: 'delivered', + timestamp: 1_799_999_990.5, + recipient: 'bob@example.com', + message: { headers: { 'message-id': '20270115.1@mg.example.com' } }, + tags: ['orders'], + 'user-variables': { orderId: '42', attempt: 2, nested: { a: 1 } }, + ...fields, +}); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; + +test('Webhooks Mailgun: a signed event, its message id that of the receipt', async () => { + const data = eventData(); + const { events } = await mailgun().verify(signedMailgun(data)); + assert.deepStrictEqual( + { ...events[0], metadata: { ...events[0].metadata } }, + { + provider: 'mailgun', + type: 'delivered', + id: 'mg-event-1', + messageId: '<20270115.1@mg.example.com>', + recipient: 'bob@example.com', + occurredAt: new Date(1_799_999_990_500), + tags: ['orders'], + metadata: { orderId: '42', attempt: '2' }, + raw: data, + }, + ); +}); + +test('Webhooks Mailgun: a failure is a bounce, a deferral or a rejection', async () => { + const verifier = mailgun(); + const cases = [ + [ + { + event: 'failed', + severity: 'permanent', + reason: 'bounce', + 'delivery-status': { code: 550, description: 'No such user' }, + }, + 'bounced', + { kind: 'hard', code: '550', diagnostic: 'No such user' }, + ], + [ + { + event: 'failed', + severity: 'temporary', + reason: 'generic', + 'delivery-status': { message: 'Try later' }, + }, + 'deferred', + { kind: 'soft', diagnostic: 'Try later' }, + ], + // Changed in the port: sently took a suppressed address for a bounce. + [{ event: 'failed', severity: 'permanent', reason: 'suppress-bounce' }, 'rejected', undefined], + [{ event: 'failed', severity: 'permanent' }, 'bounced', { kind: 'hard' }], + [{ event: 'rejected', reject: { reason: 'Sandbox domain' } }, 'rejected', undefined], + [{ event: 'accepted' }, 'accepted', undefined], + [{ event: 'complained' }, 'complained', undefined], + [{ event: 'unsubscribed' }, 'unsubscribed', undefined], + [{ event: 'opened' }, 'opened', undefined], + [{ event: 'clicked', url: 'https://e.com/a' }, 'clicked', undefined], + [{ event: 'stored' }, 'unknown', undefined], + ]; + for (let i = 0; i < cases.length; i++) { + const [fields, type, bounce] = cases[i]; + const { events } = await verifier.verify( + signedMailgun(eventData({ id: `e-${i}`, ...fields }), `t-${i}`), + ); + assert.strictEqual(events[0].type, type, JSON.stringify(fields)); + assert.deepStrictEqual(events[0].bounce, bounce); + } + const reject = await verifier.verify( + signedMailgun(eventData({ id: 'r', event: 'rejected', reject: { reason: 'Sandbox' } }), 't-r'), + ); + assert.strictEqual(reject.events[0].reason, 'Sandbox'); + const bare = await verifier.verify( + signedMailgun( + { event: 'delivered', message: { headers: { 'message-id': '' } } }, + 't-bare', + ), + ); + assert.strictEqual(bare.events[0].id, 't-bare'); + assert.strictEqual(bare.events[0].messageId, ''); + assert.deepStrictEqual(bare.events[0].tags, []); + const none = await verifier.verify(signedMailgun({ event: 'delivered' }, 't-none')); + assert.strictEqual(none.events[0].messageId, undefined); +}); + +test('Webhooks Mailgun: a forged, old or repeated request is refused', async () => { + const verifier = mailgun(); + const good = signedMailgun(eventData()); + const forged = JSON.parse(good.body); + forged.signature.signature = '00'.repeat(32); + await rejects(verifier.verify({ body: JSON.stringify(forged) }), 'signature'); + forged.signature.signature = 'not hex'; + await rejects(verifier.verify({ body: JSON.stringify(forged) }), 'signature'); + await rejects( + verifier.verify({ body: JSON.stringify({ 'event-data': eventData() }) }), + 'signature', + ); + // Changed in the port: sently took any time and any replay. + await rejects(verifier.verify(signedMailgun(eventData(), 'old', '1799990000')), 'timestamp'); + await verifier.verify(good); + await rejects(verifier.verify(good), 'replay'); + await rejects(verifier.verify(signedMailgun('nothing', 'no-data')), 'payload'); + await rejects(verifier.verify({ body: '"text"' }), 'signature'); + assert.throws(() => createMailgunWebhookVerifier({}), /signingKey/); +}); + +const POSTMARK = { username: 'hook', password: 's3cret' }; +const postmark = () => createPostmarkWebhookVerifier(POSTMARK); +const basic = (user, password) => `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; +const posted = (record, authorization = basic('hook', 's3cret')) => ({ + body: JSON.stringify(record), + headers: { Authorization: authorization }, +}); + +test('Webhooks Postmark: credentials are required, and compared', async () => { + // Changed in the port: sently parsed any request. + const record = { + RecordType: 'Delivery', + MessageID: 'pm-1', + Recipient: 'bob@example.com', + DeliveredAt: '2027-01-15T08:00:00Z', + Tag: 'welcome', + Metadata: { orderId: '42', n: 1 }, + }; + const { events } = await postmark().verify(posted(record)); + assert.deepStrictEqual( + { ...events[0], metadata: { ...events[0].metadata } }, + { + provider: 'postmark', + type: 'delivered', + id: 'Delivery:pm-1:bob@example.com:2027-01-15T08:00:00.000Z', + messageId: 'pm-1', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-15T08:00:00Z'), + tags: ['welcome'], + metadata: { orderId: '42' }, + raw: record, + }, + ); + await rejects(postmark().verify(posted(record, basic('hook', 'wrong'))), 'credentials'); + await rejects(postmark().verify(posted(record, 'Bearer x')), 'credentials'); + await rejects(postmark().verify({ body: JSON.stringify(record), headers: {} }), 'credentials'); + assert.strictEqual( + (await postmark().verify(posted(record, `basic ${basic('hook', 's3cret').slice(6)}`))).events + .length, + 1, + ); + await rejects(postmark().verify(posted({ nothing: true })), 'payload'); + assert.throws(() => createPostmarkWebhookVerifier({ username: 'hook' }), /password is required/); + assert.throws(() => createPostmarkWebhookVerifier({}), /username is required/); +}); + +test('Webhooks Postmark: a bounce is read by its type', async () => { + const verifier = postmark(); + const bounce = (type, fields = {}) => + posted({ + RecordType: 'Bounce', + ID: 4242, + Type: type, + TypeCode: 1, + Email: 'bob@example.com', + MessageID: 'pm-2', + BouncedAt: '2027-01-15T08:00:00Z', + Description: 'Bounced', + Details: 'smtp;550 5.1.1', + ...fields, + }); + const cases = [ + ['HardBounce', 'bounced', 'hard'], + ['SoftBounce', 'bounced', 'soft'], + ['Transient', 'deferred', 'soft'], + // Changed in the port: sently made each of these a bounce. + ['SpamComplaint', 'complained', undefined], + ['Unsubscribe', 'unsubscribed', undefined], + ['ManuallyDeactivated', 'rejected', undefined], + ['AutoResponder', 'unknown', undefined], + ['SomethingNew', 'bounced', 'undetermined'], + ]; + for (const [type, eventType, kind] of cases) { + const { events } = await verifier.verify(bounce(type)); + assert.strictEqual(events[0].type, eventType, type); + assert.strictEqual(events[0].bounce?.kind, kind, type); + assert.strictEqual(events[0].id, 'Bounce:4242'); + assert.strictEqual(events[0].reason, type); + } + const hard = await verifier.verify(bounce('HardBounce')); + assert.deepStrictEqual(hard.events[0].bounce, { + kind: 'hard', + code: '1', + diagnostic: 'smtp;550 5.1.1', + }); + const plain = await verifier.verify( + bounce('HardBounce', { TypeCode: undefined, Details: '', Description: undefined }), + ); + assert.deepStrictEqual(plain.events[0].bounce, { kind: 'hard' }); + const others = [ + [ + { RecordType: 'SpamComplaint', Email: 'bob@example.com', BouncedAt: '2027-01-15T08:00:00Z' }, + 'complained', + ], + [ + { RecordType: 'Open', Recipient: 'bob@example.com', ReceivedAt: '2027-01-15T08:00:00Z' }, + 'opened', + ], + [ + { RecordType: 'Click', Recipient: 'bob@example.com', OriginalLink: 'https://e.com/a' }, + 'clicked', + ], + [ + { + RecordType: 'SubscriptionChange', + Recipient: 'bob@example.com', + SuppressSending: true, + SuppressionReason: 'ManualSuppression', + ChangedAt: '2027-01-15T08:00:00Z', + }, + 'unsubscribed', + ], + [ + { RecordType: 'SubscriptionChange', Recipient: 'bob@example.com', SuppressSending: false }, + 'unknown', + ], + [{ RecordType: 'Inbound' }, 'unknown'], + ]; + for (const [record, type] of others) { + const { events } = await verifier.verify(posted(record)); + assert.strictEqual(events[0].type, type, record.RecordType); + } + const click = await verifier.verify(posted(others[2][0])); + assert.strictEqual(click.events[0].url, 'https://e.com/a'); + assert.deepStrictEqual(click.events[0].tags, []); + const change = await verifier.verify(posted(others[3][0])); + assert.strictEqual(change.events[0].reason, 'ManualSuppression'); +}); diff --git a/tests/unit/webhooks-resend.test.js b/tests/unit/webhooks-resend.test.js new file mode 100644 index 0000000..1833c08 --- /dev/null +++ b/tests/unit/webhooks-resend.test.js @@ -0,0 +1,149 @@ +// Not ported: tests of the Resend verifier of ./webhooks, ported from sently's +// Resend webhook, with requests signed as Svix signs them. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { + createResendWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); +const { SECRET, signSvix } = require('../helpers/webhooks/svix.js'); + +const NOW = 1_800_000_000_000; +const verifier = (options = {}) => + createResendWebhookVerifier({ secret: SECRET, now: () => NOW, ...options }); +const event = (type, data = {}) => + JSON.stringify({ + type, + created_at: '2027-01-15T08:00:00.000Z', + data: { + email_id: 'e-1', + created_at: '2027-01-15T07:59:59.000Z', + to: ['bob@example.com', 'carol@example.com'], + tags: { tag2: 'eu', tag1: 'orders', orderId: '42' }, + ...data, + }, + }); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; + +test('Webhooks Resend: the vector of the Svix documentation verifies', async () => { + const resend = createResendWebhookVerifier({ secret: SECRET, now: () => 1_614_265_330_000 }); + const { events } = await resend.verify({ + body: '{"test": 2432232314}', + headers: { + 'svix-id': 'msg_p5jXN8AQM9LWM0D4loKWxJek', + 'svix-timestamp': '1614265330', + 'svix-signature': 'v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=', + }, + }); + assert.strictEqual(events[0].type, 'unknown'); + assert.strictEqual(events[0].id, 'msg_p5jXN8AQM9LWM0D4loKWxJek:0'); +}); + +test('Webhooks Resend: a bounce is an event per recipient, with its kind', async () => { + const body = event('email.bounced', { + bounce: { message: 'Mailbox does not exist', subType: 'General', type: 'Permanent' }, + }); + const { events } = await verifier().verify(signSvix(body, 'msg_b')); + assert.strictEqual(events.length, 2); + assert.deepStrictEqual( + { ...events[1], metadata: { ...events[1].metadata } }, + { + provider: 'resend', + type: 'bounced', + id: 'msg_b:1', + messageId: 'e-1', + recipient: 'carol@example.com', + occurredAt: new Date('2027-01-15T08:00:00.000Z'), + bounce: { kind: 'hard', diagnostic: 'Mailbox does not exist' }, + reason: 'General', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + raw: JSON.parse(body), + }, + ); +}); + +test('Webhooks Resend: every type of email event', async () => { + const resend = verifier(); + const cases = [ + ['email.sent', 'accepted'], + ['email.scheduled', 'accepted'], + ['email.delivered', 'delivered'], + ['email.delivery_delayed', 'deferred'], + ['email.complained', 'complained'], + ['email.opened', 'opened'], + // Changed in the port: sently took a failure for a bounce, and knew no suppression. + ['email.failed', 'rejected'], + ['email.suppressed', 'rejected'], + ['email.received', 'unknown'], + ]; + for (let i = 0; i < cases.length; i++) { + const { events } = await resend.verify(signSvix(event(cases[i][0]), `msg_${i}`)); + assert.strictEqual(events[0].type, cases[i][1], cases[i][0]); + } + const click = await resend.verify( + signSvix( + event('email.clicked', { to: 'bob@example.com', click: { link: 'https://e.com/a' } }), + 'msg_c', + ), + ); + assert.strictEqual(click.events.length, 1); + assert.strictEqual(click.events[0].url, 'https://e.com/a'); + const failed = await resend.verify( + signSvix(event('email.failed', { failed: { reason: 'reached_daily_quota' } }), 'msg_f'), + ); + assert.strictEqual(failed.events[0].reason, 'reached_daily_quota'); + const soft = await resend.verify( + signSvix(event('email.bounced', { bounce: { type: 'Transient' } }), 'msg_s'), + ); + assert.strictEqual(soft.events[0].bounce.kind, 'soft'); + const bare = await resend.verify( + signSvix(JSON.stringify({ type: 'email.bounced', data: { bounce: {} } }), 'msg_e'), + ); + assert.deepStrictEqual(bare.events[0].bounce, { kind: 'undetermined' }); + assert.strictEqual(bare.events[0].recipient, undefined); + assert.deepStrictEqual(bare.events[0].tags, []); + const nothing = await resend.verify(signSvix(JSON.stringify({ type: 'email.sent' }), 'msg_n')); + assert.strictEqual(nothing.events[0].messageId, undefined); +}); + +test('Webhooks Resend: a forged, old or repeated request is refused', async () => { + // Changed in the port: sently took any time and any replay. + const resend = verifier(); + const body = event('email.delivered'); + const signed = signSvix(body, 'msg_r'); + await rejects(resend.verify({ ...signed, body: event('email.bounced') }), 'signature'); + await rejects(resend.verify({ body, headers: {} }), 'signature'); + await rejects( + resend.verify({ body, headers: { ...signed.headers, 'svix-signature': 'v2,abc v1,' } }), + 'signature', + ); + await rejects(resend.verify(signSvix(body, 'msg_old', '1799999000')), 'timestamp'); + await rejects(resend.verify(signSvix(body, 'msg_bad', 'soon')), 'timestamp'); + await resend.verify(signed); + await rejects(resend.verify(signed), 'replay'); + await rejects(resend.verify(signSvix('[1]', 'msg_array')), 'payload'); + // Standard Webhooks names its headers webhook-*. + const standard = signSvix(body, 'msg_std'); + const headers = new Headers({ + 'webhook-id': 'msg_std', + 'webhook-timestamp': standard.headers['svix-timestamp'], + 'webhook-signature': standard.headers['svix-signature'], + }); + assert.strictEqual((await resend.verify({ body: Buffer.from(body), headers })).events.length, 2); +}); + +test('Webhooks Resend: a secret that is not one is refused when the verifier is made', () => { + // Changed in the port: sently's atob threw at each request. + assert.throws(() => createResendWebhookVerifier({}), /signing secret/); + assert.throws(() => createResendWebhookVerifier({ secret: 'whsec_not base64!' }), /base64/); + assert.throws(() => createResendWebhookVerifier({ secret: 'whsec_' }), /base64/); + assert.ok(createResendWebhookVerifier({ secret: SECRET.slice(6) })); +}); diff --git a/tests/unit/webhooks-sendgrid.test.js b/tests/unit/webhooks-sendgrid.test.js new file mode 100644 index 0000000..2bd0540 --- /dev/null +++ b/tests/unit/webhooks-sendgrid.test.js @@ -0,0 +1,149 @@ +// Not ported: tests of the SendGrid verifier of ./webhooks, ported from +// sently's SendGrid webhook, with a P-256 key made for the test. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { generateKeyPairSync, sign } = require('node:crypto'); +const { + createSendGridWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); + +const { publicKey, privateKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); +const PUBLIC_KEY = publicKey.export({ format: 'der', type: 'spki' }).toString('base64'); +const NOW = 1_800_000_000_000; + +const signed = (events, timestamp = '1800000000') => { + const body = JSON.stringify(events); + const signature = sign('sha256', Buffer.from(timestamp + body), privateKey).toString('base64'); + return { + body, + headers: { + 'X-Twilio-Email-Event-Webhook-Signature': signature, + 'X-Twilio-Email-Event-Webhook-Timestamp': timestamp, + }, + }; +}; +const verifier = (options = {}) => + createSendGridWebhookVerifier({ publicKey: PUBLIC_KEY, now: () => NOW, ...options }); +const record = (event, fields = {}) => ({ + email: 'bob@example.com', + timestamp: 1_799_999_990, + event, + sg_event_id: `ev-${event}`, + sg_message_id: 'x-msg-1.filterdrecv-5645d9c87f-2qbs9-1-5E3E2C3D-1.0', + category: ['orders', 'eu'], + orderId: '42', + ...fields, +}); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; + +test('Webhooks SendGrid: a signed batch is an event per record', async () => { + const records = [ + record('bounce', { + status: '5.1.1', + reason: '550 5.1.1 unknown user', + bounce_classification: 'Invalid Address', + }), + record('bounce', { sg_event_id: 'ev-blocked', type: 'blocked', reason: 'IP blocked' }), + ]; + const { events } = await verifier().verify(signed(records)); + assert.deepStrictEqual( + { ...events[0], metadata: { ...events[0].metadata } }, + { + provider: 'sendgrid', + type: 'bounced', + id: 'ev-bounce', + messageId: 'x-msg-1', + recipient: 'bob@example.com', + occurredAt: new Date(1_799_999_990_000), + bounce: { kind: 'hard', code: '5.1.1', diagnostic: '550 5.1.1 unknown user' }, + reason: 'Invalid Address', + tags: ['orders', 'eu'], + metadata: { orderId: '42' }, + raw: records[0], + }, + ); + // Changed in the port: sently took a block for a deferral. + assert.strictEqual(events[1].type, 'bounced'); + assert.strictEqual(events[1].bounce.kind, 'soft'); +}); + +test('Webhooks SendGrid: every type of event', async () => { + const cases = [ + ['processed', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + // Changed in the port: sently took a drop for a bounce, an unsubscribe for a complaint. + ['dropped', 'rejected'], + ['spamreport', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['group_unsubscribe', 'unsubscribed'], + ['group_resubscribe', 'unknown'], + ['open', 'opened'], + ['click', 'clicked'], + ]; + const records = []; + for (const [event] of cases) records.push(record(event)); + records.push( + record('click', { sg_event_id: 'ev-url', url: 'https://e.com/a', category: 'single' }), + ); + records.push(record('delivered', { sg_event_id: undefined, sg_message_id: 'plain-id' })); + records.push('not a record'); + const { events } = await verifier().verify(signed(records)); + assert.deepStrictEqual( + events.slice(0, cases.length).map((event) => event.type), + cases.map((entry) => entry[1]), + ); + assert.strictEqual(events[cases.length].url, 'https://e.com/a'); + assert.deepStrictEqual(events[cases.length].tags, ['single']); + assert.strictEqual(events[cases.length + 1].id, '1800000000:11'); + assert.strictEqual(events[cases.length + 1].messageId, 'plain-id'); + const bare = await verifier().verify(signed([{ event: 'delivered', sg_event_id: 'bare' }])); + assert.deepStrictEqual(bare.events[0].tags, []); + assert.strictEqual(bare.events[0].messageId, undefined); +}); + +test('Webhooks SendGrid: a forged, old or repeated batch is refused', async () => { + // Changed in the port: sently verified nothing. + const sendgrid = verifier(); + const good = signed([record('delivered'), record('open')]); + await rejects(sendgrid.verify({ ...good, body: good.body.replace('bob', 'eve') }), 'signature'); + await rejects(sendgrid.verify({ body: good.body, headers: {} }), 'signature'); + await rejects(sendgrid.verify(signed([record('delivered')], '1799999000')), 'timestamp'); + await rejects(sendgrid.verify(signed([record('delivered')], 'later')), 'timestamp'); + assert.strictEqual((await sendgrid.verify(good)).events.length, 2); + await rejects(sendgrid.verify(good), 'replay'); + // A batch sent again with a new event keeps the new one alone. + const more = await sendgrid.verify(signed([record('delivered'), record('click')])); + assert.deepStrictEqual( + more.events.map((event) => event.type), + ['clicked'], + ); + await rejects(sendgrid.verify(signed({ event: 'delivered' })), 'payload'); + const failing = verifier({ + replayStore: { + add: () => { + throw new WebhookVerificationError('payload', 'store down'); + }, + }, + }); + await rejects(failing.verify(signed([record('delivered')])), 'payload'); +}); + +test('Webhooks SendGrid: the key may be PEM, and one that is not a key is refused', async () => { + const pem = publicKey.export({ format: 'pem', type: 'spki' }); + const fromPem = createSendGridWebhookVerifier({ publicKey: pem, now: () => NOW }); + assert.strictEqual((await fromPem.verify(signed([record('delivered')]))).events.length, 1); + assert.throws(() => createSendGridWebhookVerifier({}), /publicKey is the verification key/); + assert.throws( + () => createSendGridWebhookVerifier({ publicKey: 'bm90IGEga2V5' }), + /not a public key/, + ); +}); From 1e43ede390f29ddf55583fbea109efdc24999827 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 23:22:56 +0300 Subject: [PATCH 045/143] feat(webhooks): verified events of MailerSend, Mailtrap, Brevo, Mailjet and SparkPost Each verified as its documentation says: MailerSend and Mailtrap by the HMAC-SHA256 of the raw body, each event taken once by its id for a day and an hour as neither signs a time, Mailtrap's JSON or JSON Lines both read; Brevo by the token its webhook is configured with, as a Bearer token or a custom header; Mailjet and SparkPost by the Basic credentials of their webhook URL, as neither signs its events. Brevo's mapping is fixed where sently's was wrong (S7): a blocked or invalid address is `rejected`, a soft bounce soft, an unsubscribe not a complaint. Each reads back the tags and metadata its transport sends. Basic, HMAC and token checks move into the shared `request.js`. The budget of `./webhooks` is 10 KB for now. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 + docs/api/exports.md | 2 +- docs/guide/webhooks.md | 43 +++ scripts/size-budgets.json | 2 +- src/webhooks/brevo.js | 132 +++++++++ src/webhooks/index.d.ts | 101 ++++++- src/webhooks/index.js | 10 + src/webhooks/mailersend.js | 124 ++++++++ src/webhooks/mailgun.js | 28 +- src/webhooks/mailjet.js | 89 ++++++ src/webhooks/mailtrap.js | 137 +++++++++ src/webhooks/postmark.js | 43 +-- src/webhooks/request.js | 110 ++++++- src/webhooks/sparkpost.js | 119 ++++++++ tests/types/webhooks.test-d.ts | 19 ++ tests/unit/webhooks-mailgun.test.js | 2 +- tests/unit/webhooks-more.test.js | 432 ++++++++++++++++++++++++++++ 17 files changed, 1329 insertions(+), 72 deletions(-) create mode 100644 src/webhooks/brevo.js create mode 100644 src/webhooks/mailersend.js create mode 100644 src/webhooks/mailjet.js create mode 100644 src/webhooks/mailtrap.js create mode 100644 src/webhooks/sparkpost.js create mode 100644 tests/unit/webhooks-more.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 951d13c..cb0e059 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -602,6 +602,14 @@ fails on upstream's behavior): failed Resend email a bounce, and every Postmark bounce `bounced` whatever its type. `./webhooks` verifies first: Svix for Resend, ECDSA P-256 for SendGrid, HMAC for Mailgun, all within five minutes and once by id; Basic credentials, required, for Postmark. +- S7. sently (`src/webhooks/brevo.ts`): any request was parsed, Brevo signing nothing; a blocked + or invalid address and a soft bounce were all `bounced`, and an unsubscribe a complaint. + `./webhooks` requires the token the webhook is configured with (`Authorization: Bearer` or a + custom header), compared in constant time; a blocked or invalid address is `rejected`, a soft + bounce soft, and the metadata comes back from `X-Mailin-custom`, as `./brevo` sends it. + MailerSend, Mailtrap, Mailjet and SparkPost, which sently and unemail lack, are verified from + their documentation: an HMAC of the body (MailerSend, Mailtrap; no time is signed, so each + event is taken once by its id for a day and an hour), Basic credentials (Mailjet, SparkPost). ## Repository layout diff --git a/docs/api/exports.md b/docs/api/exports.md index d7507d1..1ca6df6 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -44,7 +44,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | | `@alexify/mailer/render` | `renderContent`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | -| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | +| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMailerSendWebhookVerifier`, `createMailtrapWebhookVerifier`, `createBrevoWebhookVerifier`, `createMailjetWebhookVerifier`, `createSparkPostWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | | `@alexify/mailer/sendmail` | `SendmailTransport` | [Sendmail](/transports/sendmail) | | `@alexify/mailer/ses` | `SesTransport` | [Amazon SES](/transports/ses) | diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index e7ccee7..6c1508b 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -153,3 +153,46 @@ signed, nothing holds a request to a window or takes it once: deduplicate by the `id` of the events. A bounce is read by its type: `HardBounce` is hard, `SoftBounce` soft, `Transient` deferred, `SpamComplaint` a complaint, `ManuallyDeactivated` a rejection. + +## MailerSend and Mailtrap + +```js +const mailersend = createMailerSendWebhookVerifier({ secret: process.env.MAILERSEND_WEBHOOK_SECRET }); +const mailtrap = createMailtrapWebhookVerifier({ secret: process.env.MAILTRAP_WEBHOOK_SECRET }); +``` + +Both sign the raw body with HMAC-SHA256, in hexadecimal in `Signature` +(MailerSend) or `Mailtrap-Signature`. Neither signs a time, so no window holds +a request; each event is taken once by its id instead, kept a day and an hour, +past Mailtrap's retries of a day. Mailtrap posts JSON or JSON Lines, both read. +MailerSend's metadata comes back from the `X-Metadata-` fields of the email +when the event carries them; Mailtrap's category and `tag_` variables are the +tags and its other custom variables the metadata, as `./mailtrap` sends them. + +## Brevo + +```js +const brevo = createBrevoWebhookVerifier({ token: process.env.BREVO_WEBHOOK_TOKEN }); +``` + +Brevo signs nothing: give the webhook a token, its `auth` token, which comes as +`Authorization: Bearer `, or a custom header with `header: 'x-webhook-token'`. +The verifier requires it and compares it in constant time. A blocked or invalid +address is `rejected`, a soft bounce soft, and the metadata comes back from +`X-Mailin-custom`, as `./brevo` sends it. + +## Mailjet and SparkPost + +```js +// The callback URL in Mailjet: https://hook:s3cret@example.com/webhooks/mailjet +const mailjet = createMailjetWebhookVerifier({ username: 'hook', password: 's3cret' }); +const sparkpost = createSparkPostWebhookVerifier({ username: 'hook', password: 's3cret' }); +``` + +Neither signs its events, so their webhooks carry Basic credentials (in +Mailjet's callback URL, SparkPost's `auth_type: basic`), which the verifiers +require. Mailjet's message id is the MessageUUID of the receipt, its custom +campaign the tag and its event payload the metadata; SparkPost's message id is +the transmission's, and a bounce is hard for the classes 10, 30 and 90, +undetermined for 1, soft for the others. As for Postmark, nothing holds a +request to a window or takes it once. diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 6fdbb18..bd893c0 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4, "./render": 3, - "./webhooks": 8, + "./webhooks": 10, "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, diff --git a/src/webhooks/brevo.js b/src/webhooks/brevo.js new file mode 100644 index 0000000..19bd2e7 --- /dev/null +++ b/src/webhooks/brevo.js @@ -0,0 +1,132 @@ +// Ported from sently v1.2.2 (src/webhooks/brevo.ts): the events of Brevo's +// transactional webhooks, `BREVO_EVENT_MAP` and `parse`. +// https://github.com/omqkhafi/sently — MIT, Copyright 2026 sendx contributors +// Changed in the port: Brevo signs nothing, so the token the webhook is +// configured with (its `auth` token, or a custom header) is required and +// compared in constant time, where sently parsed any request; an unsubscribe +// is `unsubscribed`, not a spam complaint; a blocked or invalid address is +// `rejected`, not a bounce; a soft bounce is soft; and the metadata comes +// back from `X-Mailin-custom`, as `./brevo` sends it. + +const { + WebhookVerificationError, + bodyBytesOf, + createEvent, + dateOf, + equalBytes, + headerOf, + isRecord, + jsonRecordOf, + parseJsonOf, + resolveCommonOptions, + stringOf, + stringsOf, +} = require('./request.js'); + +const TYPES = new Map([ + ['request', 'accepted'], + ['sent', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + ['hard_bounce', 'bounced'], + ['hardBounce', 'bounced'], + ['soft_bounce', 'bounced'], + ['softBounce', 'bounced'], + ['blocked', 'rejected'], + ['invalid', 'rejected'], + ['invalid_email', 'rejected'], + ['error', 'rejected'], + ['spam', 'complained'], + ['complaint', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['unsubscribed', 'unsubscribed'], + ['opened', 'opened'], + ['unique_opened', 'opened'], + ['uniqueOpened', 'opened'], + ['proxy_open', 'opened'], + ['click', 'clicked'], + ['unique_click', 'clicked'], +]); +const HARD = new Set(['hard_bounce', 'hardBounce']); + +const tokenCheckOf = (options) => { + if (typeof options.token !== 'string' || options.token === '') { + throw new TypeError( + 'token is required: Brevo signs nothing, so its webhook carries a token (auth or a header).', + ); + } + const header = options.header ?? 'authorization'; + if (typeof header !== 'string' || header === '') throw new TypeError('header is a field name.'); + // The auth token of a webhook comes as `Authorization: Bearer `. + const expected = Buffer.from( + header.toLowerCase() === 'authorization' ? `Bearer ${options.token}` : options.token, + 'utf8', + ); + return (request) => { + const given = headerOf(request, header); + if (given === undefined || !equalBytes(Buffer.from(given.trim(), 'utf8'), expected)) { + throw new WebhookVerificationError('credentials', 'The webhook token does not match.'); + } + }; +}; + +const eventOf = (record) => { + const type = TYPES.get(record.event) ?? 'unknown'; + const reason = stringOf(record.reason); + let bounce; + if (type === 'bounced') { + bounce = { kind: HARD.has(record.event) ? 'hard' : 'soft' }; + if (reason !== undefined) bounce.diagnostic = reason; + } + const messageId = stringOf(record['message-id']) ?? stringOf(record.messageId); + const recipient = stringOf(record.email); + const at = record.ts_event ?? record.ts_epoch ?? record.ts ?? record.date; + const tags = stringsOf(record.tags); + if (tags.length === 0 && stringOf(record.tag) !== undefined) tags.push(record.tag); + return createEvent({ + provider: 'brevo', + type, + id: `${record.event}:${messageId}:${recipient}:${at}`, + messageId, + recipient, + occurredAt: dateOf(at), + bounce, + reason, + url: stringOf(record.link), + tags, + metadata: jsonRecordOf(record['X-Mailin-custom']), + raw: record, + }); +}; + +/** + * Makes the verifier of Brevo's transactional webhooks, which carry the token + * they are configured with: their `auth` token, as `Authorization: Bearer`, + * or the value of a custom `header`. + * + * @param options `token`, `header` (`authorization`) and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createBrevoWebhookVerifier = (options = {}) => { + const checkToken = tokenCheckOf(options); + const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkToken(request); + const payload = parseJsonOf(bytes); + const records = Array.isArray(payload) ? payload : [payload]; + const events = []; + for (const record of records) { + if (isRecord(record)) events.push(eventOf(record)); + } + if (events.length === 0) { + throw new WebhookVerificationError('payload', 'The body is not a Brevo event.'); + } + return { events }; + }, + }; +}; + +module.exports = { createBrevoWebhookVerifier }; diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 52bd79e..e47e0c1 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -217,10 +217,10 @@ export declare function createMailgunWebhookVerifier( ): WebhookVerifier>; /** - * The options of {@link createPostmarkWebhookVerifier}: the credentials of the - * webhook's URL, as Postmark signs nothing. + * The options of a verifier of a webhook whose URL carries Basic credentials, + * as its provider signs nothing (Postmark, Mailjet, SparkPost). */ -export interface PostmarkWebhookVerifierOptions { +export interface BasicWebhookVerifierOptions { readonly username: string; readonly password: string; /** @@ -239,5 +239,98 @@ export interface PostmarkWebhookVerifierOptions { * @throws {TypeError} If a credential is missing. */ export declare function createPostmarkWebhookVerifier( - options: PostmarkWebhookVerifierOptions, + options: BasicWebhookVerifierOptions, ): WebhookVerifier>; + +/** + * The options of a verifier of a webhook signed with an HMAC of its body and + * no time: an event is taken once by its id, kept a day and an hour. + */ +export interface SignedBodyWebhookVerifierOptions { + /** The signing secret of the webhook. */ + readonly secret: string; + /** The ids already taken; an in-process store by default, `null` for none. */ + readonly replayStore?: ReplayStore | null; + /** + * The longest body taken, in bytes. + * @default 5242880 + */ + readonly maxBodyBytes?: number; +} + +/** + * Makes the verifier of MailerSend's webhooks: the HMAC-SHA256 of the body, + * in hexadecimal in `Signature`. The metadata is read from the + * `X-Metadata-` header fields of the email, when the event carries them. + * + * @throws {TypeError} If the secret is missing. + */ +export declare function createMailerSendWebhookVerifier( + options: SignedBodyWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * Makes the verifier of Mailtrap's sending webhooks: the HMAC-SHA256 of the + * body, in hexadecimal in `Mailtrap-Signature`; a body of JSON or JSON Lines. + * The category and the `tag_` variables are the tags, the other custom + * variables the metadata, as `./mailtrap` sends them. + * + * @throws {TypeError} If the secret is missing. + */ +export declare function createMailtrapWebhookVerifier( + options: SignedBodyWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * The options of {@link createBrevoWebhookVerifier}. + */ +export interface BrevoWebhookVerifierOptions { + /** The token the webhook is configured with. */ + readonly token: string; + /** + * The header field that carries it: `authorization`, as `Bearer ` + * for the `auth` token of the webhook, or a custom header with the token + * as its value. + * @default "authorization" + */ + readonly header?: string; + /** + * The longest body taken, in bytes. + * @default 5242880 + */ + readonly maxBodyBytes?: number; +} + +/** + * Makes the verifier of Brevo's transactional webhooks, which carry the token + * they are configured with, as Brevo signs nothing; compared in constant + * time. The metadata is read from `X-Mailin-custom`, as `./brevo` sends it. + * + * @throws {TypeError} If the token is missing. + */ +export declare function createBrevoWebhookVerifier( + options: BrevoWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * Makes the verifier of Mailjet's Event API, whose callback URL carries Basic + * credentials, as Mailjet signs nothing. The message id is the MessageUUID + * of the receipt; the custom campaign is the tag, the event payload the + * metadata. + * + * @throws {TypeError} If a credential is missing. + */ +export declare function createMailjetWebhookVerifier( + options: BasicWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * Makes the verifier of SparkPost's webhooks, configured with Basic + * authentication, as SparkPost signs nothing. The message id is that of the + * transmission, the receipt's; the tags and metadata are the recipient's. + * + * @throws {TypeError} If a credential is missing. + */ +export declare function createSparkPostWebhookVerifier( + options: BasicWebhookVerifierOptions, +): WebhookVerifier>; diff --git a/src/webhooks/index.js b/src/webhooks/index.js index 22343a2..8a8eff3 100644 --- a/src/webhooks/index.js +++ b/src/webhooks/index.js @@ -1,18 +1,28 @@ // Not ported: the entry of ./webhooks, the verifiers of the providers' events. const { WebhookVerificationError, createMemoryReplayStore } = require('./request.js'); +const { createBrevoWebhookVerifier } = require('./brevo.js'); +const { createMailerSendWebhookVerifier } = require('./mailersend.js'); const { createMailgunWebhookVerifier } = require('./mailgun.js'); +const { createMailjetWebhookVerifier } = require('./mailjet.js'); +const { createMailtrapWebhookVerifier } = require('./mailtrap.js'); const { createPostmarkWebhookVerifier } = require('./postmark.js'); const { createResendWebhookVerifier } = require('./resend.js'); const { createSendGridWebhookVerifier } = require('./sendgrid.js'); const { createSesWebhookVerifier } = require('./ses.js'); +const { createSparkPostWebhookVerifier } = require('./sparkpost.js'); module.exports = { WebhookVerificationError, + createBrevoWebhookVerifier, + createMailerSendWebhookVerifier, createMailgunWebhookVerifier, + createMailjetWebhookVerifier, + createMailtrapWebhookVerifier, createMemoryReplayStore, createPostmarkWebhookVerifier, createResendWebhookVerifier, createSendGridWebhookVerifier, createSesWebhookVerifier, + createSparkPostWebhookVerifier, }; diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js new file mode 100644 index 0000000..8fe19a1 --- /dev/null +++ b/src/webhooks/mailersend.js @@ -0,0 +1,124 @@ +// Not ported: the events of MailerSend's webhooks, signed with an HMAC of +// the raw body (https://developers.mailersend.com/api/v1/webhooks.html). +// sently and unemail have no MailerSend webhook. + +const { + WebhookVerificationError, + bodyBytesOf, + checkBodyHmac, + checkReplay, + createEvent, + dateOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, + stringsOf, +} = require('./request.js'); + +// MailerSend signs no time: an event id is kept a day and an hour, past the +// retries of a delivery. +const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; +const TYPES = new Map([ + ['activity.sent', 'accepted'], + ['activity.delivered', 'delivered'], + ['activity.deferred', 'deferred'], + ['activity.soft_bounced', 'bounced'], + ['activity.hard_bounced', 'bounced'], + ['activity.suppressed', 'rejected'], + ['activity.spam_complaint', 'complained'], + ['activity.unsubscribed', 'unsubscribed'], + ['activity.opened', 'opened'], + ['activity.opened_unique', 'opened'], + ['activity.clicked', 'clicked'], + ['activity.clicked_unique', 'clicked'], +]); +const METADATA_HEADER = /^x-metadata-(.+)$/i; + +// The metadata `./mailersend` sends as `X-Metadata-` header fields, when +// the event carries the headers of the email. +const metadataOf = (headers) => { + const metadata = Object.create(null); + const add = (name, value) => { + const match = typeof name === 'string' ? METADATA_HEADER.exec(name) : null; + if (match !== null && typeof value === 'string') metadata[match[1]] = value; + }; + if (Array.isArray(headers)) { + for (const header of headers) add(header?.name, header?.value); + } else if (isRecord(headers)) { + for (const name in headers) add(name, headers[name]); + } + return metadata; +}; + +// MailerSend has written its activity both flat (`data.recipient`) and nested +// (`data.email.recipient.email`); both are read. +const eventOf = (payload) => { + const data = isRecord(payload.data) ? payload.data : {}; + const email = isRecord(data.email) ? data.email : {}; + let meta = {}; + if (isRecord(data.meta)) meta = data.meta; + else if (isRecord(data.morph)) meta = data.morph; + const type = TYPES.get(payload.type) ?? 'unknown'; + let bounce; + if (type === 'bounced') { + bounce = { kind: payload.type === 'activity.hard_bounced' ? 'hard' : 'soft' }; + if (meta.bounce_code !== undefined && meta.bounce_code !== null) { + bounce.code = String(meta.bounce_code); + } + const diagnostic = stringOf(meta.bounce_reason) ?? stringOf(meta.reason); + if (diagnostic !== undefined) bounce.diagnostic = diagnostic; + } + const recipient = + stringOf(data.recipient) ?? + (isRecord(email.recipient) ? stringOf(email.recipient.email) : undefined); + const messageId = + stringOf(data.message_id) ?? (isRecord(email.message) ? stringOf(email.message.id) : undefined); + return createEvent({ + provider: 'mailersend', + type, + id: `${stringOf(data.id) ?? stringOf(payload.created_at)}:${payload.type}`, + messageId, + recipient, + occurredAt: dateOf(data.created_at ?? payload.created_at), + bounce, + reason: stringOf(meta.reason) ?? stringOf(meta.bounce_reason), + url: stringOf(meta.url), + tags: stringsOf(data.tags ?? email.tags), + metadata: metadataOf(email.headers ?? data.headers), + raw: payload, + }); +}; + +/** + * Makes the verifier of MailerSend's webhooks: the HMAC-SHA256 of the body, + * in hexadecimal in `Signature`, with the signing secret of the webhook. + * MailerSend signs no time, so an event is taken once by its id, kept a day + * and an hour. + * + * @param options `secret`; `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createMailerSendWebhookVerifier = (options = {}) => { + if (typeof options.secret !== 'string' || options.secret === '') { + throw new TypeError('secret is the signing secret of the MailerSend webhook.'); + } + const secret = options.secret; + const common = resolveCommonOptions(options, REPLAY_TTL_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBodyHmac(request, bytes, 'signature', secret, 'MailerSend'); + const payload = parseJsonOf(bytes); + if (!isRecord(payload) || typeof payload.type !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not a MailerSend event.'); + } + const event = eventOf(payload); + await checkReplay(common, `mailersend:${event.id}`, REPLAY_TTL_MS); + return { events: [event] }; + }, + }; +}; + +module.exports = { createMailerSendWebhookVerifier }; diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js index 903de16..bd44417 100644 --- a/src/webhooks/mailgun.js +++ b/src/webhooks/mailgun.js @@ -21,6 +21,8 @@ const { parseJsonOf, resolveCommonOptions, stringOf, + stringRecordOf, + stringsOf, } = require('./request.js'); const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; @@ -35,28 +37,6 @@ const TYPES = new Map([ ]); const HEX = /^[0-9a-f]+$/i; -const tagsOf = (list) => { - const tags = []; - if (!Array.isArray(list)) return tags; - for (const tag of list) { - if (typeof tag === 'string') tags.push(tag); - } - return tags; -}; - -const metadataOf = (variables) => { - const metadata = Object.create(null); - if (!isRecord(variables)) return metadata; - for (const name in variables) { - const value = variables[name]; - if (typeof value === 'string') metadata[name] = value; - else if (typeof value === 'number' || typeof value === 'boolean') { - metadata[name] = String(value); - } - } - return metadata; -}; - // What a failure is: a suppression or a rejection Mailgun made before any // attempt, a temporary failure it retries, or a bounce. const failureOf = (data) => { @@ -88,8 +68,8 @@ const eventOf = (data, token) => { bounce: failure?.bounce, reason: failure?.reason ?? (isRecord(data.reject) ? stringOf(data.reject.reason) : undefined), url: stringOf(data.url), - tags: tagsOf(data.tags), - metadata: metadataOf(data['user-variables']), + tags: stringsOf(data.tags), + metadata: stringRecordOf(data['user-variables']), raw: data, }); }; diff --git a/src/webhooks/mailjet.js b/src/webhooks/mailjet.js new file mode 100644 index 0000000..70ae72c --- /dev/null +++ b/src/webhooks/mailjet.js @@ -0,0 +1,89 @@ +// Not ported: the events of Mailjet's Event API, whose URL carries Basic +// credentials (https://dev.mailjet.com/email/guides/webhooks/). sently and +// unemail have no Mailjet webhook. + +const { + WebhookVerificationError, + basicCredentialsOf, + bodyBytesOf, + checkBasicCredentials, + createEvent, + dateOf, + isRecord, + jsonRecordOf, + parseJsonOf, + resolveCommonOptions, + stringOf, +} = require('./request.js'); + +const TYPES = new Map([ + ['sent', 'delivered'], + ['bounce', 'bounced'], + ['blocked', 'rejected'], + ['spam', 'complained'], + ['unsub', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], +]); + +const eventOf = (record) => { + const type = TYPES.get(record.event) ?? 'unknown'; + let bounce; + if (type === 'bounced') { + bounce = { kind: record.hard_bounce === true ? 'hard' : 'soft' }; + if (stringOf(record.error) !== undefined) bounce.diagnostic = record.error; + } + // The MessageUUID of the send's answer, the receipt's id. + const messageId = + stringOf(record.Message_GUID) ?? + (record.MessageID === undefined ? undefined : String(record.MessageID)); + const recipient = stringOf(record.email); + const customCampaign = stringOf(record.customcampaign); + return createEvent({ + provider: 'mailjet', + type, + id: `${record.event}:${messageId}:${recipient}:${record.time}`, + messageId, + recipient, + occurredAt: dateOf(record.time), + bounce, + reason: stringOf(record.error) ?? stringOf(record.source), + url: stringOf(record.url), + tags: customCampaign === undefined ? [] : [customCampaign], + metadata: jsonRecordOf(record.Payload), + raw: record, + }); +}; + +/** + * Makes the verifier of Mailjet's Event API, whose callback URL carries Basic + * credentials (`https://user:password@…`), as Mailjet signs nothing. The + * message id is the MessageUUID of the receipt; the custom campaign is the + * tag, the event payload the metadata, as `./mailjet` sends them. + * + * @param options `username`, `password` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If a credential is missing. + */ +const createMailjetWebhookVerifier = (options = {}) => { + const expected = basicCredentialsOf(options, 'Mailjet'); + const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBasicCredentials(request, expected); + const payload = parseJsonOf(bytes); + const records = Array.isArray(payload) ? payload : [payload]; + const events = []; + for (const record of records) { + if (isRecord(record)) events.push(eventOf(record)); + } + if (events.length === 0) { + throw new WebhookVerificationError('payload', 'The body is not a Mailjet event.'); + } + return { events }; + }, + }; +}; + +module.exports = { createMailjetWebhookVerifier }; diff --git a/src/webhooks/mailtrap.js b/src/webhooks/mailtrap.js new file mode 100644 index 0000000..d46857c --- /dev/null +++ b/src/webhooks/mailtrap.js @@ -0,0 +1,137 @@ +// Not ported: the events of Mailtrap's sending webhooks, signed with an HMAC +// of the raw body (https://docs.mailtrap.io/email-api-smtp/advanced/webhooks). +// sently and unemail have no Mailtrap webhook. + +const { + WebhookVerificationError, + bodyBytesOf, + checkBodyHmac, + checkReplay, + createEvent, + dateOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, + stringRecordOf, +} = require('./request.js'); + +// Mailtrap signs no time and retries for up to a day: an event id is kept a +// day and an hour. +const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; +const TYPES = new Map([ + ['delivery', 'delivered'], + ['soft bounce', 'bounced'], + ['bounce', 'bounced'], + ['suspension', 'rejected'], + ['reject', 'rejected'], + ['spam', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], +]); +const TAG_VARIABLE = /^tag_/; + +// The records of a body: `{ events: [...] }`, or JSON Lines, one event a line. +const recordsOf = (bytes) => { + let payload; + try { + payload = parseJsonOf(bytes); + } catch { + payload = undefined; + } + if (isRecord(payload)) return Array.isArray(payload.events) ? payload.events : [payload]; + if (payload !== undefined) { + throw new WebhookVerificationError('payload', 'The body is not a list of Mailtrap events.'); + } + const records = []; + for (const line of Buffer.from(bytes).toString('utf8').split('\n')) { + if (line.trim() !== '') records.push(parseJsonOf(Buffer.from(line, 'utf8'))); + } + return records; +}; + +// As `./mailtrap` sends them: the first tag is the category, the others +// `tag_` variables, and the other variables the metadata. +const tagsAndMetadataOf = (record) => { + const tags = []; + if (stringOf(record.category) !== undefined) tags.push(record.category); + const variables = stringRecordOf(record.custom_variables); + const metadata = Object.create(null); + for (const name in variables) { + if (TAG_VARIABLE.test(name)) tags.push(variables[name]); + else metadata[name] = variables[name]; + } + return { tags, metadata }; +}; + +const eventOf = (record) => { + const type = TYPES.get(record.event) ?? 'unknown'; + let bounce; + if (type === 'bounced') { + bounce = { kind: record.event === 'bounce' ? 'hard' : 'soft' }; + if (record.response_code !== undefined && record.response_code !== null) { + bounce.code = String(record.response_code); + } + if (stringOf(record.response) !== undefined) bounce.diagnostic = record.response; + } + const { tags, metadata } = tagsAndMetadataOf(record); + return createEvent({ + provider: 'mailtrap', + type, + id: stringOf(record.event_id) ?? `${record.message_id}:${record.event}:${record.email}`, + messageId: stringOf(record.message_id), + recipient: stringOf(record.email), + occurredAt: dateOf(record.timestamp), + bounce, + reason: stringOf(record.bounce_category) ?? stringOf(record.reason), + url: stringOf(record.url), + tags, + metadata, + raw: record, + }); +}; + +/** + * Makes the verifier of Mailtrap's sending webhooks: the HMAC-SHA256 of the + * body, in hexadecimal in `Mailtrap-Signature`, with the signing secret of + * the webhook; a body of JSON or JSON Lines. Mailtrap signs no time, so each + * event is taken once by its `event_id`, kept a day and an hour. + * + * @param options `secret`; `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createMailtrapWebhookVerifier = (options = {}) => { + if (typeof options.secret !== 'string' || options.secret === '') { + throw new TypeError('secret is the signing secret of the Mailtrap webhook.'); + } + const secret = options.secret; + const common = resolveCommonOptions(options, REPLAY_TTL_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBodyHmac(request, bytes, 'mailtrap-signature', secret, 'Mailtrap'); + const events = []; + let replayed = 0; + for (const record of recordsOf(bytes)) { + if (!isRecord(record)) continue; + const event = eventOf(record); + try { + await checkReplay(common, `mailtrap:${event.id}`, REPLAY_TTL_MS); + } catch (error) { + if (error.reason !== 'replay') throw error; + replayed++; + continue; + } + events.push(event); + } + if (events.length === 0 && replayed > 0) { + throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + } + return { events }; + }, + }; +}; + +module.exports = { createMailtrapWebhookVerifier }; diff --git a/src/webhooks/postmark.js b/src/webhooks/postmark.js index 709f594..bd864fe 100644 --- a/src/webhooks/postmark.js +++ b/src/webhooks/postmark.js @@ -10,15 +10,16 @@ const { WebhookVerificationError, + basicCredentialsOf, bodyBytesOf, + checkBasicCredentials, createEvent, dateOf, - equalBytes, - headerOf, isRecord, parseJsonOf, resolveCommonOptions, stringOf, + stringRecordOf, } = require('./request.js'); // What a bounce of Postmark is, by its Type. @@ -49,29 +50,6 @@ const TYPES = new Map([ ['Click', 'clicked'], ]); -const credentialsOf = (options) => { - for (const name of ['username', 'password']) { - if (typeof options[name] !== 'string' || options[name] === '') { - throw new TypeError( - `${name} is required: Postmark signs nothing, so its webhook URL carries credentials.`, - ); - } - } - return Buffer.from(`${options.username}:${options.password}`, 'utf8'); -}; - -const checkCredentials = (request, expected) => { - const header = headerOf(request, 'authorization'); - const space = header === undefined ? -1 : header.indexOf(' '); - if (space < 0 || header.slice(0, space).toLowerCase() !== 'basic') { - throw new WebhookVerificationError('credentials', 'The request has no Basic credentials.'); - } - const given = Buffer.from(header.slice(space + 1).trim(), 'base64'); - if (!equalBytes(given, expected)) { - throw new WebhookVerificationError('credentials', 'The credentials do not match.'); - } -}; - const describeRecord = (record) => { if (record.RecordType === 'Bounce') { const known = BOUNCES.get(record.Type) ?? { type: 'bounced', kind: 'undetermined' }; @@ -97,15 +75,6 @@ const describeRecord = (record) => { }; }; -const metadataOf = (values) => { - const metadata = Object.create(null); - if (!isRecord(values)) return metadata; - for (const name in values) { - if (typeof values[name] === 'string') metadata[name] = values[name]; - } - return metadata; -}; - const eventOf = (record) => { const described = describeRecord(record); const recipient = stringOf(record.Recipient) ?? stringOf(record.Email); @@ -127,7 +96,7 @@ const eventOf = (record) => { reason: described.reason, url: stringOf(record.OriginalLink), tags: tag === undefined ? [] : [tag], - metadata: metadataOf(record.Metadata), + metadata: stringRecordOf(record.Metadata), raw: record, }); }; @@ -142,13 +111,13 @@ const eventOf = (record) => { * @throws {TypeError} If an option is invalid. */ const createPostmarkWebhookVerifier = (options = {}) => { - const expected = credentialsOf(options); + const expected = basicCredentialsOf(options, 'Postmark'); // No time or id to hold a request to: the credentials alone stand for it. const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); return { async verify(request) { const bytes = bodyBytesOf(request, common); - checkCredentials(request, expected); + checkBasicCredentials(request, expected); const record = parseJsonOf(bytes); if (!isRecord(record) || typeof record.RecordType !== 'string') { throw new WebhookVerificationError('payload', 'The body is not a Postmark event.'); diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 12fe1a1..98a8971 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -4,7 +4,7 @@ // shape of an event. sently parsed a payload apart from its check; here // nothing is read before it is verified. -const { timingSafeEqual } = require('node:crypto'); +const { createHmac, timingSafeEqual } = require('node:crypto'); const { checkOptionalFunction, checkPositiveNumber } = require('../internal/options.js'); const DEFAULT_MAX_BODY_BYTES = 5 * 1024 * 1024; @@ -152,10 +152,10 @@ const checkTime = (atMs, common) => { * * @internal */ -const checkReplay = async (common, id) => { +const checkReplay = async (common, id, ttlMs = common.toleranceMs * 2) => { if (common.replayStore === null) return; - // Kept for twice the window, so a request cannot come back in it. - if ((await common.replayStore.add(id, common.toleranceMs * 2)) !== true) { + // Kept for twice the window by default, so a request cannot come back in it. + if ((await common.replayStore.add(id, ttlMs)) !== true) { throw new WebhookVerificationError('replay', `The request ${id} was taken before.`); } }; @@ -213,6 +213,102 @@ const isRecord = (value) => typeof value === 'object' && value !== null && !Arra */ const stringOf = (value) => (typeof value === 'string' && value !== '' ? value : undefined); +const HEX = /^[0-9a-f]+$/i; + +/** + * Checks the HMAC-SHA256 of the raw body, in hexadecimal in a header field. + * + * @internal + */ +const checkBodyHmac = (request, bytes, header, secret, label) => { + const given = headerOf(request, header); + if (given === undefined || !HEX.test(given.trim())) { + throw new WebhookVerificationError('signature', `The request has no ${label} signature.`); + } + const expected = createHmac('sha256', secret).update(bytes).digest(); + if (!equalBytes(expected, Buffer.from(given.trim(), 'hex'))) { + throw new WebhookVerificationError('signature', `The ${label} signature does not match.`); + } +}; + +/** + * The credentials a webhook URL carries (`https://user:password@…`), for a + * provider that signs nothing. + * + * @internal + */ +const basicCredentialsOf = (options, label) => { + for (const name of ['username', 'password']) { + if (typeof options[name] !== 'string' || options[name] === '') { + throw new TypeError( + `${name} is required: ${label} signs nothing, so its webhook URL carries credentials.`, + ); + } + } + return Buffer.from(`${options.username}:${options.password}`, 'utf8'); +}; + +/** + * Checks the Basic credentials of a request in constant time. + * + * @internal + */ +const checkBasicCredentials = (request, expected) => { + const header = headerOf(request, 'authorization'); + const space = header === undefined ? -1 : header.indexOf(' '); + if (space < 0 || header.slice(0, space).toLowerCase() !== 'basic') { + throw new WebhookVerificationError('credentials', 'The request has no Basic credentials.'); + } + const given = Buffer.from(header.slice(space + 1).trim(), 'base64'); + if (!equalBytes(given, expected)) { + throw new WebhookVerificationError('credentials', 'The credentials do not match.'); + } +}; + +/** + * The string values of a record, numbers and booleans as text. + * + * @internal + */ +const stringRecordOf = (values) => { + const record = Object.create(null); + if (!isRecord(values)) return record; + for (const name in values) { + const value = values[name]; + if (typeof value === 'string') record[name] = value; + else if (typeof value === 'number' || typeof value === 'boolean') record[name] = String(value); + } + return record; +}; + +/** + * The strings of a list. + * + * @internal + */ +const stringsOf = (list) => { + const strings = []; + if (!Array.isArray(list)) return strings; + for (const item of list) { + if (typeof item === 'string' && item !== '') strings.push(item); + } + return strings; +}; + +/** + * A record of JSON text, such as metadata a provider keeps as one string. + * + * @internal + */ +const jsonRecordOf = (text) => { + if (typeof text !== 'string' || text === '') return Object.create(null); + try { + return stringRecordOf(JSON.parse(text)); + } catch { + return Object.create(null); + } +}; + const NUMBERED_TAG = /^tag(\d+)$/; /** @@ -240,7 +336,10 @@ const splitNamedTags = (named) => { module.exports = { WebhookVerificationError, + basicCredentialsOf, bodyBytesOf, + checkBasicCredentials, + checkBodyHmac, checkReplay, checkTime, createEvent, @@ -249,8 +348,11 @@ module.exports = { equalBytes, headerOf, isRecord, + jsonRecordOf, parseJsonOf, resolveCommonOptions, splitNamedTags, stringOf, + stringRecordOf, + stringsOf, }; diff --git a/src/webhooks/sparkpost.js b/src/webhooks/sparkpost.js new file mode 100644 index 0000000..94a93ab --- /dev/null +++ b/src/webhooks/sparkpost.js @@ -0,0 +1,119 @@ +// Not ported: the events of SparkPost's webhooks, which authenticate with +// Basic credentials (https://developers.sparkpost.com/api/webhooks/). +// sently and unemail have no SparkPost webhook. + +const { + WebhookVerificationError, + basicCredentialsOf, + bodyBytesOf, + checkBasicCredentials, + createEvent, + dateOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, + stringRecordOf, + stringsOf, +} = require('./request.js'); + +const TYPES = new Map([ + ['injection', 'accepted'], + ['delivery', 'delivered'], + ['delay', 'deferred'], + ['bounce', 'bounced'], + ['out_of_band', 'bounced'], + ['spam_complaint', 'complained'], + ['policy_rejection', 'rejected'], + ['generation_failure', 'rejected'], + ['generation_rejection', 'rejected'], + ['list_unsubscribe', 'unsubscribed'], + ['link_unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['initial_open', 'opened'], + ['amp_open', 'opened'], + ['amp_initial_open', 'opened'], + ['click', 'clicked'], + ['amp_click', 'clicked'], +]); +// SparkPost's bounce classes: 10 Invalid Recipient, 30 Generic Bounce (No +// RCPT) and 90 Unsubscribe are hard; 1 Undetermined; the others soft. +const HARD_CLASSES = new Set(['10', '30', '90']); + +const bounceOf = (event) => { + const bounceClass = event.bounce_class === undefined ? undefined : String(event.bounce_class); + let kind = 'soft'; + if (bounceClass === undefined || bounceClass === '1') kind = 'undetermined'; + else if (HARD_CLASSES.has(bounceClass)) kind = 'hard'; + const bounce = { kind }; + if (stringOf(event.error_code) !== undefined) bounce.code = event.error_code; + const diagnostic = stringOf(event.raw_reason) ?? stringOf(event.reason); + if (diagnostic !== undefined) bounce.diagnostic = diagnostic; + return bounce; +}; + +// The event inside a record: `{ msys: { message_event: {...} } }`, or a +// track, generation, unsubscribe or relay event. +const innerOf = (record) => { + const msys = record.msys; + if (!isRecord(msys)) return undefined; + for (const name in msys) { + if (isRecord(msys[name])) return msys[name]; + } + return undefined; +}; + +const eventOf = (inner, index) => { + const type = TYPES.get(inner.type) ?? 'unknown'; + return createEvent({ + provider: 'sparkpost', + type, + id: stringOf(inner.event_id) ?? `${inner.transmission_id}:${inner.type}:${index}`, + // The transmission is what a send of `./sparkpost` answers with. + messageId: stringOf(inner.transmission_id), + recipient: stringOf(inner.rcpt_to) ?? stringOf(inner.raw_rcpt_to), + occurredAt: dateOf(inner.timestamp), + bounce: type === 'bounced' ? bounceOf(inner) : undefined, + reason: stringOf(inner.reason) ?? stringOf(inner.raw_reason), + url: stringOf(inner.target_link_url), + tags: stringsOf(inner.rcpt_tags), + metadata: stringRecordOf(inner.rcpt_meta), + raw: inner, + }); +}; + +/** + * Makes the verifier of SparkPost's webhooks, configured with Basic + * authentication, as SparkPost signs nothing. The message id is the + * transmission's, the receipt's id; the tags and metadata are those of the + * recipient. + * + * @param options `username`, `password` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If a credential is missing. + */ +const createSparkPostWebhookVerifier = (options = {}) => { + const expected = basicCredentialsOf(options, 'SparkPost'); + const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBasicCredentials(request, expected); + const payload = parseJsonOf(bytes); + if (!Array.isArray(payload)) { + throw new WebhookVerificationError( + 'payload', + 'The body is not a list of SparkPost events.', + ); + } + const events = []; + for (let index = 0; index < payload.length; index++) { + const inner = isRecord(payload[index]) ? innerOf(payload[index]) : undefined; + if (inner !== undefined) events.push(eventOf(inner, index)); + } + return { events }; + }, + }; +}; + +module.exports = { createSparkPostWebhookVerifier }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index 31c086d..656ff3b 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -2,12 +2,17 @@ import { expectAssignable, expectError, expectType } from 'tsd'; import type { DeliveryEvent } from '../../index.js'; import { WebhookVerificationError, + createBrevoWebhookVerifier, + createMailerSendWebhookVerifier, createMailgunWebhookVerifier, + createMailjetWebhookVerifier, + createMailtrapWebhookVerifier, createMemoryReplayStore, createPostmarkWebhookVerifier, createResendWebhookVerifier, createSendGridWebhookVerifier, createSesWebhookVerifier, + createSparkPostWebhookVerifier, type ReplayStore, type SesWebhookResult, type WebhookVerificationReason, @@ -45,3 +50,17 @@ expectAssignable[]>( ); expectError(createPostmarkWebhookVerifier({ username: 'u' })); expectError(createResendWebhookVerifier({})); + +const mailersend = createMailerSendWebhookVerifier({ secret: 's', replayStore: null }); +expectAssignable[]>( + (await mailersend.verify({ body: '' })).events, +); +createMailtrapWebhookVerifier({ secret: 's', maxBodyBytes: 1 }); +createBrevoWebhookVerifier({ token: 't', header: 'x-webhook-token' }); +createMailjetWebhookVerifier({ username: 'u', password: 'p' }); +const sparkpost = createSparkPostWebhookVerifier({ username: 'u', password: 'p' }); +expectAssignable[]>( + (await sparkpost.verify({ body: '' })).events, +); +expectError(createBrevoWebhookVerifier({})); +expectError(createMailtrapWebhookVerifier({ secret: 's', toleranceMs: 1 })); diff --git a/tests/unit/webhooks-mailgun.test.js b/tests/unit/webhooks-mailgun.test.js index d218103..695d326 100644 --- a/tests/unit/webhooks-mailgun.test.js +++ b/tests/unit/webhooks-mailgun.test.js @@ -173,7 +173,7 @@ test('Webhooks Postmark: credentials are required, and compared', async () => { recipient: 'bob@example.com', occurredAt: new Date('2027-01-15T08:00:00Z'), tags: ['welcome'], - metadata: { orderId: '42' }, + metadata: { orderId: '42', n: '1' }, raw: record, }, ); diff --git a/tests/unit/webhooks-more.test.js b/tests/unit/webhooks-more.test.js new file mode 100644 index 0000000..4d3472e --- /dev/null +++ b/tests/unit/webhooks-more.test.js @@ -0,0 +1,432 @@ +// Not ported: tests of the MailerSend, Mailtrap, Brevo, Mailjet and SparkPost +// verifiers of ./webhooks, against requests made as each provider documents. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createHmac } = require('node:crypto'); +const { + createBrevoWebhookVerifier, + createMailerSendWebhookVerifier, + createMailjetWebhookVerifier, + createMailtrapWebhookVerifier, + createSparkPostWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; +const hmac = (secret, body) => createHmac('sha256', secret).update(body).digest('hex'); +const basic = (user, password) => `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; +const plain = (event) => ({ ...event, metadata: { ...event.metadata } }); + +test('Webhooks MailerSend: the HMAC of the body, an event taken once', async () => { + const verifier = createMailerSendWebhookVerifier({ secret: 'ms-secret' }); + const payload = { + type: 'activity.hard_bounced', + created_at: '2027-01-15T08:00:00.000000Z', + data: { + id: 'act-1', + message_id: 'msg-1', + recipient: 'bob@example.com', + tags: ['orders'], + created_at: '2027-01-15T07:59:00.000000Z', + meta: { bounce_code: 550, bounce_reason: 'Mailbox not found', bounce_type: 'hard' }, + headers: [ + { name: 'X-Metadata-orderId', value: '42' }, + { name: 'X-Other', value: 'x' }, + ], + }, + }; + const body = JSON.stringify(payload); + const { events } = await verifier.verify({ + body, + headers: { Signature: hmac('ms-secret', body) }, + }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'mailersend', + type: 'bounced', + id: 'act-1:activity.hard_bounced', + messageId: 'msg-1', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-15T07:59:00.000Z'), + bounce: { kind: 'hard', code: '550', diagnostic: 'Mailbox not found' }, + reason: 'Mailbox not found', + tags: ['orders'], + metadata: { orderId: '42' }, + raw: payload, + }); + await rejects( + verifier.verify({ body, headers: { Signature: hmac('ms-secret', body) } }), + 'replay', + ); + await rejects( + verifier.verify({ body, headers: { Signature: hmac('other', body) } }), + 'signature', + ); + await rejects(verifier.verify({ body, headers: {} }), 'signature'); + // The nested form of MailerSend's activity. + const nested = { + type: 'activity.clicked_unique', + created_at: '2027-01-15T08:00:00Z', + data: { + id: 'act-2', + email: { + message: { id: 'msg-2' }, + recipient: { email: 'carol@example.com' }, + tags: ['news'], + headers: { 'X-Metadata-tenant': 'acme' }, + }, + morph: { url: 'https://e.com/a' }, + }, + }; + const nestedBody = JSON.stringify(nested); + const clicked = await verifier.verify({ + body: nestedBody, + headers: { signature: hmac('ms-secret', nestedBody) }, + }); + assert.strictEqual(clicked.events[0].type, 'clicked'); + assert.strictEqual(clicked.events[0].messageId, 'msg-2'); + assert.strictEqual(clicked.events[0].recipient, 'carol@example.com'); + assert.strictEqual(clicked.events[0].url, 'https://e.com/a'); + assert.deepStrictEqual({ ...clicked.events[0].metadata }, { tenant: 'acme' }); + for (const [type, expected] of [ + ['activity.sent', 'accepted'], + ['activity.soft_bounced', 'bounced'], + ['activity.suppressed', 'rejected'], + ['activity.unsubscribed', 'unsubscribed'], + ['activity.spam_complaint', 'complained'], + ['activity.survey_opened', 'unknown'], + ]) { + const typed = JSON.stringify({ type, created_at: type, data: {} }); + const { events: typedEvents } = await verifier.verify({ + body: typed, + headers: { signature: hmac('ms-secret', typed) }, + }); + assert.strictEqual(typedEvents[0].type, expected, type); + } + const soft = JSON.stringify({ type: 'activity.soft_bounced', data: { id: 's' } }); + const softEvents = await verifier.verify({ + body: soft, + headers: { signature: hmac('ms-secret', soft) }, + }); + assert.deepStrictEqual(softEvents.events[0].bounce, { kind: 'soft' }); + const notEvent = JSON.stringify([1]); + await rejects( + verifier.verify({ body: notEvent, headers: { signature: hmac('ms-secret', notEvent) } }), + 'payload', + ); + assert.throws(() => createMailerSendWebhookVerifier({}), /secret/); +}); + +test('Webhooks Mailtrap: a signed batch, in JSON or JSON Lines, each event once', async () => { + const verifier = createMailtrapWebhookVerifier({ secret: 'mt-secret' }); + const delivery = { + event: 'delivery', + message_id: 'mt-1', + event_id: 'ev-1', + email: 'bob@example.com', + timestamp: 1_800_000_000, + category: 'orders', + custom_variables: { tag_eu: 'eu', orderId: '42', attempt: 2 }, + }; + const bounce = { + event: 'bounce', + message_id: 'mt-1', + event_id: 'ev-2', + email: 'carol@example.com', + response: '550 unknown', + response_code: 550, + bounce_category: 'invalid', + }; + const body = JSON.stringify({ events: [delivery, bounce] }); + const { events } = await verifier.verify({ + body, + headers: { 'Mailtrap-Signature': hmac('mt-secret', body) }, + }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'mailtrap', + type: 'delivered', + id: 'ev-1', + messageId: 'mt-1', + recipient: 'bob@example.com', + occurredAt: new Date(1_800_000_000_000), + tags: ['orders', 'eu'], + metadata: { orderId: '42', attempt: '2' }, + raw: delivery, + }); + assert.deepStrictEqual(events[1].bounce, { + kind: 'hard', + code: '550', + diagnostic: '550 unknown', + }); + assert.strictEqual(events[1].reason, 'invalid'); + await rejects( + verifier.verify({ body, headers: { 'Mailtrap-Signature': hmac('mt-secret', body) } }), + 'replay', + ); + const lines = [ + { event: 'soft bounce', event_id: 'ev-3', email: 'bob@example.com' }, + { event: 'click', event_id: 'ev-4', url: 'https://e.com/a' }, + { event: 'suspension', event_id: 'ev-5', reason: 'domain suspended' }, + { event: 'spam', event_id: 'ev-6' }, + { event: 'unsubscribe', event_id: 'ev-7' }, + { event: 'open', event_id: 'ev-8' }, + { event: 'reject', event_id: 'ev-9' }, + { event: 'later', message_id: 'mt-2', email: 'x@example.com' }, + { event: 'delivery', event_id: 'ev-1' }, + ]; + const jsonl = `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`; + const fromLines = await verifier.verify({ + body: jsonl, + headers: { 'mailtrap-signature': hmac('mt-secret', jsonl) }, + }); + assert.deepStrictEqual( + fromLines.events.map((event) => event.type), + [ + 'bounced', + 'clicked', + 'rejected', + 'complained', + 'unsubscribed', + 'opened', + 'rejected', + 'unknown', + ], + ); + assert.strictEqual(fromLines.events[0].bounce.kind, 'soft'); + assert.strictEqual(fromLines.events[7].id, 'mt-2:later:x@example.com'); + const single = JSON.stringify({ event: 'delivery', event_id: 'ev-single' }); + assert.strictEqual( + ( + await verifier.verify({ + body: single, + headers: { 'mailtrap-signature': hmac('mt-secret', single) }, + }) + ).events.length, + 1, + ); + const numbers = '[1, 2]'; + await rejects( + verifier.verify({ + body: numbers, + headers: { 'mailtrap-signature': hmac('mt-secret', numbers) }, + }), + 'payload', + ); + const broken = '{"event":\n"x"'; + await rejects( + verifier.verify({ body: broken, headers: { 'mailtrap-signature': hmac('mt-secret', broken) } }), + 'payload', + ); + const scalars = '1\n2\n'; + assert.deepStrictEqual( + ( + await verifier.verify({ + body: scalars, + headers: { 'mailtrap-signature': hmac('mt-secret', scalars) }, + }) + ).events, + [], + ); + assert.throws(() => createMailtrapWebhookVerifier({}), /secret/); +}); + +test('Webhooks Brevo: the configured token is required', async () => { + const verifier = createBrevoWebhookVerifier({ token: 'brevo-token' }); + const record = { + event: 'hard_bounce', + email: 'bob@example.com', + 'message-id': '<1@smtp-relay.mailin.fr>', + ts_event: 1_800_000_000, + reason: 'unknown user', + tags: ['orders'], + 'X-Mailin-custom': '{"orderId":"42"}', + }; + const body = JSON.stringify(record); + // Changed in the port: sently parsed any request. + const { events } = await verifier.verify({ + body, + headers: { Authorization: 'Bearer brevo-token' }, + }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'brevo', + type: 'bounced', + id: 'hard_bounce:<1@smtp-relay.mailin.fr>:bob@example.com:1800000000', + messageId: '<1@smtp-relay.mailin.fr>', + recipient: 'bob@example.com', + occurredAt: new Date(1_800_000_000_000), + bounce: { kind: 'hard', diagnostic: 'unknown user' }, + reason: 'unknown user', + tags: ['orders'], + metadata: { orderId: '42' }, + raw: record, + }); + await rejects( + verifier.verify({ body, headers: { Authorization: 'Bearer other' } }), + 'credentials', + ); + await rejects(verifier.verify({ body, headers: {} }), 'credentials'); + const custom = createBrevoWebhookVerifier({ token: 't0k', header: 'X-Webhook-Token' }); + const batch = JSON.stringify([ + { event: 'softBounce', email: 'a@example.com' }, + // Changed in the port: sently took these for a complaint and bounces. + { event: 'unsubscribed', email: 'a@example.com' }, + { event: 'blocked', email: 'a@example.com' }, + { event: 'invalid_email', email: 'a@example.com' }, + { event: 'click', link: 'https://e.com/a', tag: 'single' }, + { event: 'uniqueOpened', 'X-Mailin-custom': 'not json' }, + { event: 'request' }, + { event: 'mystery' }, + 'skipped', + ]); + const many = await custom.verify({ body: batch, headers: { 'x-webhook-token': 't0k' } }); + assert.deepStrictEqual( + many.events.map((event) => event.type), + ['bounced', 'unsubscribed', 'rejected', 'rejected', 'clicked', 'opened', 'accepted', 'unknown'], + ); + assert.strictEqual(many.events[0].bounce.kind, 'soft'); + assert.deepStrictEqual(many.events[4].tags, ['single']); + assert.strictEqual(many.events[4].url, 'https://e.com/a'); + assert.deepStrictEqual({ ...many.events[5].metadata }, {}); + await rejects(custom.verify({ body: '"x"', headers: { 'x-webhook-token': 't0k' } }), 'payload'); + assert.throws(() => createBrevoWebhookVerifier({}), /token is required/); + assert.throws(() => createBrevoWebhookVerifier({ token: 't', header: '' }), /header/); +}); + +test('Webhooks Mailjet: Basic credentials are required; the MessageUUID is the id', async () => { + const verifier = createMailjetWebhookVerifier({ username: 'mj', password: 'pw' }); + const record = { + event: 'bounce', + time: 1_800_000_000, + MessageID: 1234, + Message_GUID: 'uuid-1', + email: 'bob@example.com', + customcampaign: 'autumn', + Payload: '{"orderId":"42"}', + hard_bounce: true, + error: 'user unknown', + }; + const body = JSON.stringify(record); + const { events } = await verifier.verify({ body, headers: { authorization: basic('mj', 'pw') } }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'mailjet', + type: 'bounced', + id: 'bounce:uuid-1:bob@example.com:1800000000', + messageId: 'uuid-1', + recipient: 'bob@example.com', + occurredAt: new Date(1_800_000_000_000), + bounce: { kind: 'hard', diagnostic: 'user unknown' }, + reason: 'user unknown', + tags: ['autumn'], + metadata: { orderId: '42' }, + raw: record, + }); + await rejects( + verifier.verify({ body, headers: { authorization: basic('mj', 'no') } }), + 'credentials', + ); + const batch = JSON.stringify([ + { event: 'sent', MessageID: 7 }, + { event: 'bounce', hard_bounce: false }, + { event: 'blocked', error: 'preblocked' }, + { event: 'spam', source: 'JMRPP' }, + { event: 'unsub' }, + { event: 'open' }, + { event: 'click', url: 'https://e.com/a' }, + { event: 'other' }, + ]); + const many = await verifier.verify({ + body: batch, + headers: { authorization: basic('mj', 'pw') }, + }); + assert.deepStrictEqual( + many.events.map((event) => event.type), + [ + 'delivered', + 'bounced', + 'rejected', + 'complained', + 'unsubscribed', + 'opened', + 'clicked', + 'unknown', + ], + ); + assert.strictEqual(many.events[0].messageId, '7'); + assert.deepStrictEqual(many.events[1].bounce, { kind: 'soft' }); + assert.strictEqual(many.events[3].reason, 'JMRPP'); + await rejects( + verifier.verify({ body: '[]', headers: { authorization: basic('mj', 'pw') } }), + 'payload', + ); + assert.throws(() => createMailjetWebhookVerifier({ username: 'mj' }), /password is required/); +}); + +test('Webhooks SparkPost: Basic credentials are required; the transmission is the message', async () => { + const verifier = createSparkPostWebhookVerifier({ username: 'sp', password: 'pw' }); + const bounce = { + type: 'bounce', + event_id: 'ev-1', + transmission_id: 'tx-1', + rcpt_to: 'bob@example.com', + timestamp: '1800000000', + bounce_class: '10', + error_code: '550', + raw_reason: '550 5.1.1 unknown', + reason: '550 5.1.1 unknown', + rcpt_tags: ['orders'], + rcpt_meta: { orderId: '42' }, + }; + const body = JSON.stringify([{ msys: { message_event: bounce } }, { msys: {} }, 'x']); + const { events } = await verifier.verify({ body, headers: { Authorization: basic('sp', 'pw') } }); + assert.strictEqual(events.length, 1); + assert.deepStrictEqual(plain(events[0]), { + provider: 'sparkpost', + type: 'bounced', + id: 'ev-1', + messageId: 'tx-1', + recipient: 'bob@example.com', + occurredAt: new Date(1_800_000_000_000), + bounce: { kind: 'hard', code: '550', diagnostic: '550 5.1.1 unknown' }, + reason: '550 5.1.1 unknown', + tags: ['orders'], + metadata: { orderId: '42' }, + raw: bounce, + }); + await rejects(verifier.verify({ body, headers: {} }), 'credentials'); + const kinds = [ + [{ type: 'bounce', bounce_class: 21 }, 'bounced', 'soft'], + [{ type: 'out_of_band', bounce_class: '1' }, 'bounced', 'undetermined'], + [{ type: 'bounce' }, 'bounced', 'undetermined'], + [{ type: 'injection' }, 'accepted', undefined], + [{ type: 'delivery' }, 'delivered', undefined], + [{ type: 'delay' }, 'deferred', undefined], + [{ type: 'policy_rejection' }, 'rejected', undefined], + [{ type: 'spam_complaint' }, 'complained', undefined], + [{ type: 'link_unsubscribe' }, 'unsubscribed', undefined], + [{ type: 'initial_open' }, 'opened', undefined], + [{ type: 'click', target_link_url: 'https://e.com/a' }, 'clicked', undefined], + [{ type: 'sms_status' }, 'unknown', undefined], + ]; + const batch = []; + for (const [inner] of kinds) batch.push({ msys: { track_event: inner } }); + const many = await verifier.verify({ + body: JSON.stringify(batch), + headers: { Authorization: basic('sp', 'pw') }, + }); + for (let i = 0; i < kinds.length; i++) { + assert.strictEqual(many.events[i].type, kinds[i][1], kinds[i][0].type); + assert.strictEqual(many.events[i].bounce?.kind, kinds[i][2], kinds[i][0].type); + } + assert.strictEqual(many.events[3].id, 'undefined:injection:3'); + assert.strictEqual(many.events[10].url, 'https://e.com/a'); + await rejects( + verifier.verify({ body: '{}', headers: { Authorization: basic('sp', 'pw') } }), + 'payload', + ); + assert.throws(() => createSparkPostWebhookVerifier({}), /username is required/); +}); From dda1c7d9075156c10b5fdeb22dada3a065485c52 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 23:31:15 +0300 Subject: [PATCH 046/143] feat(webhooks): verified events of Lettermint, Maileroo, SMTP2GO, Azure and Scaleway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each verified as its documentation says: Lettermint by the HMAC of the time and body (`t=…,v1=…`), within five minutes, once by its delivery id; Maileroo by the HMAC of the body, each event once; SMTP2GO by the Basic credentials of its webhook URL, its body JSON or a form; Azure by a token the Event Grid subscription sends as a secret delivery property, each event once, the validation of a new subscription handed back; Scaleway as SNS messages of Topics and Events, its signing certificate checked against the trust chain the application pins. The SNS protocol moves from the SES verifier to `sns.js`, which both use. ZeptoMail, whose signing is not documented for it, Plunk, which posts no events, and Cloudflare, whose events go to a Queue, have no verifier; the guide says so. The budget of `./webhooks`, which holds fifteen verifiers, is 12.5 KB. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 5 + docs/api/exports.md | 2 +- docs/guide/webhooks.md | 79 ++++ scripts/size-budgets.json | 2 +- src/webhooks/azure.js | 133 ++++++ src/webhooks/brevo.js | 26 +- src/webhooks/index.d.ts | 131 ++++++ src/webhooks/index.js | 10 + src/webhooks/lettermint.js | 114 +++++ src/webhooks/maileroo.js | 106 +++++ src/webhooks/mailersend.js | 21 +- src/webhooks/request.js | 50 +++ src/webhooks/scaleway.js | 104 +++++ src/webhooks/ses.js | 185 +------- src/webhooks/smtp2go.js | 103 +++++ src/webhooks/sns.js | 215 ++++++++++ tests/fixtures/webhooks/scaleway-ca.pem | 19 + tests/fixtures/webhooks/scaleway-cert.pem | 19 + tests/fixtures/webhooks/scaleway-key.pem | 28 ++ tests/helpers/webhooks/sns.js | 24 +- tests/types/webhooks.test-d.ts | 18 + tests/unit/webhooks-others.test.js | 492 ++++++++++++++++++++++ 22 files changed, 1664 insertions(+), 222 deletions(-) create mode 100644 src/webhooks/azure.js create mode 100644 src/webhooks/lettermint.js create mode 100644 src/webhooks/maileroo.js create mode 100644 src/webhooks/scaleway.js create mode 100644 src/webhooks/smtp2go.js create mode 100644 src/webhooks/sns.js create mode 100644 tests/fixtures/webhooks/scaleway-ca.pem create mode 100644 tests/fixtures/webhooks/scaleway-cert.pem create mode 100644 tests/fixtures/webhooks/scaleway-key.pem create mode 100644 tests/unit/webhooks-others.test.js diff --git a/CLAUDE.md b/CLAUDE.md index cb0e059..da763c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -610,6 +610,11 @@ fails on upstream's behavior): MailerSend, Mailtrap, Mailjet and SparkPost, which sently and unemail lack, are verified from their documentation: an HMAC of the body (MailerSend, Mailtrap; no time is signed, so each event is taken once by its id for a day and an hour), Basic credentials (Mailjet, SparkPost). + So are Lettermint (HMAC of the time and body), Maileroo (HMAC of the body), SMTP2GO (Basic), + Azure (Event Grid, a token in a secret delivery property, the validation handed back) and + Scaleway (SNS messages of Topics and Events, the certificate checked against the trust chain + the application pins). ZeptoMail, whose signing is not documented, Plunk, which posts no + events, and Cloudflare, whose events go to a Queue, have none. ## Repository layout diff --git a/docs/api/exports.md b/docs/api/exports.md index 1ca6df6..a15a34c 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -44,7 +44,7 @@ The message model and the transport contract, ported from `@upyo/core`. | `@alexify/mailer/mime` | `composeMessage`, `MimeAttachmentReplayError` | [MIME](/messages/mime) | | `@alexify/mailer/attachments` | `fromPath`, `fromUrl`, `fromDataUri`, `fromStream`, `inlineDataUrls`, `isPublicUrl` | [Attachments](/messages/attachments#files-urls-data-uris-and-streams) | | `@alexify/mailer/render` | `renderContent`, `createReactEmailRenderer`, `htmlToText` | [Rendering templates](/messages/render) | -| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMailerSendWebhookVerifier`, `createMailtrapWebhookVerifier`, `createBrevoWebhookVerifier`, `createMailjetWebhookVerifier`, `createSparkPostWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | +| `@alexify/mailer/webhooks` | `createSesWebhookVerifier`, `createResendWebhookVerifier`, `createSendGridWebhookVerifier`, `createMailgunWebhookVerifier`, `createPostmarkWebhookVerifier`, `createMailerSendWebhookVerifier`, `createMailtrapWebhookVerifier`, `createBrevoWebhookVerifier`, `createMailjetWebhookVerifier`, `createSparkPostWebhookVerifier`, `createLettermintWebhookVerifier`, `createMailerooWebhookVerifier`, `createSmtp2goWebhookVerifier`, `createAzureWebhookVerifier`, `createScalewayWebhookVerifier`, `createMemoryReplayStore`, `WebhookVerificationError` | [Webhooks](/guide/webhooks) | | `@alexify/mailer/smtp` | `SmtpTransport` and its error classes (`SmtpResponseError`, `SmtpAuthError`, …), `isSmtpResponseProviderDetails`, `createHttpProxyConnector` | [SMTP](/transports/smtp) | | `@alexify/mailer/sendmail` | `SendmailTransport` | [Sendmail](/transports/sendmail) | | `@alexify/mailer/ses` | `SesTransport` | [Amazon SES](/transports/ses) | diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 6c1508b..97329fb 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -196,3 +196,82 @@ campaign the tag and its event payload the metadata; SparkPost's message id is the transmission's, and a bounce is hard for the classes 10, 30 and 90, undetermined for 1, soft for the others. As for Postmark, nothing holds a request to a window or takes it once. + +## Lettermint and Maileroo + +```js +const lettermint = createLettermintWebhookVerifier({ secret: process.env.LETTERMINT_WEBHOOK_SECRET }); +const maileroo = createMailerooWebhookVerifier({ secret: process.env.MAILEROO_WEBHOOK_SECRET }); +``` + +Lettermint signs the time and the body: `X-Lettermint-Signature: t=…,v1=…`, the +HMAC-SHA256 of `t.body` with the secret as it is given, `whsec_` and all; a +request is taken within five minutes and once by the `id` its retries keep. +Maileroo signs the body alone, in `x-maileroo-signature`, so each event is +taken once by its `event_id`, kept a day and an hour; its message id is the +reference of the message, the receipt's id, and an event names its recipients, +one event each. + +## SMTP2GO + +```js +// The webhook URL in SMTP2GO: https://hook:s3cret@example.com/webhooks/smtp2go +const smtp2go = createSmtp2goWebhookVerifier({ username: 'hook', password: 's3cret' }); +``` + +SMTP2GO signs nothing, so its webhook URL carries Basic credentials, which the +verifier requires. The body may be JSON or a form, as the webhook is set up. +The message id is the `email_id` of the receipt; the metadata comes back from +`X-Metadata-` fields when the webhook adds the headers of the email to its +payload. + +## Azure Communication Services + +```js +const azure = createAzureWebhookVerifier({ token: process.env.EVENT_GRID_TOKEN }); + +app.post('/webhooks/azure', express.raw({ type: '*/*' }), async (req, res) => { + const { events, validation } = await azure.verify({ body: req.body, headers: req.headers }); + if (validation) return res.json({ validationResponse: validation.validationCode }); + for (const event of events) await deliveries.record(event); + res.sendStatus(200); +}); +``` + +The email events of Azure come through an Event Grid subscription to the +Communication Services resource, in the Event Grid schema. Event Grid signs +nothing, so give the subscription a secret delivery property: `Authorization` +with `Bearer `, or a header of your own with `header`. Every +notification must carry it; each event is taken once by its id. The +validation of a new subscription carries no event and is handed back for the +application to answer. Delivery reports are `delivered`, `bounced`, +`rejected` (Suppressed) or `deferred` (Quarantined); engagement reports +`opened` or `clicked`. The message id is that of the send operation, the +receipt's. + +## Scaleway + +```js +const trustChain = fs.readFileSync('scaleway-sns-trust-chain.pem', 'utf8'); +const scaleway = createScalewayWebhookVerifier({ + topicArns: ['arn:scw:sns:fr-par:project-…:tem-events'], + trustChain, +}); +``` + +Scaleway's Topics and Events posts the events of Transactional Email as SNS +messages, verified as those of SES are, from a listed topic, within an hour and +once, but with a certificate on a Scaleway host that must have been issued by +Scaleway's SNS trust chain. Download that chain once, from +`https://messaging.s3.fr-par.scw.cloud/fr-par/sns/sns-trust-chain.pem`, and pass +it as `trustChain`: the verifier trusts no certificate the chain did not issue. +A subscription confirmation is handed back, as for SES. + +## Providers without a verifier + +- **ZeptoMail:** its documentation mentions a webhook authentication key + without describing for ZeptoMail how a request is signed, so there is no + verifier yet. +- **Plunk:** Plunk posts no events of the messages it sends. +- **Cloudflare:** Email Service publishes its events to a Cloudflare Queue, + which a Worker or an HTTP pull consumer reads, not to a webhook. diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index bd893c0..d6ad948 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4, "./render": 3, - "./webhooks": 10, + "./webhooks": 12.5, "./smtp": 23, "./sendmail": 10.5, "./jmap": 12.5, diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js new file mode 100644 index 0000000..09312b8 --- /dev/null +++ b/src/webhooks/azure.js @@ -0,0 +1,133 @@ +// Not ported: the email events of Azure Communication Services, delivered by +// Event Grid (https://learn.microsoft.com/azure/event-grid/communication-services-email-events). +// sently and unemail have no Azure webhook. + +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + createEvent, + dateOf, + headerOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, + tokenCheckOf, +} = require('./request.js'); + +// Event Grid retries a delivery for a day: an event id is kept a day and an +// hour. +const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; +const VALIDATION = 'Microsoft.EventGrid.SubscriptionValidationEvent'; +const DELIVERY = 'Microsoft.Communication.EmailDeliveryReportReceived'; +const ENGAGEMENT = 'Microsoft.Communication.EmailEngagementTrackingReportReceived'; +const DELIVERY_TYPES = new Map([ + ['Delivered', 'delivered'], + ['Bounced', 'bounced'], + ['Failed', 'bounced'], + ['FilteredSpam', 'bounced'], + ['Suppressed', 'rejected'], + ['Quarantined', 'deferred'], +]); + +const eventOf = (record) => { + const data = isRecord(record.data) ? record.data : {}; + let type = 'unknown'; + let at = record.eventTime; + let url; + if (record.eventType === DELIVERY) { + type = DELIVERY_TYPES.get(data.status) ?? 'unknown'; + at = data.deliveryAttemptTimeStamp ?? at; + } else if (record.eventType === ENGAGEMENT) { + type = data.engagementType === 'click' ? 'clicked' : 'opened'; + at = data.userActionTimeStamp ?? at; + if (type === 'clicked') url = stringOf(data.engagementContext); + } + const details = isRecord(data.deliveryStatusDetails) ? data.deliveryStatusDetails : {}; + const reason = stringOf(details.statusMessage); + let bounce; + if (type === 'bounced') { + bounce = { kind: 'undetermined' }; + if (reason !== undefined) bounce.diagnostic = reason; + } + return createEvent({ + provider: 'azure', + type, + id: stringOf(record.id), + // The id of the send operation, the receipt's. + messageId: stringOf(data.messageId), + recipient: stringOf(data.recipient), + occurredAt: dateOf(at), + bounce, + reason: reason ?? stringOf(data.status), + url, + raw: record, + }); +}; + +/** + * Makes the verifier of the email events of Azure Communication Services, + * delivered by an Event Grid subscription with the Event Grid schema. Event + * Grid signs nothing, so the subscription carries a token as a secret + * delivery property (`Authorization: Bearer `, or a custom `header`), + * required on every notification and compared in constant time; each event is + * taken once by its id. The validation of a new subscription is handed back, + * for the application to answer `{ validationResponse: validationCode }`. + * + * @param options `token`, `header`; `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, validation? }`. + * @throws {TypeError} If an option is invalid. + */ +const createAzureWebhookVerifier = (options = {}) => { + const checkToken = tokenCheckOf(options, 'Event Grid'); + const common = resolveCommonOptions(options, REPLAY_TTL_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + const kind = headerOf(request, 'aeg-event-type'); + // A validation carries no event, and answering it gives nothing away. + if (kind === 'SubscriptionValidation') { + const payload = parseJsonOf(bytes); + const first = Array.isArray(payload) ? payload[0] : undefined; + if (!isRecord(first) || first.eventType !== VALIDATION || !isRecord(first.data)) { + throw new WebhookVerificationError('payload', 'The body is not a validation event.'); + } + return { + events: [], + validation: { + validationCode: first.data.validationCode, + validationUrl: first.data.validationUrl, + }, + }; + } + checkToken(request); + const payload = parseJsonOf(bytes); + if (!Array.isArray(payload)) { + throw new WebhookVerificationError( + 'payload', + 'The body is not a list of Event Grid events.', + ); + } + const events = []; + let replayed = 0; + for (const record of payload) { + if (!isRecord(record) || typeof record.id !== 'string') continue; + try { + await checkReplay(common, `azure:${record.id}`, REPLAY_TTL_MS); + } catch (error) { + if (error.reason !== 'replay') throw error; + replayed++; + continue; + } + events.push(eventOf(record)); + } + if (events.length === 0 && replayed > 0) { + throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + } + return { events }; + }, + }; +}; + +module.exports = { createAzureWebhookVerifier }; diff --git a/src/webhooks/brevo.js b/src/webhooks/brevo.js index 19bd2e7..fedc366 100644 --- a/src/webhooks/brevo.js +++ b/src/webhooks/brevo.js @@ -13,14 +13,13 @@ const { bodyBytesOf, createEvent, dateOf, - equalBytes, - headerOf, isRecord, jsonRecordOf, parseJsonOf, resolveCommonOptions, stringOf, stringsOf, + tokenCheckOf, } = require('./request.js'); const TYPES = new Map([ @@ -49,27 +48,6 @@ const TYPES = new Map([ ]); const HARD = new Set(['hard_bounce', 'hardBounce']); -const tokenCheckOf = (options) => { - if (typeof options.token !== 'string' || options.token === '') { - throw new TypeError( - 'token is required: Brevo signs nothing, so its webhook carries a token (auth or a header).', - ); - } - const header = options.header ?? 'authorization'; - if (typeof header !== 'string' || header === '') throw new TypeError('header is a field name.'); - // The auth token of a webhook comes as `Authorization: Bearer `. - const expected = Buffer.from( - header.toLowerCase() === 'authorization' ? `Bearer ${options.token}` : options.token, - 'utf8', - ); - return (request) => { - const given = headerOf(request, header); - if (given === undefined || !equalBytes(Buffer.from(given.trim(), 'utf8'), expected)) { - throw new WebhookVerificationError('credentials', 'The webhook token does not match.'); - } - }; -}; - const eventOf = (record) => { const type = TYPES.get(record.event) ?? 'unknown'; const reason = stringOf(record.reason); @@ -109,7 +87,7 @@ const eventOf = (record) => { * @throws {TypeError} If an option is invalid. */ const createBrevoWebhookVerifier = (options = {}) => { - const checkToken = tokenCheckOf(options); + const checkToken = tokenCheckOf(options, 'Brevo'); const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); return { async verify(request) { diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index e47e0c1..8cf8299 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -334,3 +334,134 @@ export declare function createMailjetWebhookVerifier( export declare function createSparkPostWebhookVerifier( options: BasicWebhookVerifierOptions, ): WebhookVerifier>; + +/** + * The options of {@link createScalewayWebhookVerifier}. + */ +export interface ScalewayWebhookVerifierOptions extends WebhookVerifierOptions { + /** The topics of Topics and Events taken; a message of another is refused. */ + readonly topicArns: readonly string[]; + /** + * The PEM of Scaleway's SNS trust chain, which issued the certificate a + * message is signed with. + */ + readonly trustChain: string; + /** The fetch the signing certificates are read with. */ + readonly fetch?: typeof globalThis.fetch; + /** + * How far from now a message may have been published. + * @default 3600000 + */ + readonly toleranceMs?: number; +} + +/** + * What the Scaleway verifier reads: the events of a notification, or the + * confirmation of a subscription. + */ +export interface ScalewayWebhookResult extends WebhookResult<'scaleway'> { + readonly confirmation?: SnsConfirmation; +} + +/** + * Makes the verifier of the events of Scaleway Transactional Email, which + * Topics and Events posts as SNS messages: from a listed topic, signed with a + * certificate on a Scaleway host that the pinned trust chain issued, within an + * hour, taken once. The metadata comes back from the `X-Metadata-` + * fields of the email, as `./scaleway` sends it. + * + * @throws {TypeError} If an option is invalid. + */ +export declare function createScalewayWebhookVerifier( + options: ScalewayWebhookVerifierOptions, +): WebhookVerifier; + +/** + * The options of {@link createLettermintWebhookVerifier}. + */ +export interface LettermintWebhookVerifierOptions extends WebhookVerifierOptions { + /** The signing secret of the webhook, `whsec_…`, used as it is. */ + readonly secret: string; +} + +/** + * Makes the verifier of Lettermint's webhooks: the HMAC-SHA256 of the time + * and body in `X-Lettermint-Signature` (`t=…,v1=…`), within five minutes, its + * delivery id taken once. + * + * @throws {TypeError} If the secret is missing. + */ +export declare function createLettermintWebhookVerifier( + options: LettermintWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * Makes the verifier of Maileroo's webhooks: the HMAC-SHA256 of the body, in + * hexadecimal in `x-maileroo-signature`, each event taken once. The message + * id is the reference of the message, the receipt's. + * + * @throws {TypeError} If the secret is missing. + */ +export declare function createMailerooWebhookVerifier( + options: SignedBodyWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * Makes the verifier of SMTP2GO's webhooks, whose URL carries Basic + * credentials, as SMTP2GO signs nothing; a body of JSON or of a form. + * + * @throws {TypeError} If a credential is missing. + */ +export declare function createSmtp2goWebhookVerifier( + options: BasicWebhookVerifierOptions, +): WebhookVerifier>; + +/** + * The options of {@link createAzureWebhookVerifier}. + */ +export interface AzureWebhookVerifierOptions { + /** The token the Event Grid subscription sends as a secret delivery property. */ + readonly token: string; + /** + * The header field of the token: `authorization`, as `Bearer `, or a + * custom one with the token as its value. + * @default "authorization" + */ + readonly header?: string; + /** The ids already taken; an in-process store by default, `null` for none. */ + readonly replayStore?: ReplayStore | null; + /** + * The longest body taken, in bytes. + * @default 5242880 + */ + readonly maxBodyBytes?: number; +} + +/** + * The validation of a new Event Grid subscription, which the application + * answers with `{ validationResponse: validationCode }`. + */ +export interface EventGridValidation { + readonly validationCode: string; + readonly validationUrl?: string; +} + +/** + * What the Azure verifier reads: the events of a notification, or the + * validation of a subscription. + */ +export interface AzureWebhookResult extends WebhookResult<'azure'> { + readonly validation?: EventGridValidation; +} + +/** + * Makes the verifier of the email events of Azure Communication Services, + * delivered by Event Grid in its own schema: the delivery reports and the + * engagement reports. Event Grid signs nothing, so every notification must + * carry the token of the subscription; each event is taken once by its id. + * + * @throws {TypeError} If the token is missing. + */ +export declare function createAzureWebhookVerifier( + options: AzureWebhookVerifierOptions, +): WebhookVerifier; diff --git a/src/webhooks/index.js b/src/webhooks/index.js index 8a8eff3..a89f5ac 100644 --- a/src/webhooks/index.js +++ b/src/webhooks/index.js @@ -1,20 +1,28 @@ // Not ported: the entry of ./webhooks, the verifiers of the providers' events. const { WebhookVerificationError, createMemoryReplayStore } = require('./request.js'); +const { createAzureWebhookVerifier } = require('./azure.js'); const { createBrevoWebhookVerifier } = require('./brevo.js'); +const { createLettermintWebhookVerifier } = require('./lettermint.js'); +const { createMailerooWebhookVerifier } = require('./maileroo.js'); const { createMailerSendWebhookVerifier } = require('./mailersend.js'); const { createMailgunWebhookVerifier } = require('./mailgun.js'); const { createMailjetWebhookVerifier } = require('./mailjet.js'); const { createMailtrapWebhookVerifier } = require('./mailtrap.js'); const { createPostmarkWebhookVerifier } = require('./postmark.js'); const { createResendWebhookVerifier } = require('./resend.js'); +const { createScalewayWebhookVerifier } = require('./scaleway.js'); const { createSendGridWebhookVerifier } = require('./sendgrid.js'); const { createSesWebhookVerifier } = require('./ses.js'); +const { createSmtp2goWebhookVerifier } = require('./smtp2go.js'); const { createSparkPostWebhookVerifier } = require('./sparkpost.js'); module.exports = { WebhookVerificationError, + createAzureWebhookVerifier, createBrevoWebhookVerifier, + createLettermintWebhookVerifier, + createMailerooWebhookVerifier, createMailerSendWebhookVerifier, createMailgunWebhookVerifier, createMailjetWebhookVerifier, @@ -22,7 +30,9 @@ module.exports = { createMemoryReplayStore, createPostmarkWebhookVerifier, createResendWebhookVerifier, + createScalewayWebhookVerifier, createSendGridWebhookVerifier, createSesWebhookVerifier, + createSmtp2goWebhookVerifier, createSparkPostWebhookVerifier, }; diff --git a/src/webhooks/lettermint.js b/src/webhooks/lettermint.js new file mode 100644 index 0000000..033231c --- /dev/null +++ b/src/webhooks/lettermint.js @@ -0,0 +1,114 @@ +// Not ported: the events of Lettermint's webhooks, signed with an HMAC of +// their time and raw body (https://lettermint.co/docs/platform/webhooks/signing). +// sently and unemail have no Lettermint webhook. + +const { createHmac } = require('node:crypto'); +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + checkTime, + createEvent, + dateOf, + equalBytes, + headerOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + stringOf, + stringRecordOf, + stringsOf, +} = require('./request.js'); + +const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; +const TYPES = new Map([ + ['message.created', 'accepted'], + ['message.sent', 'accepted'], + ['message.delivered', 'delivered'], + ['message.hard_bounced', 'bounced'], + ['message.soft_bounced', 'bounced'], + ['message.spam_complaint', 'complained'], + ['message.failed', 'rejected'], + ['message.suppressed', 'rejected'], + ['message.policy_rejected', 'rejected'], + ['message.unsubscribed', 'unsubscribed'], + ['message.opened', 'opened'], + ['message.clicked', 'clicked'], +]); +const SIGNATURE = /^t=(\d+),v1=([0-9a-f]+)$/i; + +// `t=,v1=`: the HMAC-SHA256 of `.`, with the +// secret as it is given, `whsec_` and all. +const checkSignature = (request, bytes, secret, common) => { + const match = SIGNATURE.exec(headerOf(request, 'x-lettermint-signature')?.trim() ?? ''); + if (match === null) { + throw new WebhookVerificationError('signature', 'The request has no Lettermint signature.'); + } + const expected = createHmac('sha256', secret).update(`${match[1]}.`).update(bytes).digest(); + if (!equalBytes(expected, Buffer.from(match[2], 'hex'))) { + throw new WebhookVerificationError('signature', 'The Lettermint signature does not match.'); + } + checkTime(Number(match[1]) * 1000, common); +}; + +const bounceOf = (event, data) => { + const bounce = { kind: event === 'message.hard_bounced' ? 'hard' : 'soft' }; + const response = isRecord(data.response) ? data.response : {}; + const code = stringOf(response.enhanced_status_code) ?? response.status_code; + if (code !== undefined && code !== null) bounce.code = String(code); + if (stringOf(response.content) !== undefined) bounce.diagnostic = response.content; + return bounce; +}; + +const eventOf = (payload) => { + const data = isRecord(payload.data) ? payload.data : {}; + const type = TYPES.get(payload.event) ?? 'unknown'; + const tags = stringsOf(data.tags); + if (tags.length === 0 && stringOf(data.tag) !== undefined) tags.push(data.tag); + return createEvent({ + provider: 'lettermint', + type, + id: stringOf(payload.id), + messageId: stringOf(data.message_id), + recipient: stringOf(data.recipient), + occurredAt: dateOf(payload.timestamp), + bounce: type === 'bounced' ? bounceOf(payload.event, data) : undefined, + reason: stringOf(data.reason_code) ?? stringOf(data.reason), + url: stringOf(data.destination_url), + tags, + metadata: stringRecordOf(data.metadata), + raw: payload, + }); +}; + +/** + * Makes the verifier of Lettermint's webhooks: the HMAC-SHA256 of the time + * and body in `X-Lettermint-Signature` (`t=…,v1=…`), with the signing secret + * as it is given, within five minutes, its delivery id taken once. + * + * @param options `secret`; `now`, `toleranceMs`, `replayStore` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If an option is invalid. + */ +const createLettermintWebhookVerifier = (options = {}) => { + if (typeof options.secret !== 'string' || options.secret === '') { + throw new TypeError('secret is the signing secret of the Lettermint webhook.'); + } + const secret = options.secret; + const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkSignature(request, bytes, secret, common); + const payload = parseJsonOf(bytes); + if (!isRecord(payload) || typeof payload.id !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not a Lettermint event.'); + } + await checkReplay(common, `lettermint:${payload.id}`); + return { events: [eventOf(payload)] }; + }, + }; +}; + +module.exports = { createLettermintWebhookVerifier }; diff --git a/src/webhooks/maileroo.js b/src/webhooks/maileroo.js new file mode 100644 index 0000000..9b1a91f --- /dev/null +++ b/src/webhooks/maileroo.js @@ -0,0 +1,106 @@ +// Not ported: the events of Maileroo's webhooks, signed with an HMAC of the +// raw body (https://maileroo.com/docs/api-reference/emails/webhooks). +// sently and unemail have no Maileroo webhook. + +const { + WebhookVerificationError, + bodyBytesOf, + checkBodyHmac, + checkReplay, + createEvent, + dateOf, + isRecord, + parseJsonOf, + resolveCommonOptions, + splitNamedTags, + stringOf, +} = require('./request.js'); + +// Maileroo signs no time: an event id is kept a day and an hour. +const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; +const TYPES = new Map([ + ['accepted', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + ['failed', 'bounced'], + ['rejected', 'rejected'], + ['complained', 'complained'], + ['opened', 'opened'], + ['clicked', 'clicked'], +]); + +const recipientsOf = (to) => { + if (typeof to === 'string') return [to]; + const recipients = []; + if (!Array.isArray(to)) return recipients; + for (const address of to) { + if (typeof address === 'string') recipients.push(address); + } + return recipients; +}; + +const eventsOf = (payload) => { + const data = isRecord(payload.event_data) ? payload.event_data : {}; + const type = TYPES.get(payload.event_type) ?? 'unknown'; + const reason = stringOf(data.reason) ?? stringOf(data.reject_reason); + let bounce; + if (type === 'bounced' || type === 'deferred') { + bounce = { kind: type === 'bounced' ? 'hard' : 'soft' }; + if (reason !== undefined) bounce.diagnostic = reason; + } + const { tags, metadata } = splitNamedTags(payload.tags); + const recipients = recipientsOf(data.to); + const list = recipients.length > 0 ? recipients : [undefined]; + const events = []; + for (let index = 0; index < list.length; index++) { + events.push( + createEvent({ + provider: 'maileroo', + type, + id: `${payload.event_id}:${index}`, + // The reference of the message is the receipt's id. + messageId: stringOf(payload.message_reference_id), + recipient: list[index], + occurredAt: dateOf(payload.event_time), + bounce, + reason, + url: stringOf(data.original_url), + tags, + metadata, + raw: payload, + }), + ); + } + return events; +}; + +/** + * Makes the verifier of Maileroo's webhooks: the HMAC-SHA256 of the body, in + * hexadecimal in `x-maileroo-signature`. Maileroo signs no time, so an event + * is taken once by its id, kept a day and an hour. + * + * @param options `secret`; `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If the secret is missing. + */ +const createMailerooWebhookVerifier = (options = {}) => { + if (typeof options.secret !== 'string' || options.secret === '') { + throw new TypeError('secret is the signing secret of the Maileroo webhook.'); + } + const secret = options.secret; + const common = resolveCommonOptions(options, REPLAY_TTL_MS); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBodyHmac(request, bytes, 'x-maileroo-signature', secret, 'Maileroo'); + const payload = parseJsonOf(bytes); + if (!isRecord(payload) || typeof payload.event_id !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not a Maileroo event.'); + } + await checkReplay(common, `maileroo:${payload.event_id}`, REPLAY_TTL_MS); + return { events: eventsOf(payload) }; + }, + }; +}; + +module.exports = { createMailerooWebhookVerifier }; diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js index 8fe19a1..80a0480 100644 --- a/src/webhooks/mailersend.js +++ b/src/webhooks/mailersend.js @@ -10,6 +10,7 @@ const { createEvent, dateOf, isRecord, + metadataFromHeaders, parseJsonOf, resolveCommonOptions, stringOf, @@ -33,24 +34,6 @@ const TYPES = new Map([ ['activity.clicked', 'clicked'], ['activity.clicked_unique', 'clicked'], ]); -const METADATA_HEADER = /^x-metadata-(.+)$/i; - -// The metadata `./mailersend` sends as `X-Metadata-` header fields, when -// the event carries the headers of the email. -const metadataOf = (headers) => { - const metadata = Object.create(null); - const add = (name, value) => { - const match = typeof name === 'string' ? METADATA_HEADER.exec(name) : null; - if (match !== null && typeof value === 'string') metadata[match[1]] = value; - }; - if (Array.isArray(headers)) { - for (const header of headers) add(header?.name, header?.value); - } else if (isRecord(headers)) { - for (const name in headers) add(name, headers[name]); - } - return metadata; -}; - // MailerSend has written its activity both flat (`data.recipient`) and nested // (`data.email.recipient.email`); both are read. const eventOf = (payload) => { @@ -85,7 +68,7 @@ const eventOf = (payload) => { reason: stringOf(meta.reason) ?? stringOf(meta.bounce_reason), url: stringOf(meta.url), tags: stringsOf(data.tags ?? email.tags), - metadata: metadataOf(email.headers ?? data.headers), + metadata: metadataFromHeaders(email.headers ?? data.headers), raw: payload, }); }; diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 98a8971..bb404d9 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -309,6 +309,54 @@ const jsonRecordOf = (text) => { } }; +const METADATA_HEADER = /^x-metadata-(.+)$/i; + +/** + * The metadata the transports send as `X-Metadata-` header fields, from + * the headers an event carries: a record, or a list of `{ name | key, value }`. + * + * @internal + */ +const metadataFromHeaders = (headers) => { + const metadata = Object.create(null); + const add = (name, value) => { + const match = typeof name === 'string' ? METADATA_HEADER.exec(name) : null; + if (match !== null && typeof value === 'string') metadata[match[1]] = value; + }; + if (Array.isArray(headers)) { + for (const header of headers) add(header?.name ?? header?.key, header?.value); + } else if (isRecord(headers)) { + for (const name in headers) add(name, headers[name]); + } + return metadata; +}; + +/** + * Checks the token a webhook is configured with: `Authorization: Bearer + * `, or the value of a custom header field, in constant time. + * + * @internal + */ +const tokenCheckOf = (options, label) => { + if (typeof options.token !== 'string' || options.token === '') { + throw new TypeError( + `token is required: ${label} signs nothing, so its webhook carries a token.`, + ); + } + const header = options.header ?? 'authorization'; + if (typeof header !== 'string' || header === '') throw new TypeError('header is a field name.'); + const expected = Buffer.from( + header.toLowerCase() === 'authorization' ? `Bearer ${options.token}` : options.token, + 'utf8', + ); + return (request) => { + const given = headerOf(request, header); + if (given === undefined || !equalBytes(Buffer.from(given.trim(), 'utf8'), expected)) { + throw new WebhookVerificationError('credentials', 'The webhook token does not match.'); + } + }; +}; + const NUMBERED_TAG = /^tag(\d+)$/; /** @@ -349,10 +397,12 @@ module.exports = { headerOf, isRecord, jsonRecordOf, + metadataFromHeaders, parseJsonOf, resolveCommonOptions, splitNamedTags, stringOf, stringRecordOf, stringsOf, + tokenCheckOf, }; diff --git a/src/webhooks/scaleway.js b/src/webhooks/scaleway.js new file mode 100644 index 0000000..ae3a8e4 --- /dev/null +++ b/src/webhooks/scaleway.js @@ -0,0 +1,104 @@ +// Not ported: the events of Scaleway Transactional Email, which Scaleway's +// Topics and Events, an implementation of the SNS protocol, posts to a +// webhook (https://www.scaleway.com/en/docs/transactional-email/reference-content/webhook-events-payloads/). +// sently and unemail have no Scaleway webhook. + +const { X509Certificate } = require('node:crypto'); +const { createEvent, dateOf, metadataFromHeaders, stringOf } = require('./request.js'); +const { certificateError, createSnsVerifier, isHttpsUrlOf } = require('./sns.js'); + +const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; +const SCALEWAY_HOST = /(^|\.)(scw\.cloud|scaleway\.com)$/; +const PEM_CERTIFICATE = /-----BEGIN CERTIFICATE-----[^-]+-----END CERTIFICATE-----/g; +const TYPES = new Map([ + ['email_queued', 'accepted'], + ['email_delivered', 'delivered'], + ['email_deferred', 'deferred'], + ['email_dropped', 'bounced'], + ['email_mailbox_not_found', 'bounced'], + ['email_blocklisted', 'rejected'], + ['email_spam', 'complained'], +]); + +// The certificates of the trust chain the application pins, as Scaleway +// asks a certificate to be checked against it rather than trusted. +const trustChainOf = (pem) => { + if (typeof pem !== 'string') { + throw new TypeError("trustChain is the PEM of Scaleway's SNS trust chain."); + } + const authorities = []; + for (const match of pem.matchAll(PEM_CERTIFICATE)) { + authorities.push(new X509Certificate(match[0])); + } + if (authorities.length === 0) throw new TypeError('trustChain holds no certificate.'); + return authorities; +}; + +const eventOf = (record, snsMessageId) => { + const type = TYPES.get(record.type) ?? 'unknown'; + let bounce; + if (type === 'bounced' || type === 'deferred') { + bounce = { kind: type === 'bounced' ? 'hard' : 'soft' }; + if (record.email_response_code !== undefined && record.email_response_code !== null) { + bounce.code = String(record.email_response_code); + } + const diagnostic = stringOf(record.email_response_message) ?? stringOf(record.email_error); + if (diagnostic !== undefined) bounce.diagnostic = diagnostic; + } + return createEvent({ + provider: 'scaleway', + type, + id: stringOf(record.id) ?? snsMessageId, + messageId: stringOf(record.email_id), + recipient: stringOf(record.email_to), + occurredAt: dateOf(record.created_at), + bounce, + reason: stringOf(record.blocklist_reason) ?? stringOf(record.blocklist_type), + metadata: metadataFromHeaders(record.email_headers), + raw: record, + }); +}; + +/** + * Makes the verifier of the events of Scaleway Transactional Email, posted by + * Topics and Events: an SNS message from a listed topic, signed with a + * certificate on a Scaleway host that the trust chain the application pins + * issued, published within an hour, taken once. + * + * @param options `topicArns`; `trustChain`, the PEM of Scaleway's SNS trust + * chain; `fetch`, `now`, `toleranceMs`, `replayStore` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, confirmation? }`. + * @throws {TypeError} If an option is invalid. + */ +const createScalewayWebhookVerifier = (options = {}) => { + const authorities = trustChainOf(options.trustChain); + const verifySns = createSnsVerifier(options, { + provider: 'scaleway', + defaultToleranceMs: DEFAULT_TOLERANCE_MS, + isCertificateUrl: (value) => isHttpsUrlOf(value, SCALEWAY_HOST), + trust: (certificate) => { + for (const authority of authorities) { + if (certificate.checkIssued(authority) && certificate.verify(authority.publicKey)) return; + } + throw certificateError('The signing certificate was not issued by the trust chain.'); + }, + }); + return { + async verify(request) { + const { message, record, confirmation } = await verifySns(request); + if (confirmation !== undefined) return { events: [], confirmation }; + if (record === undefined) { + const event = createEvent({ + provider: 'scaleway', + type: 'unknown', + id: message.MessageId, + raw: message, + }); + return { events: [event] }; + } + return { events: [eventOf(record, message.MessageId)] }; + }, + }; +}; + +module.exports = { createScalewayWebhookVerifier }; diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index 80a165d..4c50833 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -10,44 +10,13 @@ // hard or soft; and a reject or a rendering failure is `rejected`, not // `bounced`. -const { X509Certificate, verify: verifySignature } = require('node:crypto'); -const { readText } = require('../internal/body.js'); -const { checkOptionalFunction } = require('../internal/options.js'); -const { - WebhookVerificationError, - bodyBytesOf, - checkReplay, - checkTime, - createEvent, - dateOf, - isRecord, - parseJsonOf, - resolveCommonOptions, - stringOf, -} = require('./request.js'); +const { createEvent, dateOf, isRecord, stringOf } = require('./request.js'); +const { createSnsVerifier, isHttpsUrlOf } = require('./sns.js'); // SNS retries a delivery with the time it was published, for up to an hour // by default. const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; -const MAX_CERTIFICATE_BYTES = 64 * 1024; -const MAX_CERTIFICATES = 32; const SNS_HOST = /^sns\.[a-z0-9-]+\.amazonaws\.com(\.cn)?$/; -const ALGORITHMS = new Map([ - ['1', 'RSA-SHA1'], - ['2', 'RSA-SHA256'], -]); -// The fields signed, in order, by type of message. -const SIGNED_FIELDS = new Map([ - ['Notification', ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']], - [ - 'SubscriptionConfirmation', - ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'], - ], - [ - 'UnsubscribeConfirmation', - ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'], - ], -]); const BOUNCE_KINDS = new Map([ ['Permanent', 'hard'], ['Transient', 'soft'], @@ -55,34 +24,8 @@ const BOUNCE_KINDS = new Map([ // The tags SES writes itself, and the one `./ses` writes for the priority. const OWN_TAG = /^(ses|aws):/; -const isSnsUrl = (value) => { - let url; - try { - url = new URL(value); - } catch { - return false; - } - return ( - url.protocol === 'https:' && - SNS_HOST.test(url.hostname) && - url.port === '' && - url.username === '' && - url.password === '' - ); -}; - -const stringToSign = (message, fields) => { - let text = ''; - for (const field of fields) { - const value = message[field]; - if (value === undefined && field === 'Subject') continue; - if (typeof value !== 'string') { - throw new WebhookVerificationError('payload', `The SNS message has no ${field}.`); - } - text += `${field}\n${value}\n`; - } - return text; -}; +// The certificates and subscriptions of SNS are on its hosts, over HTTPS. +const isSnsUrl = (value) => isHttpsUrlOf(value, SNS_HOST); // The tags and metadata of `mail.tags`: `category` holds the message's tags, // as `./ses` sends them, and the others its metadata. @@ -242,72 +185,6 @@ const eventsOf = (record, snsMessageId) => { return events; }; -const certificateError = (message, cause) => - new WebhookVerificationError('certificate', message, cause === undefined ? undefined : { cause }); - -const loadCertificate = async (fetchOption, url) => { - const fetch = fetchOption ?? globalThis.fetch; - let response; - try { - response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(10_000) }); - } catch (error) { - throw certificateError('The signing certificate could not be read.', error); - } - if (!response.ok) { - throw certificateError(`The signing certificate answered HTTP ${response.status}.`); - } - try { - return new X509Certificate(await readText(response, MAX_CERTIFICATE_BYTES)); - } catch (error) { - throw certificateError('The signing certificate is not one.', error); - } -}; - -// The certificates by URL: each read once while it is cached, the oldest -// dropped past MAX_CERTIFICATES, one that failed to load not kept. -const createCertificateCache = (fetchOption) => { - const certificates = new Map(); - return (url) => { - let pending = certificates.get(url); - if (pending === undefined) { - pending = loadCertificate(fetchOption, url); - pending.catch(() => certificates.delete(url)); - certificates.set(url, pending); - if (certificates.size > MAX_CERTIFICATES) { - certificates.delete(certificates.keys().next().value); - } - } - return pending; - }; -}; - -const checkSignature = async (message, certificateOf, common) => { - const fields = SIGNED_FIELDS.get(message.Type); - if (fields === undefined) { - throw new WebhookVerificationError( - 'payload', - `The SNS message type ${message.Type} is unknown.`, - ); - } - const algorithm = ALGORITHMS.get(message.SignatureVersion); - if (algorithm === undefined || typeof message.Signature !== 'string') { - throw new WebhookVerificationError('signature', 'The SNS signature version is unknown.'); - } - if (typeof message.SigningCertURL !== 'string' || !isSnsUrl(message.SigningCertURL)) { - throw certificateError('The signing certificate is not one of SNS.'); - } - const text = stringToSign(message, fields); - const certificate = await certificateOf(message.SigningCertURL); - const now = common.now(); - if (now < Date.parse(certificate.validFrom) || now > Date.parse(certificate.validTo)) { - throw certificateError('The signing certificate is not valid now.'); - } - const signature = Buffer.from(message.Signature, 'base64'); - if (!verifySignature(algorithm, Buffer.from(text, 'utf8'), certificate.publicKey, signature)) { - throw new WebhookVerificationError('signature', 'The SNS signature does not match.'); - } -}; - /** * Makes the verifier of the SNS messages of Amazon SES. * @@ -318,18 +195,11 @@ const checkSignature = async (message, certificateOf, common) => { * @throws {TypeError} If an option is invalid. */ const createSesWebhookVerifier = (options = {}) => { - const topicArns = options.topicArns; - if (!Array.isArray(topicArns) || topicArns.length === 0) { - throw new TypeError('topicArns must list the SNS topics the verifier takes.'); - } - const topics = new Set(); - for (const arn of topicArns) { - if (typeof arn !== 'string' || arn === '') throw new TypeError('A topic ARN is a string.'); - topics.add(arn); - } - const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); - const certificateOf = createCertificateCache(checkOptionalFunction(options.fetch, 'fetch')); - + const verifySns = createSnsVerifier(options, { + provider: 'ses', + defaultToleranceMs: DEFAULT_TOLERANCE_MS, + isCertificateUrl: isSnsUrl, + }); return { /** * Verifies an SNS message and reads its events. @@ -339,38 +209,9 @@ const createSesWebhookVerifier = (options = {}) => { * @throws {WebhookVerificationError} If the message is refused. */ async verify(request) { - const message = parseJsonOf(bodyBytesOf(request, common)); - if (!isRecord(message) || typeof message.TopicArn !== 'string') { - throw new WebhookVerificationError('payload', 'The body is not an SNS message.'); - } - if (!topics.has(message.TopicArn)) { - throw new WebhookVerificationError('topic', `The topic ${message.TopicArn} is not taken.`); - } - await checkSignature(message, certificateOf, common); - checkTime(Date.parse(message.Timestamp), common); - await checkReplay(common, `ses:${message.MessageId}`); - if (message.Type !== 'Notification') { - if (!isSnsUrl(message.SubscribeURL)) { - throw new WebhookVerificationError('payload', 'The subscription URL is not one of SNS.'); - } - return { - events: [], - confirmation: { - type: message.Type, - topicArn: message.TopicArn, - subscribeUrl: message.SubscribeURL, - token: message.Token, - message: message.Message, - }, - }; - } - let record; - try { - record = JSON.parse(message.Message); - } catch { - record = undefined; - } - if (!isRecord(record)) { + const { message, record, confirmation } = await verifySns(request); + if (confirmation !== undefined) return { events: [], confirmation }; + if (record === undefined) { const event = createEvent({ provider: 'ses', type: 'unknown', @@ -384,4 +225,4 @@ const createSesWebhookVerifier = (options = {}) => { }; }; -module.exports = { createSesWebhookVerifier, isSnsUrl }; +module.exports = { createSesWebhookVerifier }; diff --git a/src/webhooks/smtp2go.js b/src/webhooks/smtp2go.js new file mode 100644 index 0000000..4f56d92 --- /dev/null +++ b/src/webhooks/smtp2go.js @@ -0,0 +1,103 @@ +// Not ported: the events of SMTP2GO's webhooks, whose URL carries Basic +// credentials (https://developers.smtp2go.com/docs/webhooks-overview). +// sently and unemail have no SMTP2GO webhook. + +const { + WebhookVerificationError, + basicCredentialsOf, + bodyBytesOf, + checkBasicCredentials, + createEvent, + dateOf, + isRecord, + metadataFromHeaders, + resolveCommonOptions, + stringOf, +} = require('./request.js'); + +const TYPES = new Map([ + ['processed', 'accepted'], + ['delivered', 'delivered'], + ['bounce', 'bounced'], + ['reject', 'rejected'], + ['spam', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], +]); +const UTF8 = new TextDecoder('utf-8', { fatal: true }); + +// A body of JSON or of a form, as the webhook is set up. +const recordOf = (bytes) => { + let text; + try { + text = UTF8.decode(bytes); + } catch (error) { + throw new WebhookVerificationError('payload', 'The body is not UTF-8.', { cause: error }); + } + if (text.trimStart().startsWith('{')) { + try { + return JSON.parse(text); + } catch (error) { + throw new WebhookVerificationError('payload', 'The body is not JSON.', { cause: error }); + } + } + const record = Object.create(null); + for (const [name, value] of new URLSearchParams(text)) record[name] = value; + return record; +}; + +const eventOf = (record) => { + const type = TYPES.get(record.event) ?? 'unknown'; + let bounce; + if (type === 'bounced') { + let kind = 'undetermined'; + if (record.bounce === 'hard' || record.bounce === 'soft') kind = record.bounce; + bounce = { kind }; + if (stringOf(record.message) !== undefined) bounce.diagnostic = record.message; + } + const messageId = stringOf(record.email_id); + const recipient = stringOf(record.rcpt); + const at = record.time ?? record.sendtime; + return createEvent({ + provider: 'smtp2go', + type, + id: stringOf(record.id) ?? `${record.event}:${messageId}:${recipient}:${at}`, + messageId, + recipient, + occurredAt: dateOf(at), + bounce, + reason: stringOf(record.message), + url: stringOf(record.url), + metadata: metadataFromHeaders(record), + raw: record, + }); +}; + +/** + * Makes the verifier of SMTP2GO's webhooks, whose URL carries Basic + * credentials, as SMTP2GO signs nothing; a body of JSON or of a form. The + * message id is the `email_id`, the receipt's; the metadata comes back from + * `X-Metadata-` fields when the webhook adds them to its payload. + * + * @param options `username`, `password` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events }`. + * @throws {TypeError} If a credential is missing. + */ +const createSmtp2goWebhookVerifier = (options = {}) => { + const expected = basicCredentialsOf(options, 'SMTP2GO'); + const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + return { + async verify(request) { + const bytes = bodyBytesOf(request, common); + checkBasicCredentials(request, expected); + const record = recordOf(bytes); + if (!isRecord(record) || typeof record.event !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not an SMTP2GO event.'); + } + return { events: [eventOf(record)] }; + }, + }; +}; + +module.exports = { createSmtp2goWebhookVerifier }; diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js new file mode 100644 index 0000000..eb59d9f --- /dev/null +++ b/src/webhooks/sns.js @@ -0,0 +1,215 @@ +// Not ported: the messages of the SNS protocol, which Amazon SNS and +// Scaleway's Topics and Events post to a webhook: their topic, signature +// (versions 1 and 2), certificate, time and id, verified before their +// message is read +// (https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html). + +const { X509Certificate, verify: verifySignature } = require('node:crypto'); +const { readText } = require('../internal/body.js'); +const { checkOptionalFunction } = require('../internal/options.js'); +const { + WebhookVerificationError, + bodyBytesOf, + checkReplay, + checkTime, + isRecord, + parseJsonOf, + resolveCommonOptions, +} = require('./request.js'); + +const MAX_CERTIFICATE_BYTES = 64 * 1024; +const MAX_CERTIFICATES = 32; +const ALGORITHMS = new Map([ + ['1', 'RSA-SHA1'], + ['2', 'RSA-SHA256'], +]); +// The fields signed, in order, by type of message. +const SUBSCRIPTION_FIELDS = [ + 'Message', + 'MessageId', + 'SubscribeURL', + 'Timestamp', + 'Token', + 'TopicArn', + 'Type', +]; +const SIGNED_FIELDS = new Map([ + ['Notification', ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']], + ['SubscriptionConfirmation', SUBSCRIPTION_FIELDS], + ['UnsubscribeConfirmation', SUBSCRIPTION_FIELDS], +]); + +/** + * Whether a value is an HTTPS URL on a host the pattern takes, with no port + * or credentials. + * + * @internal + */ +const isHttpsUrlOf = (value, host) => { + let url; + try { + url = new URL(value); + } catch { + return false; + } + return ( + url.protocol === 'https:' && + host.test(url.hostname) && + url.port === '' && + url.username === '' && + url.password === '' + ); +}; + +const stringToSign = (message, fields) => { + let text = ''; + for (const field of fields) { + const value = message[field]; + if (value === undefined && field === 'Subject') continue; + if (typeof value !== 'string') { + throw new WebhookVerificationError('payload', `The SNS message has no ${field}.`); + } + text += `${field}\n${value}\n`; + } + return text; +}; + +const certificateError = (message, cause) => + new WebhookVerificationError('certificate', message, cause === undefined ? undefined : { cause }); + +const loadCertificate = async (fetchOption, url) => { + const fetch = fetchOption ?? globalThis.fetch; + let response; + try { + response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(10_000) }); + } catch (error) { + throw certificateError('The signing certificate could not be read.', error); + } + if (!response.ok) { + throw certificateError(`The signing certificate answered HTTP ${response.status}.`); + } + try { + return new X509Certificate(await readText(response, MAX_CERTIFICATE_BYTES)); + } catch (error) { + throw certificateError('The signing certificate is not one.', error); + } +}; + +// The certificates by URL: each read once while it is cached, the oldest +// dropped past MAX_CERTIFICATES, one that failed to load not kept. +const createCertificateCache = (fetchOption, trust) => { + const certificates = new Map(); + return (url) => { + let pending = certificates.get(url); + if (pending === undefined) { + pending = loadCertificate(fetchOption, url).then((certificate) => { + trust(certificate); + return certificate; + }); + pending.catch(() => certificates.delete(url)); + certificates.set(url, pending); + if (certificates.size > MAX_CERTIFICATES) { + certificates.delete(certificates.keys().next().value); + } + } + return pending; + }; +}; + +const checkSignature = async (message, certificateOf, isCertificateUrl, common) => { + const fields = SIGNED_FIELDS.get(message.Type); + if (fields === undefined) { + throw new WebhookVerificationError( + 'payload', + `The SNS message type ${message.Type} is unknown.`, + ); + } + const algorithm = ALGORITHMS.get(message.SignatureVersion); + if (algorithm === undefined || typeof message.Signature !== 'string') { + throw new WebhookVerificationError('signature', 'The SNS signature version is unknown.'); + } + if (typeof message.SigningCertURL !== 'string' || !isCertificateUrl(message.SigningCertURL)) { + throw certificateError('The signing certificate is not on a host of the service.'); + } + const text = stringToSign(message, fields); + const certificate = await certificateOf(message.SigningCertURL); + const now = common.now(); + if (now < Date.parse(certificate.validFrom) || now > Date.parse(certificate.validTo)) { + throw certificateError('The signing certificate is not valid now.'); + } + const signature = Buffer.from(message.Signature, 'base64'); + if (!verifySignature(algorithm, Buffer.from(text, 'utf8'), certificate.publicKey, signature)) { + throw new WebhookVerificationError('signature', 'The SNS signature does not match.'); + } +}; + +const topicsOf = (topicArns) => { + if (!Array.isArray(topicArns) || topicArns.length === 0) { + throw new TypeError('topicArns must list the topics the verifier takes.'); + } + const topics = new Set(); + for (const arn of topicArns) { + if (typeof arn !== 'string' || arn === '') throw new TypeError('A topic ARN is a string.'); + topics.add(arn); + } + return topics; +}; + +/** + * Makes the verification of SNS messages: `verify(request)` gives `{ message, + * record }` for a notification, `record` its parsed message when it is a JSON + * object, or `{ message, confirmation }` for a subscription message. + * + * @param options `topicArns`, `fetch`, and the options every verifier takes. + * @param service `provider`, `defaultToleranceMs`, `isCertificateUrl` (the + * hosts of certificates and subscriptions), and `trust(certificate)`, which + * throws for a certificate the service did not issue. + * @internal + */ +const createSnsVerifier = (options, service) => { + const topics = topicsOf(options.topicArns); + const common = resolveCommonOptions(options, service.defaultToleranceMs); + const certificateOf = createCertificateCache( + checkOptionalFunction(options.fetch, 'fetch'), + service.trust ?? (() => {}), + ); + return async (request) => { + const message = parseJsonOf(bodyBytesOf(request, common)); + if (!isRecord(message) || typeof message.TopicArn !== 'string') { + throw new WebhookVerificationError('payload', 'The body is not an SNS message.'); + } + if (!topics.has(message.TopicArn)) { + throw new WebhookVerificationError('topic', `The topic ${message.TopicArn} is not taken.`); + } + await checkSignature(message, certificateOf, service.isCertificateUrl, common); + checkTime(Date.parse(message.Timestamp), common); + await checkReplay(common, `${service.provider}:${message.MessageId}`); + if (message.Type !== 'Notification') { + if (!service.isCertificateUrl(message.SubscribeURL)) { + throw new WebhookVerificationError( + 'payload', + 'The subscription URL is not of the service.', + ); + } + return { + message, + confirmation: { + type: message.Type, + topicArn: message.TopicArn, + subscribeUrl: message.SubscribeURL, + token: message.Token, + message: message.Message, + }, + }; + } + let record; + try { + record = JSON.parse(message.Message); + } catch { + record = undefined; + } + return { message, record: isRecord(record) ? record : undefined }; + }; +}; + +module.exports = { certificateError, createSnsVerifier, isHttpsUrlOf }; diff --git a/tests/fixtures/webhooks/scaleway-ca.pem b/tests/fixtures/webhooks/scaleway-ca.pem new file mode 100644 index 0000000..d3df1f5 --- /dev/null +++ b/tests/fixtures/webhooks/scaleway-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGTCCAgGgAwIBAgIUNfE1QvGkFiqVj+PpP2joATmnsqUwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQVGVzdCBTY2FsZXdheSBDQTAgFw0yNjEwMTcyMDI3MDVa +GA8yMTI2MDkyMzIwMjcwNVowGzEZMBcGA1UEAwwQVGVzdCBTY2FsZXdheSBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKnEbf/g80sgh++NSrH86I/G +aKWkll2eueOO4N6owO9VnHrArJcbRdSmQwn0ZpcD5CzhUrEImLNBbXEMfOuvTwA6 +bzMthwduB90/ddRFQdFOini8CQ6/huiOTui8uKCVsyMQ2GXa/Co/oZWulfctoW5h +VfqxLG9pxIM3OtyrpzpBrCZbnYfTvK3fbpO4VzSE5szJVJLXFCKuld9qrWcVBgIP +4R9ZPZTba/d+1gKLAXUbdAY3k63q514+nOil2nrqbSfF2Oo7cG9at7qr76MUFI8s +02G18RAKQt3ZL0SjLcsgfGQJh6jTy19BURT4ugDkgkB/STyYL2OuLQBBDzbOCV8C +AwEAAaNTMFEwHQYDVR0OBBYEFJFlU+h6V/JlG3VhxeQ0otG+a5IdMB8GA1UdIwQY +MBaAFJFlU+h6V/JlG3VhxeQ0otG+a5IdMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAEuzUJQ5EUTxb57+eVSTKQcegBI36YBT1GlwKah+6y1wq9q4 +PFv/Y4d1KcX2dZJbIyL3F0pnHJfDZdUozDzUzGEJm2H7lRjne4OceDoAvVjxdeVl +Pj7VrJWbDnkEJlMXR/Di2bNxYlBpjLg5jF2Ww6Hi+L1dGlOuyxGw8gqOsfQYBDLD +nBTPCInAz8j35gsjtpfusLNt3zDXfdwbHwvcD5Z4RSUnGkZvM1FWqWu33gZEHz0J +XJB5H+yWkszWJhNKhTD6mgsSaKYV9E1dQMgGSpbsWsbBOkv1xD7bV5cn5C42mtRk +lgjUV21/763NoRNLfcw9p81BeFuLLPFlrwojjjQ= +-----END CERTIFICATE----- diff --git a/tests/fixtures/webhooks/scaleway-cert.pem b/tests/fixtures/webhooks/scaleway-cert.pem new file mode 100644 index 0000000..1829ea1 --- /dev/null +++ b/tests/fixtures/webhooks/scaleway-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIUJn8BL26Wi9P6nozHkmZIEqjR+QYwDQYJKoZIhvcNAQEL +BQAwGzEZMBcGA1UEAwwQVGVzdCBTY2FsZXdheSBDQTAgFw0yNjEwMTcyMDI3MDVa +GA8yMTI2MDkyMzIwMjcwNVowJjEkMCIGA1UEAwwbc25zLm1ucS5mci1wYXIuc2Nh +bGV3YXkuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAknXsLgIo +rcgg+qXNZ/lmRFGRHt/rOZUMGX+deTZA0p2hNLqIL3r3yJvLj5+bmzzcGCPdjznu +soa9LGiS6UOGiodLddE7WPfL3LgkNLhA5NM2XyKFbJmLe5O6Ue2gylgZhlYyrSqs +Ls+DK2nZT94SXR12wg6qz9Bct+LyBhckvH5R5UEIS9QyYMNzp0Ghj4BauiwLpoWc +oYcHoN8XCPcong2MHViK9Ur32+8PtqIGVX8RYLoD0Hwq0rJTibPOyRObwMDbPle/ +lJxnZ8wzZ4S2EVCBGvYDrrJ657C4U70sxiA8zqqEBJS1Jx3uVAkALBxGUX/vcMmW +xhcZtoq8w7iL8wIDAQABo0IwQDAdBgNVHQ4EFgQUUJH7MwbC3OPoxmxFfEenGwSU +8dQwHwYDVR0jBBgwFoAUkWVT6HpX8mUbdWHF5DSi0b5rkh0wDQYJKoZIhvcNAQEL +BQADggEBAEnA++YcK7LVtgE3tS+aod01rnJU6K+wOHx8Ee0yVZWveodcxtj0s55t +Dfi/IFtkXlKflyrevl4e8H+x27lBNzR2aVPnRcduo5l5tzFfaiWlPhlQ5gUUFDRK +tAkcliyx444FEYittsAJ98l4yx8/3JJ52Vsw7/ciu8rVMeH7HHRENtAYPg6yqJu2 +vEtAyo78kZ5rgKbfEqKHofvqPv1LGqv6/m4fDqCntIGpj0PgGCGKnVWSkRav0V16 +GUxNDEUB1BW4yb/sWqiKPytYOR8ZJLowWMvRFqhTI+nQ+BtSXUALigAcf3TfAn4j ++qPRSti/GdpPzWndhEm6QYOorLwAJts= +-----END CERTIFICATE----- diff --git a/tests/fixtures/webhooks/scaleway-key.pem b/tests/fixtures/webhooks/scaleway-key.pem new file mode 100644 index 0000000..e62c280 --- /dev/null +++ b/tests/fixtures/webhooks/scaleway-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCSdewuAiityCD6 +pc1n+WZEUZEe3+s5lQwZf515NkDSnaE0uogvevfIm8uPn5ubPNwYI92POe6yhr0s +aJLpQ4aKh0t10TtY98vcuCQ0uEDk0zZfIoVsmYt7k7pR7aDKWBmGVjKtKqwuz4Mr +adlP3hJdHXbCDqrP0Fy34vIGFyS8flHlQQhL1DJgw3OnQaGPgFq6LAumhZyhhweg +3xcI9yieDYwdWIr1Svfb7w+2ogZVfxFgugPQfCrSslOJs87JE5vAwNs+V7+UnGdn +zDNnhLYRUIEa9gOusnrnsLhTvSzGIDzOqoQElLUnHe5UCQAsHEZRf+9wyZbGFxm2 +irzDuIvzAgMBAAECggEACOfKQ4rENN/CbZjmTbFi/boJYduS+i5tHyHKjj8sVktG +9iWa3LQ5EqppUZpFX4+Tt94MtGqPS609Qixd7C+E/0PIm6AjFuNwoGeeP/fSJS6h +uDlN7E7pgTjlr1AMkJ7dLeLaBmmnDGzBBmfTdzywoZc9HpQHl0bWnZHJBSjIqrNa +laE0eIwhKILJa5vylKc2wEhCP7nV8KMesNPv/tpB5S8HIdfYkoN7C29R+rzM18A1 +tYGXM6rxpPnSuOQJAzupA4vLMVgAD0wqT2lm4c/9IIrHwWJUS3zBX92cuogzNkMv +eFPS+DRfmZqxxFD0t/RxDXHwvsEiQwpC6SNng4F75QKBgQDOsmeH/xVev2nRPHca +htvZeHw1gI0BtuRTiVy1lMowtOKeE5R/EZpIdpkJMBPTdQN5mLN3Og/1uOOE4E48 +1o1IOEDaxpQZEoJMua9mETIS8MGwo4UYsAm8rMvgAWUDimkj+eqob1rG2GvE/p24 +MhNHHGvJ6HIu3pEsiTYb8o8zFwKBgQC1ZUsMEKsFt17ORgxupb3Lfj0r73X8XUMy +Hqel2AKJ+Tb8rzfQ6RQZvqxc9tP5ZmDR/5EIUc2Ex6Yf7rmU/HibyHhM5dqDVbzQ +MOxZa+z0gZCSL+XEcDjcVDATWKPdJBXX+4KCrLZWyObx0GfEv+r8eHboE77LEbXY +U+pMPlmnhQKBgH573a7zAXjBWVeY0KmC7PXfLOLwj1kFOiOEYx8atb+ocSSahrxb +YTYGnQ9qRDeZ206FiwzikvGi+ldNi5LfGAWhps45xlaxZNrepd6nlyTP/zbAWaoJ +z2uNshSZxd6szDofrOD1g2868HfT91/IBNP3KF6dx4G24M+IB3yCqe5xAoGAZ6ns +moeIBStHDeDwR+eGMJ4DPXGK3t85gJIOQvPQd6MprnxADK624wOZo0MX3uaIHFrO +Oo/tQJ2MLaGTAzbJglWtklylZPsBfYUH/Q51i0HkKfcfewSbKicell/IvZ9asGS2 +sBEd3zjO6pCsTL9U/lWltnlaRtjnMCq8sjfLqrUCgYEAtDzKpt63itkjkfET1jzc +O4WXqlF59s237q71TtNX/zluEdGgy/3+21KbczLFLKJRi8mIw0ZZi77VTz3g7X3s +sChHndF4ucO4PwYlUlsFNr5kdMnV8VY4BdfTvZeCXdu3iOO5O948wdwv4XL8huXC +t0TofE3GvYmWj6PV2sDOxe8= +-----END PRIVATE KEY----- diff --git a/tests/helpers/webhooks/sns.js b/tests/helpers/webhooks/sns.js index 301214e..6f209d8 100644 --- a/tests/helpers/webhooks/sns.js +++ b/tests/helpers/webhooks/sns.js @@ -8,6 +8,12 @@ const { sign } = require('node:crypto'); const FIXTURES = path.join(__dirname, '..', '..', 'fixtures', 'webhooks'); const certificate = readFileSync(path.join(FIXTURES, 'sns-cert.pem'), 'utf8'); const privateKey = readFileSync(path.join(FIXTURES, 'sns-key.pem'), 'utf8'); +// Scaleway's: a certificate a test CA issued, and that CA as the trust chain. +const scaleway = { + certificate: readFileSync(path.join(FIXTURES, 'scaleway-cert.pem'), 'utf8'), + privateKey: readFileSync(path.join(FIXTURES, 'scaleway-key.pem'), 'utf8'), + trustChain: readFileSync(path.join(FIXTURES, 'scaleway-ca.pem'), 'utf8'), +}; const TOPIC = 'arn:aws:sns:eu-west-1:123456789012:ses-events'; const CERT_URL = 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-abc.pem'; @@ -24,7 +30,7 @@ const SUBSCRIPTION = [ 'Type', ]; -const signSns = (message, version = '2') => { +const signSns = (message, version = '2', key = privateKey) => { const fields = message.Type === 'Notification' ? NOTIFICATION : SUBSCRIPTION; let text = ''; for (const field of fields) { @@ -35,7 +41,7 @@ const signSns = (message, version = '2') => { ...message, SignatureVersion: version, SigningCertURL: message.SigningCertURL ?? CERT_URL, - Signature: sign(algorithm, Buffer.from(text, 'utf8'), privateKey).toString('base64'), + Signature: sign(algorithm, Buffer.from(text, 'utf8'), key).toString('base64'), }; }; @@ -50,13 +56,21 @@ const notification = (record, fields = {}) => }); // A fetch that serves the test certificate, and counts its requests. -const certificateFetch = () => { +const certificateFetch = (served = certificate) => { const requests = []; const fetch = async (url, init) => { requests.push({ url: String(url), init }); - return new Response(certificate, { status: 200 }); + return new Response(served, { status: 200 }); }; return { fetch, requests }; }; -module.exports = { CERT_URL, TOPIC, certificate, certificateFetch, notification, signSns }; +module.exports = { + CERT_URL, + TOPIC, + certificate, + certificateFetch, + notification, + scaleway, + signSns, +}; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index 656ff3b..154258c 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -2,7 +2,10 @@ import { expectAssignable, expectError, expectType } from 'tsd'; import type { DeliveryEvent } from '../../index.js'; import { WebhookVerificationError, + createAzureWebhookVerifier, createBrevoWebhookVerifier, + createLettermintWebhookVerifier, + createMailerooWebhookVerifier, createMailerSendWebhookVerifier, createMailgunWebhookVerifier, createMailjetWebhookVerifier, @@ -10,8 +13,10 @@ import { createMemoryReplayStore, createPostmarkWebhookVerifier, createResendWebhookVerifier, + createScalewayWebhookVerifier, createSendGridWebhookVerifier, createSesWebhookVerifier, + createSmtp2goWebhookVerifier, createSparkPostWebhookVerifier, type ReplayStore, type SesWebhookResult, @@ -64,3 +69,16 @@ expectAssignable[]>( ); expectError(createBrevoWebhookVerifier({})); expectError(createMailtrapWebhookVerifier({ secret: 's', toleranceMs: 1 })); + +const scaleway = createScalewayWebhookVerifier({ + topicArns: ['arn:scw:sns:fr-par:p:t'], + trustChain: '-----BEGIN CERTIFICATE-----', +}); +expectType((await scaleway.verify({ body: '' })).confirmation?.token); +createLettermintWebhookVerifier({ secret: 'whsec_x', toleranceMs: 60_000 }); +createMailerooWebhookVerifier({ secret: 's' }); +createSmtp2goWebhookVerifier({ username: 'u', password: 'p' }); +const azure = createAzureWebhookVerifier({ token: 't', header: 'x-webhook-token' }); +expectType((await azure.verify({ body: '' })).validation?.validationCode); +expectError(createScalewayWebhookVerifier({ topicArns: ['arn'] })); +expectError(createAzureWebhookVerifier({})); diff --git a/tests/unit/webhooks-others.test.js b/tests/unit/webhooks-others.test.js new file mode 100644 index 0000000..6ede4f4 --- /dev/null +++ b/tests/unit/webhooks-others.test.js @@ -0,0 +1,492 @@ +// Not ported: tests of the Scaleway, Lettermint, Maileroo, SMTP2GO and Azure +// verifiers of ./webhooks, against requests made as each provider documents. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createHmac } = require('node:crypto'); +const { + createAzureWebhookVerifier, + createLettermintWebhookVerifier, + createMailerooWebhookVerifier, + createScalewayWebhookVerifier, + createSmtp2goWebhookVerifier, + WebhookVerificationError, +} = require('../../src/webhooks/index.js'); +const { certificate, certificateFetch, scaleway, signSns } = require('../helpers/webhooks/sns.js'); + +const rejects = async (promise, reason) => { + await assert.rejects(promise, (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + return true; + }); +}; +const plain = (event) => ({ ...event, metadata: { ...event.metadata } }); +const basic = (user, password) => `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; + +const SCW_TOPIC = 'arn:scw:sns:fr-par:project-1:tem-events'; +const SCW_CERT = 'https://messaging.s3.fr-par.scw.cloud/fr-par/sns/cert-1.pem'; +const scwMessage = (record, fields = {}) => + signSns( + { + Type: 'Notification', + MessageId: fields.MessageId ?? 'scw-1', + TopicArn: SCW_TOPIC, + Message: typeof record === 'string' ? record : JSON.stringify(record), + Timestamp: '2027-01-01T12:00:00.000Z', + SigningCertURL: SCW_CERT, + ...fields, + }, + '2', + scaleway.privateKey, + ); + +test('Webhooks Scaleway: an SNS message signed by a certificate the trust chain issued', async () => { + const served = certificateFetch(scaleway.certificate); + const verifier = createScalewayWebhookVerifier({ + topicArns: [SCW_TOPIC], + trustChain: scaleway.trustChain, + fetch: served.fetch, + now: () => Date.parse('2027-01-01T12:00:30Z'), + }); + const record = { + id: 'ev-1', + type: 'email_dropped', + created_at: '2027-01-01T11:59:00Z', + email_id: 'em-1', + email_to: 'bob@example.com', + email_headers: [ + { key: 'X-Metadata-orderId', value: '42' }, + { key: 'Subject', value: 'Hi' }, + ], + email_response_code: 550, + email_response_message: 'No such mailbox', + }; + const { events } = await verifier.verify({ body: JSON.stringify(scwMessage(record)) }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'scaleway', + type: 'bounced', + id: 'ev-1', + messageId: 'em-1', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-01T11:59:00Z'), + bounce: { kind: 'hard', code: '550', diagnostic: 'No such mailbox' }, + tags: [], + metadata: { orderId: '42' }, + raw: record, + }); + for (const [type, expected] of [ + ['email_queued', 'accepted'], + ['email_delivered', 'delivered'], + ['email_deferred', 'deferred'], + ['email_mailbox_not_found', 'bounced'], + ['email_blocklisted', 'rejected'], + ['email_spam', 'complained'], + ['blocklist_created', 'unknown'], + ]) { + const message = scwMessage({ type, blocklist_type: 'mailbox_full' }, { MessageId: type }); + const result = await verifier.verify({ body: JSON.stringify(message) }); + assert.strictEqual(result.events[0].type, expected, type); + assert.strictEqual(result.events[0].id, type); + } + const text = await verifier.verify({ + body: JSON.stringify(scwMessage('text', { MessageId: 'text' })), + }); + assert.strictEqual(text.events[0].type, 'unknown'); + const subscription = signSns( + { + Type: 'SubscriptionConfirmation', + MessageId: 'sub-1', + Token: 't', + TopicArn: SCW_TOPIC, + Message: 'Confirm', + SubscribeURL: 'https://sns.mnq.fr-par.scaleway.com/confirm?token=t', + Timestamp: '2027-01-01T12:00:00.000Z', + SigningCertURL: SCW_CERT, + }, + '2', + scaleway.privateKey, + ); + const confirmed = await verifier.verify({ body: JSON.stringify(subscription) }); + assert.strictEqual(confirmed.confirmation.subscribeUrl, subscription.SubscribeURL); + assert.strictEqual(served.requests.length, 1); +}); + +test('Webhooks Scaleway: a certificate the trust chain did not issue, or off Scaleway, is refused', async () => { + // The SES test certificate is self-signed, not issued by the test CA. + const foreign = createScalewayWebhookVerifier({ + topicArns: [SCW_TOPIC], + trustChain: scaleway.trustChain, + fetch: certificateFetch(certificate).fetch, + now: () => Date.parse('2027-01-01T12:00:30Z'), + }); + await rejects( + foreign.verify({ body: JSON.stringify(scwMessage({ type: 'email_spam' })) }), + 'certificate', + ); + const offHost = scwMessage( + { type: 'email_spam' }, + { SigningCertURL: 'https://sns.eu-west-1.amazonaws.com/c.pem' }, + ); + await rejects(foreign.verify({ body: JSON.stringify(offHost) }), 'certificate'); + assert.throws( + () => createScalewayWebhookVerifier({ topicArns: [SCW_TOPIC] }), + /trustChain is the PEM/, + ); + assert.throws( + () => createScalewayWebhookVerifier({ topicArns: [SCW_TOPIC], trustChain: 'no certificate' }), + /holds no certificate/, + ); + assert.throws( + () => createScalewayWebhookVerifier({ trustChain: scaleway.trustChain }), + /topicArns/, + ); +}); + +test('Webhooks Lettermint: the HMAC of the time and body, within a window, once', async () => { + const secret = 'whsec_lettermint'; + const verifier = createLettermintWebhookVerifier({ secret, now: () => 1_800_000_000_000 }); + const sign = (body, t = '1800000000') => ({ + body, + headers: { + 'X-Lettermint-Signature': `t=${t},v1=${createHmac('sha256', secret).update(`${t}.${body}`).digest('hex')}`, + }, + }); + const payload = { + id: 'del-1', + event: 'message.hard_bounced', + timestamp: '2027-01-15T08:00:00Z', + data: { + message_id: 'lm-1', + recipient: 'bob@example.com', + response: { status_code: 550, enhanced_status_code: '5.1.1', content: 'unknown user' }, + metadata: { orderId: '42' }, + tag: 'welcome', + tags: [], + }, + }; + const body = JSON.stringify(payload); + const { events } = await verifier.verify(sign(body)); + assert.deepStrictEqual(plain(events[0]), { + provider: 'lettermint', + type: 'bounced', + id: 'del-1', + messageId: 'lm-1', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-15T08:00:00Z'), + bounce: { kind: 'hard', code: '5.1.1', diagnostic: 'unknown user' }, + tags: ['welcome'], + metadata: { orderId: '42' }, + raw: payload, + }); + await rejects(verifier.verify(sign(body)), 'replay'); + await rejects(verifier.verify({ body, headers: {} }), 'signature'); + await rejects(verifier.verify({ ...sign(body), body: body.replace('bob', 'eve') }), 'signature'); + await rejects( + verifier.verify(sign(JSON.stringify({ ...payload, id: 'old' }), '1799990000')), + 'timestamp', + ); + const cases = [ + ['message.created', 'accepted'], + ['message.delivered', 'delivered'], + ['message.soft_bounced', 'bounced'], + ['message.spam_complaint', 'complained'], + ['message.failed', 'rejected'], + ['message.suppressed', 'rejected'], + ['message.unsubscribed', 'unsubscribed'], + ['message.opened', 'opened'], + ['message.clicked', 'clicked'], + ['webhook.test', 'unknown'], + ]; + for (const [event, type] of cases) { + const each = JSON.stringify({ + id: event, + event, + data: { + tags: ['a', 'b'], + destination_url: 'https://e.com', + reason_code: 'enforced_tls_failed', + response: { status_code: 421 }, + }, + }); + const result = await verifier.verify(sign(each)); + assert.strictEqual(result.events[0].type, type, event); + } + const soft = await verifier.verify( + sign(JSON.stringify({ id: 'soft', event: 'message.soft_bounced', data: {} })), + ); + assert.deepStrictEqual(soft.events[0].bounce, { kind: 'soft' }); + await rejects(verifier.verify(sign('[]')), 'payload'); + assert.throws(() => createLettermintWebhookVerifier({}), /secret/); +}); + +test('Webhooks Maileroo: the HMAC of the body, an event per recipient, once', async () => { + const verifier = createMailerooWebhookVerifier({ secret: 'mr-secret' }); + const sign = (body) => ({ + body, + headers: { + 'x-maileroo-signature': createHmac('sha256', 'mr-secret').update(body).digest('hex'), + }, + }); + const payload = { + event_id: 'ev-1', + event_type: 'rejected', + event_time: 1_800_000_000, + message_id: '', + message_reference_id: 'ref-1', + tags: { tag1: 'orders', orderId: '42' }, + event_data: { + from: ['shop@example.com'], + to: ['bob@example.com', 'carol@example.com'], + reject_reason: 'One or more of the recipients are on the suppression list.', + }, + }; + const body = JSON.stringify(payload); + const { events } = await verifier.verify(sign(body)); + assert.strictEqual(events.length, 2); + assert.deepStrictEqual(plain(events[1]), { + provider: 'maileroo', + type: 'rejected', + id: 'ev-1:1', + messageId: 'ref-1', + recipient: 'carol@example.com', + occurredAt: new Date(1_800_000_000_000), + reason: 'One or more of the recipients are on the suppression list.', + tags: ['orders'], + metadata: { orderId: '42' }, + raw: payload, + }); + await rejects(verifier.verify(sign(body)), 'replay'); + const cases = [ + ['accepted', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + ['failed', 'bounced'], + ['complained', 'complained'], + ['opened', 'opened'], + ['clicked', 'clicked'], + ['later', 'unknown'], + ]; + for (const [eventType, type] of cases) { + const each = JSON.stringify({ + event_id: eventType, + event_type: eventType, + tags: null, + event_data: { + to: 'bob@example.com', + reason: 'mailbox full', + original_url: 'https://e.com/a', + }, + }); + const result = await verifier.verify(sign(each)); + assert.strictEqual(result.events[0].type, type, eventType); + } + const failed = await verifier.verify( + sign( + JSON.stringify({ + event_id: 'f2', + event_type: 'failed', + event_data: { to: 'b@example.com', reason: 'no user' }, + }), + ), + ); + assert.deepStrictEqual(failed.events[0].bounce, { kind: 'hard', diagnostic: 'no user' }); + const nobody = await verifier.verify( + sign(JSON.stringify({ event_id: 'n', event_type: 'opened' })), + ); + assert.strictEqual(nobody.events[0].recipient, undefined); + await rejects( + verifier.verify({ body, headers: { 'x-maileroo-signature': 'beef' } }), + 'signature', + ); + await rejects(verifier.verify(sign('{}')), 'payload'); + assert.throws(() => createMailerooWebhookVerifier({}), /secret/); +}); + +test('Webhooks SMTP2GO: Basic credentials, a body of JSON or of a form', async () => { + const verifier = createSmtp2goWebhookVerifier({ username: 'hook', password: 'pw' }); + const record = { + event: 'bounce', + email_id: '1abc-2def', + rcpt: 'bob@example.com', + time: '2027-01-15T08:00:00Z', + bounce: 'hard', + message: '550 unknown user', + host: 'mx.example.com', + 'X-Metadata-orderId': '42', + }; + const body = JSON.stringify(record); + const { events } = await verifier.verify({ + body, + headers: { authorization: basic('hook', 'pw') }, + }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'smtp2go', + type: 'bounced', + id: 'bounce:1abc-2def:bob@example.com:2027-01-15T08:00:00Z', + messageId: '1abc-2def', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-15T08:00:00Z'), + bounce: { kind: 'hard', diagnostic: '550 unknown user' }, + reason: '550 unknown user', + tags: [], + metadata: { orderId: '42' }, + raw: record, + }); + await rejects( + verifier.verify({ body, headers: { authorization: basic('hook', 'no') } }), + 'credentials', + ); + const form = new URLSearchParams({ + event: 'click', + email_id: 'e2', + rcpt: 'carol@example.com', + url: 'https://e.com/a', + id: 'evt-9', + }).toString(); + const clicked = await verifier.verify({ + body: form, + headers: { authorization: basic('hook', 'pw') }, + }); + assert.strictEqual(clicked.events[0].type, 'clicked'); + assert.strictEqual(clicked.events[0].id, 'evt-9'); + assert.strictEqual(clicked.events[0].url, 'https://e.com/a'); + for (const [event, type] of [ + ['processed', 'accepted'], + ['delivered', 'delivered'], + ['reject', 'rejected'], + ['spam', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['resubscribe', 'unknown'], + ]) { + const each = await verifier.verify({ + body: JSON.stringify({ event }), + headers: { authorization: basic('hook', 'pw') }, + }); + assert.strictEqual(each.events[0].type, type, event); + } + const soft = await verifier.verify({ + body: JSON.stringify({ event: 'bounce', bounce: 'soft' }), + headers: { authorization: basic('hook', 'pw') }, + }); + assert.deepStrictEqual(soft.events[0].bounce, { kind: 'soft' }); + const unclear = await verifier.verify({ + body: JSON.stringify({ event: 'bounce' }), + headers: { authorization: basic('hook', 'pw') }, + }); + assert.deepStrictEqual(unclear.events[0].bounce, { kind: 'undetermined' }); + await rejects( + verifier.verify({ body: '{"event":', headers: { authorization: basic('hook', 'pw') } }), + 'payload', + ); + await rejects( + verifier.verify({ body: Buffer.from([0xff]), headers: { authorization: basic('hook', 'pw') } }), + 'payload', + ); + await rejects( + verifier.verify({ body: 'nothing=1', headers: { authorization: basic('hook', 'pw') } }), + 'payload', + ); + assert.throws(() => createSmtp2goWebhookVerifier({}), /username is required/); +}); + +test('Webhooks Azure: Event Grid events with the token, and the validation handed back', async () => { + const verifier = createAzureWebhookVerifier({ token: 'eg-token' }); + const validation = [ + { + id: 'v-1', + eventType: 'Microsoft.EventGrid.SubscriptionValidationEvent', + data: { + validationCode: 'code-1', + validationUrl: 'https://rp-eastus.eventgrid.azure.net/validate?x=1', + }, + }, + ]; + const validated = await verifier.verify({ + body: JSON.stringify(validation), + headers: { 'aeg-event-type': 'SubscriptionValidation' }, + }); + assert.deepStrictEqual(validated, { + events: [], + validation: { validationCode: 'code-1', validationUrl: validation[0].data.validationUrl }, + }); + await rejects( + verifier.verify({ + body: '[{"eventType":"Other"}]', + headers: { 'aeg-event-type': 'SubscriptionValidation' }, + }), + 'payload', + ); + const report = { + id: 'eg-1', + eventType: 'Microsoft.Communication.EmailDeliveryReportReceived', + eventTime: '2027-01-15T08:00:01Z', + data: { + sender: 'DoNotReply@example.com', + recipient: 'bob@example.com', + messageId: 'op-1', + status: 'Bounced', + deliveryStatusDetails: { statusMessage: '550 mailbox unavailable' }, + deliveryAttemptTimeStamp: '2027-01-15T08:00:00Z', + }, + }; + const click = { + id: 'eg-2', + eventType: 'Microsoft.Communication.EmailEngagementTrackingReportReceived', + data: { + recipient: 'bob@example.com', + messageId: 'op-1', + engagementType: 'click', + engagementContext: 'https://e.com/a', + userActionTimeStamp: '2027-01-15T09:00:00Z', + }, + }; + const body = JSON.stringify([report, click, { eventType: 'no id' }]); + const headers = { 'aeg-event-type': 'Notification', Authorization: 'Bearer eg-token' }; + const { events } = await verifier.verify({ body, headers }); + assert.deepStrictEqual(plain(events[0]), { + provider: 'azure', + type: 'bounced', + id: 'eg-1', + messageId: 'op-1', + recipient: 'bob@example.com', + occurredAt: new Date('2027-01-15T08:00:00Z'), + bounce: { kind: 'undetermined', diagnostic: '550 mailbox unavailable' }, + reason: '550 mailbox unavailable', + tags: [], + metadata: {}, + raw: report, + }); + assert.strictEqual(events[1].type, 'clicked'); + assert.strictEqual(events[1].url, 'https://e.com/a'); + await rejects(verifier.verify({ body, headers }), 'replay'); + await rejects( + verifier.verify({ body, headers: { 'aeg-event-type': 'Notification' } }), + 'credentials', + ); + const statuses = ['Delivered', 'Failed', 'FilteredSpam', 'Suppressed', 'Quarantined', 'Expanded']; + const many = []; + for (const status of statuses) { + many.push({ + id: `s-${status}`, + eventType: 'Microsoft.Communication.EmailDeliveryReportReceived', + data: { status }, + }); + } + many.push({ + id: 's-view', + eventType: 'Microsoft.Communication.EmailEngagementTrackingReportReceived', + data: { engagementType: 'view' }, + }); + many.push({ id: 's-other', eventType: 'Microsoft.Communication.Other' }); + const typed = await verifier.verify({ body: JSON.stringify(many), headers }); + assert.deepStrictEqual( + typed.events.map((event) => event.type), + ['delivered', 'bounced', 'bounced', 'rejected', 'deferred', 'unknown', 'opened', 'unknown'], + ); + await rejects(verifier.verify({ body: '{}', headers }), 'payload'); + const custom = createAzureWebhookVerifier({ token: 't', header: 'x-webhook-token' }); + const ok = await custom.verify({ body: '[]', headers: { 'x-webhook-token': 't' } }); + assert.deepStrictEqual(ok.events, []); + assert.throws(() => createAzureWebhookVerifier({}), /token is required/); +}); From 74161eb8b5680dbed95cb7fa34bf3983abdd55fd Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sat, 17 Oct 2026 23:59:26 +0300 Subject: [PATCH 047/143] ci: run the package on Bun and Deno Bun 1.4.3 and Deno 2.9.6 run the unit suite and the new tests/runtime in CI (Linux, versions pinned), each file in a fresh global (--parallel), as node --test gives each its own process. tests/runtime runs the package by its name with no mock, on Node too: every entry required and imported by name; SMTP over plain TCP, implicit TLS, STARTTLS and a CONNECT proxy, with DKIM checked by the independent verifier; an HTTP provider and fromUrl on the runtime's own fetch (a redirect, a body over 4 MiB, a lost answer); and ./sendmail spawning a shell script. package.json declares "type": "commonjs", so Deno reads the .js files as CommonJS without --unstable-detect-cjs. What differs on them, as the spike found: - Deno's fetch sends a request again when a reused HTTP/1.1 connection closes before its answer, so a message the provider took may go twice where Node and Bun report deliveryUnknown. The docs give Deno a fetch on Deno.createHttpClient({ poolMaxIdlePerHost: 0 }); tests/runtime pins both the replay and that fetch, and the JMAP tests of a lost answer are skipped on Deno. - Bun (BoringSSL) takes no self-signed certificate that is not a CA as a root of trust: tests/fixtures/tls is a CA and a certificate it issued. - A TLS server reads no SNI as null on Deno and undefined on Bun; Deno keeps mocked timers enabled after a test; export-parity spawns the node CLI, which Deno is not. The SMTP test servers move to tests/helpers/smtp/servers.js. CLAUDE.md (Platform, Commands, Testing notes), getting started, known issues, README and the npm-publish skill name the runtimes, Deno's permissions and fetch. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/npm-publish/SKILL.md | 9 +- .github/workflows/ci.yml | 34 +++- CLAUDE.md | 54 +++++- README.md | 4 +- docs/guide/getting-started.md | 43 +++++ docs/guide/known-issues.md | 11 ++ package.json | 6 +- tests/fixtures/tls/README.md | 31 ++++ tests/fixtures/tls/ca-cert.pem | 19 ++ tests/fixtures/tls/server-cert.pem | 20 ++ tests/fixtures/tls/server-key.pem | 28 +++ tests/helpers/smtp/servers.js | 126 +++++++++++++ tests/runtime/exports.test.js | 21 +++ tests/runtime/http.test.js | 171 ++++++++++++++++++ tests/runtime/sendmail.test.js | 66 +++++++ tests/runtime/smtp.test.js | 125 +++++++++++++ tests/unit/export-parity.test.js | 5 +- .../unit/jmap-raw-message.integration.test.js | 11 +- tests/unit/ratelimit.test.js | 2 + tests/unit/smtp-connect.test.js | 113 +----------- tests/unit/smtp.test.js | 6 +- 21 files changed, 775 insertions(+), 130 deletions(-) create mode 100644 tests/fixtures/tls/README.md create mode 100644 tests/fixtures/tls/ca-cert.pem create mode 100644 tests/fixtures/tls/server-cert.pem create mode 100644 tests/fixtures/tls/server-key.pem create mode 100644 tests/helpers/smtp/servers.js create mode 100644 tests/runtime/exports.test.js create mode 100644 tests/runtime/http.test.js create mode 100644 tests/runtime/sendmail.test.js create mode 100644 tests/runtime/smtp.test.js diff --git a/.claude/skills/npm-publish/SKILL.md b/.claude/skills/npm-publish/SKILL.md index 9b0add9..9b14f98 100644 --- a/.claude/skills/npm-publish/SKILL.md +++ b/.claude/skills/npm-publish/SKILL.md @@ -30,19 +30,24 @@ pnpm install --frozen-lockfile # install deps exactly as locked pnpm run lint # oxlint pnpm run format:check # oxfmt pnpm run test:coverage # tests under the c8 coverage gate +pnpm run test:runtime # tests/runtime: sockets, TLS, fetch and spawn, no mock pnpm run test:types # tsd pnpm run check:dts # tsc over the d.ts of every entry of the exports map pnpm size --check # the budgets of scripts/size-budgets.json pnpm run check:pack # the tarball: every file of the exports map, nothing else ``` -These are the same gates `prepublishOnly` runs. CI adds two more; run them too: +These are the same gates `prepublishOnly` runs. CI adds three more; run them too: ```bash pnpm docs:comparison && git diff --exit-code docs/guide/comparison.md # the table matches docs/comparison.json pnpm run docs:build # the docs site; a dead link fails it +pnpm run test:bun && pnpm run test:deno # the unit and runtime suites on Bun and Deno ``` +The last needs the versions of Bun and Deno that `.github/workflows/ci.yml` pins; without them +installed, say so and rely on CI. + Then in parallel: `pnpm outdated` and `pnpm audit` (report to user; devDependencies only, the package has no runtime dependencies). If any gate fails, stop and fix issues before continuing. @@ -183,7 +188,7 @@ Report the file count and the bundle size from `pnpm run size` to the user. Run the gates one more time after all edits: ```bash -pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run test:types && pnpm run check:dts && pnpm size --check && pnpm run check:pack && pnpm run docs:build +pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run test:runtime && pnpm run test:types && pnpm run check:dts && pnpm size --check && pnpm run check:pack && pnpm run docs:build ``` Must exit 0. If it fails, fix and re-run. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 287fa1d..140689f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,38 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm test + # Sockets, TLS, fetch and child processes of the runtime, with no mock; + # the same files run on Bun and Deno below. + - run: pnpm run test:runtime + + # Bun and Deno run the unit and runtime suites through their own node:test. + # The versions are pinned, as the docs name the ones tested; engines stays + # Node >= 22, and Node installs the devDependencies the tests import. + runtimes: + name: test (${{ matrix.runtime }}) + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + runtime: [bun, deno] + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - if: matrix.runtime == 'bun' + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.3 + - if: matrix.runtime == 'deno' + uses: denoland/setup-deno@v2 + with: + deno-version: 2.9.6 + - run: pnpm run test:${{ matrix.runtime }} coverage: runs-on: ubuntu-latest @@ -123,7 +155,7 @@ jobs: ci-success: runs-on: ubuntu-latest timeout-minutes: 5 - needs: [lint, types, test, coverage, docs] + needs: [lint, types, test, runtimes, coverage, docs] if: always() steps: - name: Fail when any needed job failed, was cancelled or was skipped diff --git a/CLAUDE.md b/CLAUDE.md index da763c9..ee6ac8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,8 @@ Nodemailer, emailjs or Upyo, made of two parts: `docs/comparison.json` has: a persistent queue and outbox, an append-only audit log, one model of delivery events, suppression, idempotency and failover between providers. -It has zero runtime dependencies and runs on Node.js ≥ 22 only (no browser entry). +It has zero runtime dependencies and runs on Node.js ≥ 22, and on Bun and Deno through their +Node compatibility (see "Platform"; no browser entry). **Status: 1.0.0 is the ported transport, the reliability layer is being designed.** The root entry exports the message model and the transport contract of upyo, each transport and decorator @@ -33,6 +34,9 @@ Package manager is **pnpm** (pinned in `packageManager`). ```bash pnpm test # node --test tests/unit/*.test.js +pnpm test:runtime # tests/runtime: the package by its name, no mock (sockets, TLS, fetch, spawn) +pnpm test:bun # tests/unit and tests/runtime on Bun (bun test --parallel) +pnpm test:deno # tests/unit and tests/runtime on Deno (deno test --parallel) pnpm test:mailpit # opt-in: tests/mailpit against a local Mailpit (compose file there) pnpm test:e2e # opt-in: tests/e2e against real providers, from env variables node --test tests/unit/mailer.test.js # a single file @@ -49,9 +53,9 @@ pnpm docs:comparison # renders docs/comparison.json into docs/guid pnpm run docs:dev # VitePress dev server for docs/ (docs:build, docs:preview) ``` -`prepublishOnly` runs lint + format:check + test:coverage + test:types + check:dts + the size -gate + check:pack. CI adds the comparison drift check and the docs build. Treat that as the -pre-merge gate. There is no build step. +`prepublishOnly` runs lint + format:check + test:coverage + test:runtime + test:types + check:dts + +the size gate + check:pack. CI adds the comparison drift check, the docs build, and the unit and +runtime suites on Bun and Deno. Treat that as the pre-merge gate. There is no build step. ## No build step: CommonJS + hand-written types @@ -66,7 +70,8 @@ pre-merge gate. There is no build step. runtime exports with the declared value exports and imports every name from ESM. `tests/types/index.test-d.ts` (tsd) pins signatures. `check:dts` needs `--target es2022` because there is no tsconfig. -- `package.json` has a closed `exports` map (`types` first, then `default`), `sideEffects: false` +- `package.json` has a closed `exports` map (`types` first, then `default`), `sideEffects: false`, + `"type": "commonjs"` (Node's default, declared for Deno, which otherwise reads `.js` as ESM) and a `files` allowlist: `index.js`, `index.d.ts`, `src`, README, CHANGELOG, LICENSE and SECURITY. - **Subpaths, not a monorepo.** A subpath is a directory `src//` with an `index.js` that @@ -630,8 +635,9 @@ src/ headers.js, provider.js, batch.js, validate-ssl.js) tests/ unit/*.test.js node:test suites, one per area (-.test.js for a subpath) + runtime/*.test.js the package by its name on Node, Bun and Deno, with no mock helpers// shared test code of a subpath (fake servers, fixtures builders) - fixtures// static fixtures (certificates) + fixtures// static fixtures (certificates; tls/ is a CA and a certificate it issued) types/*.test-d.ts tsd assertions, one file per entry scripts/ size.js bundle-size report; --check gates each entry of the exports map @@ -759,9 +765,27 @@ whether Fly.io restricts outgoing SMTP (it would make HTTP-first a requirement). ## Platform -Node.js ≥ 22 only: `engines` is `>=22.0.0`, CI runs 22 and 24 on Linux and Windows. There is no -`browser.js` and no `browser` field: sending from a browser would expose the credentials, and -SMTP needs sockets. `src/` may use `node:*` modules freely. Edge runtimes are not a target for +Node.js ≥ 22 is the target: `engines` is `>=22.0.0`, and CI runs 22 and 24 on Linux and Windows. +Bun 1.4.3 and Deno 2.9.6 run the unit suite and `tests/runtime` in CI too, on Linux, through +their Node compatibility; their versions are pinned in `ci.yml` and named in the docs +(getting started), and `engines` names neither. A Deno user imports `npm:@alexify/mailer`. What +differs on them, as of those versions: + +- **Deno's `fetch` sends a request again** when a reused HTTP/1.1 connection closes before its + answer (hyper retries it), so a message the provider took may go twice where Node and Bun + report it as `deliveryUnknown`; upstream streamed the JMAP bodies against it (bug 81), and the + HTTP providers always sent a buffer. A streamed body or a client that keeps no idle connection + stops it. The docs give Deno users a `fetch` on `Deno.createHttpClient({ poolMaxIdlePerHost: + 0 })` (known issues, getting started), rather than the library sniffing the runtime; + `tests/runtime/http.test.js` pins both the replay and that `fetch`. +- **Deno's permissions:** `--allow-net`; `--allow-read` for `fromPath`; `--allow-run` and + `--allow-env` for `./sendmail` (its `child_process` passes the environment on); and + `--allow-sys=hostname` for the default EHLO name, which is `[127.0.0.1]` without it. +- **Bun's TLS (BoringSSL)** takes no self-signed certificate that is not a CA as a root of + trust, as Node and Deno do, so the tests that pass `ca` use `tests/fixtures/tls`. + +There is no `browser.js` and no `browser` field: sending from a browser would expose the +credentials, and SMTP needs sockets. `src/` may use `node:*` modules freely. Edge runtimes are not a target for now; if they become one, the transport over `fetch` and the core would move behind a separate entry with no `node:*`, and a platform test would enforce it as in protoarray. @@ -849,6 +873,18 @@ change. **This file wins** if they ever disagree. 401 to a raw message of unknown size, which streams, is `todo` on Node 24 (known upstream bug 14). `tests/unit/export-parity.test.js` also checks that every d.ts using `AsyncDisposable` references `esnext.disposable` itself. +- **`tests/runtime/`** runs the package as a consumer does, by its name, with no mock: every entry + required and imported by name; SMTP over plain TCP, implicit TLS, STARTTLS and an HTTP CONNECT + proxy, with DKIM checked by `tests/helpers/mime/verify-dkim.js`; an HTTP provider and + `fromUrl` on the runtime's own `fetch` (a redirect, a body over 4 MiB, a lost answer); and + `./sendmail` spawning a shell script (POSIX only). The same files run on Node, Bun and Deno, so + they use `node:test` and `node:assert` alone, never `t.mock`. +- **Bun and Deno** (`pnpm test:bun`, `pnpm test:deno`) run each file in a fresh global + (`--parallel`), as `node --test` runs each in a process of its own; in one global, the mocks of + one file reach the next. Deno keeps mocked timers enabled after a test (`t.after` resets + them), a TLS server reads no SNI as `null` on Deno and `undefined` on Bun (`false` on Node), the + JMAP tests of a lost answer are skipped on Deno, whose `fetch` sends it again, and the import + by name of `export-parity` spawns the node CLI, which Deno is not. ## Documentation site diff --git a/README.md b/README.md index dc39a8b..c945353 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,9 @@ pnpm add @alexify/mailer npm install @alexify/mailer ``` -Requires Node.js 22 or newer. +Requires Node.js 22 or newer. Bun and Deno run it too, tested in CI; on Deno, see +[Bun and Deno](https://mailerat.vercel.app/guide/getting-started#bun-and-deno) for the +permissions and the `fetch` to pass. ## Usage diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9991f3b..fa9a614 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -18,6 +18,49 @@ Node.js 22 or newer. The package has no runtime dependencies: every transport is package, reached through a subpath (`@alexify/mailer/smtp`), and what a decorator needs from your application (an OpenTelemetry provider, a logger) is passed in. +### Bun and Deno + +The package runs on Bun and Deno through their Node compatibility. CI runs its +tests on Bun 1.4.3 and Deno 2.9.6 (Linux), sending over real sockets, TLS and +`fetch`. + +```bash +bun add @alexify/mailer +deno add npm:@alexify/mailer +``` + +On Deno, import it by its npm name, a subpath included +(`import { SmtpTransport } from 'npm:@alexify/mailer/smtp'`), and grant what the +transports you use need: + +- `--allow-net` to send: SMTP, the HTTP providers, JMAP, `fromUrl` and the + certificates of SNS for `./webhooks`; +- `--allow-read` for `fromPath`; +- `--allow-run` and `--allow-env` for `./sendmail`, as Deno's `child_process` + passes the environment on; +- `--allow-sys=hostname` for the default EHLO name of `./smtp`; without it, the + name is `[127.0.0.1]` unless you set `localName`. + +Deno's `fetch` sends a request again when a reused connection closes before its +answer, so a message the provider took may go twice, where Node and Bun report it +as of unknown delivery. Give the HTTP transports and JMAP a `fetch` that keeps no +idle connection: + +```js +import { PostmarkTransport } from 'npm:@alexify/mailer/postmark'; + +const client = Deno.createHttpClient({ poolMaxIdlePerHost: 0 }); +const fetchOnce = (input, init) => fetch(input, { ...init, client }); + +const transport = new PostmarkTransport({ + serverToken: Deno.env.get('POSTMARK_TOKEN'), + fetch: fetchOnce, +}); +``` + +Each request then opens a connection of its own, which costs a handshake; see +[known issues](/guide/known-issues#deno). + ## Choosing a transport Each transport implements the same `Transport` contract of the root entry, so they diff --git a/docs/guide/known-issues.md b/docs/guide/known-issues.md index e91e54d..783f335 100644 --- a/docs/guide/known-issues.md +++ b/docs/guide/known-issues.md @@ -10,3 +10,14 @@ still open; the fixed ones are in the [changelog](https://github.com/Alexis-Tech size is not known ahead becomes a retryable network error, as Node's `fetch` reports an early answer to a streamed body so. Requests, and raw messages of a known size up to 4 MiB, go from a buffer and are not affected. + +## Deno + +- Deno's `fetch` (2.9.6) sends a request again when a reused HTTP/1.1 connection closes before + its answer. When the provider took the request before the connection was lost, the message + goes twice; on Node and Bun the same failure is reported as of unknown delivery + (`deliveryUnknown`) and not retried. Pass the HTTP transports and JMAP a `fetch` on a client + that keeps no idle connection, `Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`, as + [getting started](/guide/getting-started#bun-and-deno) shows. The providers that take an + idempotency key, which the transport sends with every message (Resend, Lettermint, Brevo, + Azure), see the second request as the first and do not send the message again. diff --git a/package.json b/package.json index eaf1af1..8059fdb 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "description": "Zero-dependency email for Node.js: its own SMTP client, MIME builder, DKIM, Amazon SES and seven more HTTP providers, JMAP, failover, retries and OpenTelemetry; a reliability layer (persistent queue, audit log, delivery events) is next", "license": "MIT", + "type": "commonjs", "author": "Alex Dolid ", "repository": { "type": "git", @@ -182,6 +183,9 @@ }, "scripts": { "test": "node --test tests/unit/*.test.js", + "test:runtime": "node --test tests/runtime/*.test.js", + "test:bun": "bun test --parallel --timeout 30000 tests/unit tests/runtime", + "test:deno": "deno test -A --no-check --parallel tests/unit/ tests/runtime/", "test:mailpit": "node --test --test-concurrency=1 tests/mailpit/*.test.js", "test:e2e": "node --test --test-concurrency=1 tests/e2e/*.test.js", "test:coverage": "c8 --all --include 'src/**' --check-coverage --lines 98 --statements 98 --branches 90 --functions 100 --reporter text --reporter lcov node --test tests/unit/*.test.js", @@ -197,7 +201,7 @@ "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs", "docs:comparison": "node scripts/comparison.js", - "prepublishOnly": "pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run test:types && pnpm run check:dts && pnpm size --check && pnpm run check:pack", + "prepublishOnly": "pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run test:runtime && pnpm run test:types && pnpm run check:dts && pnpm size --check && pnpm run check:pack", "release": "pnpm publish" }, "tsd": { diff --git a/tests/fixtures/tls/README.md b/tests/fixtures/tls/README.md new file mode 100644 index 0000000..1f169e7 --- /dev/null +++ b/tests/fixtures/tls/README.md @@ -0,0 +1,31 @@ + + +TLS test certificates +===================== + +`server-cert.pem` and `server-key.pem` are a certificate for `localhost`, +`127.0.0.1` and `::1`, issued by the test CA of `ca-cert.pem`, for the tests +that pass `ca` rather than `rejectUnauthorized: false`. Bun, on BoringSSL, does +not take a self-signed certificate that is not a CA, such as the one of +`../smtp`, as a root of trust, while Node and Deno do. They were made with + + openssl req -x509 -newkey rsa:2048 -nodes -days 36500 \ + -subj "/CN=mailer test CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout ca-key.pem -out ca-cert.pem + openssl req -newkey rsa:2048 -nodes -subj "/CN=localhost" \ + -keyout server-key.pem -out server.csr + openssl x509 -req -days 36500 -in server.csr -CA ca-cert.pem \ + -CAkey ca-key.pem -set_serial 1 -extfile server.ext -out server-cert.pem + +where `server.ext` holds + + basicConstraints=critical,CA:FALSE + keyUsage=critical,digitalSignature,keyEncipherment + extendedKeyUsage=serverAuth + subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1 + +The key of the CA was not kept; a new certificate means a new CA. The private +key of the server is intentionally committed and must never be used outside the +tests. diff --git a/tests/fixtures/tls/ca-cert.pem b/tests/fixtures/tls/ca-cert.pem new file mode 100644 index 0000000..3b28e5d --- /dev/null +++ b/tests/fixtures/tls/ca-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIUE3zC1qg99wRVRQfsHbI2KjM6OEEwDQYJKoZIhvcNAQEL +BQAwGTEXMBUGA1UEAwwObWFpbGVyIHRlc3QgQ0EwIBcNMjYxMDE3MjA0NDQ5WhgP +MjEyNjA5MjMyMDQ0NDlaMBkxFzAVBgNVBAMMDm1haWxlciB0ZXN0IENBMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxQ8j98qQUa3ytS7Ul4e6wdrgrRIo +5q9te04Mu7MbDySM1f2fuRZpPURuJts1Lgfdwy93ki3bjUC2KZXMsHhlTBkUGRct +V/dpcSt9h8FoXmAFRMqUU9LBNPBOFkeJ5MCUENc+FtPcsZASz2oc9WCw5XfG04Ze +2zDDsjxLHIRxyRlqLjWyS+sBLFChTbrK0pGx3i1XWYwPqD3X+4NaLWsYQOydV9Tg +qXCS4yjhXu3xEH69LkWoIOkE/LKKtapqq/Qiae5VyfKXa74G5GuxLDE6mg3fn0xQ +oo6anRSWA1ifddoRk0DWR2VBgpjCI4x2vGjNchGUxbjDneC2dAiNCqV2AwIDAQAB +o2MwYTAdBgNVHQ4EFgQUt7yJoXxg0ivdDhW4iYNeG6WUFYMwHwYDVR0jBBgwFoAU +t7yJoXxg0ivdDhW4iYNeG6WUFYMwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAQYwDQYJKoZIhvcNAQELBQADggEBAEuZ6ERD5W9Xup2HNYy+18+cDgTzREoD +orZ4p0pdRWy8lLWzZIxnB6cZS9ddwuhnlDfmr3zF7WN+c4iaMpxjCHC5XpFdIuyv +sFcLQY0FDTcRcKni5nqoQKB3b+M/JiN87oTea+teFoLJOZcLF5m9FxZE9u6APlR6 +plyMqMA3pnS3NxgfLTftovLsI2b9gO8MKTh+23gdLFDpwrVA4XFvZRoug06GH3Td +hnx2wUb3Oecocsr2sUn3wjvs5ne3tiBa0NbFhE/pc04c1tEmGBRWosKHfPBcYzml +R/2tLyaMFbHSLTUswecHExHRFz67X9m8/gWWHGeyRiE14kWl7hxJXS4= +-----END CERTIFICATE----- diff --git a/tests/fixtures/tls/server-cert.pem b/tests/fixtures/tls/server-cert.pem new file mode 100644 index 0000000..d2740c0 --- /dev/null +++ b/tests/fixtures/tls/server-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDTzCCAjegAwIBAgIBATANBgkqhkiG9w0BAQsFADAZMRcwFQYDVQQDDA5tYWls +ZXIgdGVzdCBDQTAgFw0yNjEwMTcyMDQ0NDlaGA8yMTI2MDkyMzIwNDQ0OVowFDES +MBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAq7etjvHVwIBkIvrDqd+ExpJn0I0FKf582PLapZqsSYhjBl4RGiV3w8ob/XF1 +doPnvMGaaq+3mK97FAFlfB+d2GJbqWrW7Vw8sCkw6IqB9R36JXOiGjNgMxr3n+JM +iGddOtOdtKkjOng8up95OVL3A00g6d7yU4B2K3V5C7/z1wecTIVpeOF/rVlhb0U9 +oYL9L0JITmmi6aL9XiTpcb26pwKnQ6jGWwVpGRqadvgs6PFINLyBpNOpcWJWVXuT +/nyBL7WkRhv/u0RpjAoYaSL6JLuRQnHLymwWmq2IbCPszUyZJZvDcXnxmMSadrdA +dGbMDu0qVFXPUMMa24oQrOvpCwIDAQABo4GkMIGhMAwGA1UdEwEB/wQCMAAwDgYD +VR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMCwGA1UdEQQlMCOCCWxv +Y2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATAdBgNVHQ4EFgQU7030NRbI +P/xSn9jDgnBmEMqhHQwwHwYDVR0jBBgwFoAUt7yJoXxg0ivdDhW4iYNeG6WUFYMw +DQYJKoZIhvcNAQELBQADggEBAMSmOHJ4uoRDfozsY0buF3OX1eQT1qJQzo3C01i2 +C7RjuPWQZpTwi8z+Y+j1xsIOezfSwfHP077D0sH6FNPlR6jWYEpyHZ/Wibd0srLR +6mzF0MXdLHdurj0PJFMGP3GZstDKwRyscHijV/Kpj4DEgBpuej7uG6jXxL1UcrvI +Rm0vfO0DxdzgxOG8VfVd5ADjPB8QlE0/sfNUIQMUVEseGTpTMILr2LwM4ORiZTIY +UfbMO4RL/PjQN1hh7AkCiOIaEOHhFgJNsEwpr+GG6f3irmQVtOcmHWTjSJUTkusE +Zx/t0yfXZUhY7drrdNrzb0r3nzMqmPdg+lhkRV7XP4srX/k= +-----END CERTIFICATE----- diff --git a/tests/fixtures/tls/server-key.pem b/tests/fixtures/tls/server-key.pem new file mode 100644 index 0000000..3021637 --- /dev/null +++ b/tests/fixtures/tls/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCrt62O8dXAgGQi ++sOp34TGkmfQjQUp/nzY8tqlmqxJiGMGXhEaJXfDyhv9cXV2g+e8wZpqr7eYr3sU +AWV8H53YYlupatbtXDywKTDoioH1Hfolc6IaM2AzGvef4kyIZ1060520qSM6eDy6 +n3k5UvcDTSDp3vJTgHYrdXkLv/PXB5xMhWl44X+tWWFvRT2hgv0vQkhOaaLpov1e +JOlxvbqnAqdDqMZbBWkZGpp2+Czo8Ug0vIGk06lxYlZVe5P+fIEvtaRGG/+7RGmM +ChhpIvoku5FCccvKbBaarYhsI+zNTJklm8NxefGYxJp2t0B0ZswO7SpUVc9Qwxrb +ihCs6+kLAgMBAAECggEAEjomgZhY/JyERJRCpar0sJFE1MXLSQha3zp0DAUNiaah +AEGG5jQY8N7otq/O7JwWwpn4Vp+9SMdDtL1rNNA0VI4oNaUCGGLkjDxrFRvyrH6Y +mVYbXFWcvf/8Teva/NWf90kUIeOcOi4eIB8iYnvUfXS37Cs+pK9rm7RV+3HOI8Jd +xPpd/fy9rNftCqFGTQf+29+WrV6kreYXVtycJrpmUh8uCnb+kmDdarTWkVKRUREn +BsbNlDoGv7ejf2vX2rqvZkDAAIhK8Xi0pqJftzbchSBCfpeK4RH+Okdg8s79im+D +eU/Tvs7fZOl1t4CEAFmE8acWCsrPmP0iShIbSge4BQKBgQDp9h/5p2dZsuMoNAtH +9rBnBbme4MXUi8zKnRCIsVfAVZwvo2xxQlh5hiuxKrLRlZbrPxWlfQ70Eg1cuscB +cjljDLx2PfmB5v1fNO/TDk8p0E43S8vWUUI6QbX6aXHfI6nBO2fqMg4bg51FkDHC +NCBtnIWFRDqH5sBUR6dobUfP1wKBgQC75JGGQNgX01ujxAA0U50jcVUx6+OnC0i6 +18AWOAidNn/VwVq/ESb4+UiL20Vs2xwYCRtf3KMXrMgrfBM7CNj2ir6C08aViYM4 +9BdLrmEUr7yBCEF6K5yqh1AWYoZ4HBUccW6PpH0IairSqpDFtBWGPWleAaGRDvos +MZPkSG+Z7QKBgQCeH0dLDrYN7JYUBOCe+X4NktP6KgXqebWmNnMHP3yENBgEJlpT +TbjIIUMEUUxb09MVGSTDHPtvP4giZOW893sLk38IFyi7f+c8YmW36hLcurkyLBsO +tQqinuij3oIkglDnlQPNR5k7RT+U1Tib/WjxS/j9sbgxQ2Y3T9o3/KutLwKBgQCj +wvOTL7EoEu8sLQiAkt/CpGS8vfwM+6jIMIMa2nuninigtYWCQlDVb2/rdZFs7wvt +ZwKU6pi7abyNKfcY3P085HBnVm0eqFvWp+gl7jZu8Tw6uhdXg+toNcd3uye4R8Ge +qL7UM2lGjEIGmuLTbcHU3nGXnZRT+dnR6uQbc4bRgQKBgQC7kqx/IAdTf8PyyBeG +wtJUPqrJR+FjV0LMC5gvEq0YomR8DBXabF8/hXhIJdRk6vJs9vs5wSqZ9PCadbqB +RUOoyz6/Pmhstu8cjaS2GcXI1RAABiYpmfWQsqIPOitX68dOiTPDUuXFaqGIVMgx +XlMiexUPmaB9mnlsdutaetYhsA== +-----END PRIVATE KEY----- diff --git a/tests/helpers/smtp/servers.js b/tests/helpers/smtp/servers.js new file mode 100644 index 0000000..85edae5 --- /dev/null +++ b/tests/helpers/smtp/servers.js @@ -0,0 +1,126 @@ +// Not ported: local servers for the SMTP tests: an SMTP server that answers a +// whole transaction over plain TCP (with AUTH offered in clear), implicit TLS +// or STARTTLS, and an HTTP or HTTPS proxy that tunnels CONNECT. +const fs = require('node:fs'); +const http = require('node:http'); +const https = require('node:https'); +const net = require('node:net'); +const path = require('node:path'); +const tls = require('node:tls'); + +// A certificate issued by a test CA: Bun takes no self-signed certificate +// that is not a CA as a root of trust. +const credentials = { + key: fs.readFileSync(path.join(__dirname, '../../fixtures/tls/server-key.pem')), + cert: fs.readFileSync(path.join(__dirname, '../../fixtures/tls/server-cert.pem')), +}; +const ca = [fs.readFileSync(path.join(__dirname, '../../fixtures/tls/ca-cert.pem'), 'utf8')]; + +const listen = (server, host = '127.0.0.1') => + new Promise((resolve) => server.listen(0, host, () => resolve(server.address().port))); +const close = (server) => + new Promise((resolve) => { + server.closeAllConnections?.(); + server.close(() => resolve()); + }); + +// An SMTP server answering a whole transaction; `mode` is 'plain' (with AUTH +// offered in clear), 'implicit' or 'starttls'. +const smtpServer = async (t, mode = 'plain') => { + // `data` holds each message as it came after DATA, dot-stuffing undone. + const state = { commands: [], messages: 0, data: [], sockets: new Set() }; + const handle = (socket, upgraded) => { + state.sockets.add(socket); + socket.on('error', () => {}); + if (!upgraded) socket.write('220 ready\r\n'); + let buffer = ''; + let inData = false; + let lines = []; + const onData = (bytes) => { + buffer += bytes.toString('latin1'); + let end; + while ((end = buffer.indexOf('\r\n')) >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + if (inData) { + if (line === '.') { + inData = false; + state.messages++; + state.data.push(lines.join('\r\n') + '\r\n'); + lines = []; + socket.write('250 queued\r\n'); + } else { + lines.push(line.startsWith('.') ? line.slice(1) : line); + } + continue; + } + state.commands.push(line); + const verb = line.split(' ')[0].toUpperCase(); + if (verb === 'EHLO') { + const extra = mode === 'starttls' && !upgraded ? 'STARTTLS' : 'AUTH PLAIN'; + socket.write(`250-hello\r\n250 ${extra}\r\n`); + } else if (verb === 'STARTTLS') { + socket.off('data', onData); + socket.write('220 go\r\n'); + const secure = new tls.TLSSocket(socket, { isServer: true, ...credentials }); + handle(secure, true); + return; + } else if (verb === 'DATA') { + inData = true; + socket.write('354 go\r\n'); + } else if (verb === 'QUIT') { + socket.end('221 bye\r\n'); + } else if (verb === 'AUTH') { + socket.write('235 ok\r\n'); + } else { + socket.write('250 ok\r\n'); + } + } + }; + socket.on('data', onData); + }; + const server = + mode === 'implicit' + ? tls.createServer(credentials, (socket) => handle(socket, false)) + : net.createServer((socket) => handle(socket, false)); + const port = await listen(server); + t.after(() => { + for (const socket of state.sockets) socket.destroy(); + return close(server); + }); + return { port, state }; +}; + +// An HTTP proxy that tunnels CONNECT to 127.0.0.1, or answers as `reply` says. +const proxyServer = async (t, { secure = false, reply } = {}) => { + const requests = []; + // A socket handed over to 'connect' is no longer the server's to close, + // and an HTTP server keeps it half open after the client leaves. + const tunnels = new Set(); + const server = secure ? https.createServer(credentials) : http.createServer(); + server.on('connect', (request, client, head) => { + requests.push({ url: request.url, headers: request.headers }); + tunnels.add(client); + client.on('error', () => {}); + if (reply !== undefined) { + reply(request, client); + return; + } + const port = Number(request.url.slice(request.url.lastIndexOf(':') + 1)); + const upstream = net.connect(port, '127.0.0.1', () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head.length > 0) upstream.write(head); + upstream.pipe(client); + client.pipe(upstream); + }); + upstream.on('error', () => client.destroy()); + }); + const port = await listen(server); + t.after(() => { + for (const tunnel of tunnels) tunnel.destroy(); + return close(server); + }); + return { port, requests, url: `${secure ? 'https' : 'http'}://127.0.0.1:${port}` }; +}; + +module.exports = { ca, credentials, close, listen, proxyServer, smtpServer }; diff --git a/tests/runtime/exports.test.js b/tests/runtime/exports.test.js new file mode 100644 index 0000000..a6bbdab --- /dev/null +++ b/tests/runtime/exports.test.js @@ -0,0 +1,21 @@ +// Not ported: every entry of the exports map, required and imported by name on +// the runtime that runs the test (Node, Bun or Deno), as a consumer would. +const { test } = require('node:test'); +const assert = require('node:assert'); +const pkg = require('../../package.json'); + +for (const subpath of Object.keys(pkg.exports)) { + const specifier = subpath === '.' ? pkg.name : `${pkg.name}/${subpath.slice(2)}`; + + test(`Runtime: ${specifier} is required, and imported by name from ESM`, async () => { + const required = require(specifier); + const imported = await import(specifier); + const names = Object.keys(required); + assert.ok(names.length > 0, `${specifier} exports nothing`); + for (const name of names) { + assert.notStrictEqual(imported[name], undefined, `${specifier} has no ESM export ${name}`); + assert.strictEqual(imported[name], required[name], name); + } + assert.strictEqual(imported.default, required); + }); +} diff --git a/tests/runtime/http.test.js b/tests/runtime/http.test.js new file mode 100644 index 0000000..e96d44a --- /dev/null +++ b/tests/runtime/http.test.js @@ -0,0 +1,171 @@ +// Not ported: an HTTP provider and ./attachments on the fetch of the runtime +// that runs the test (Node, Bun or Deno), with no mock, against a local server. +const { test } = require('node:test'); +const assert = require('node:assert'); +const http = require('node:http'); +const { createMessage, readAttachmentContent } = require('@alexify/mailer'); +const { PostmarkTransport } = require('@alexify/mailer/postmark'); +const { fromUrl } = require('@alexify/mailer/attachments'); + +// A server that records each request with its body, then answers as +// `answer(request, response, count)` says. +const server = async (t, answer) => { + const requests = []; + const sockets = new Set(); + const listener = http.createServer((request, response) => { + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + requests.push({ method: request.method, url: request.url, headers: request.headers, body }); + answer(request, response, requests.length); + }); + }); + listener.on('connection', (socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + t.after( + () => + new Promise((resolve) => { + for (const socket of sockets) socket.destroy(); + listener.close(() => resolve()); + }), + ); + return { requests, url: `http://127.0.0.1:${listener.address().port}` }; +}; + +const accepted = (response, id = 'pm-1') => { + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify({ MessageID: id, ErrorCode: 0, Message: 'OK' })); +}; + +const message = () => + createMessage({ + from: { name: 'Shop', address: 'shop@example.com' }, + to: 'bob@example.com', + subject: 'Grüße', + content: { text: 'Hello', html: '

Hello

' }, + attachments: [new File(['report'], 'report.txt', { type: 'text/plain' })], + }); + +const postmark = (url, config) => + new PostmarkTransport({ serverToken: 'pm-token', baseUrl: url, ...config }); + +// The fetch the documentation gives for Deno, whose own fetch sends a request +// again when a reused connection closes before its answer: a client that keeps +// no idle connection. +const fetchWithoutReuse = (t) => { + const client = globalThis.Deno.createHttpClient({ poolMaxIdlePerHost: 0 }); + t.after(() => client.close()); + return (input, init) => fetch(input, { ...init, client }); +}; + +test('Runtime HTTP: a request goes through the fetch of the runtime and its answer is read', async (t) => { + const { requests, url } = await server(t, (_request, response) => accepted(response)); + const receipt = await postmark(url).send(message()); + assert.deepStrictEqual(receipt, { successful: true, messageId: 'pm-1', provider: 'postmark' }); + const [request] = requests; + assert.strictEqual(request.method, 'POST'); + assert.strictEqual(request.url, '/email'); + assert.strictEqual(request.headers['x-postmark-server-token'], 'pm-token'); + const body = JSON.parse(request.body); + assert.strictEqual(body.From, 'Shop '); + assert.strictEqual(body.Subject, 'Grüße'); + assert.deepStrictEqual(body.Attachments[0].Name, 'report.txt'); + assert.strictEqual(Buffer.from(body.Attachments[0].Content, 'base64').toString(), 'report'); +}); + +test('Runtime HTTP: a redirect is a failure, and is not followed', async (t) => { + const { requests, url } = await server(t, (request, response) => { + if (request.url === '/elsewhere') return accepted(response); + response.writeHead(307, { location: '/elsewhere' }).end(); + }); + const receipt = await postmark(url).send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.retryable, false); + assert.deepStrictEqual( + requests.map((request) => request.url), + ['/email'], + ); +}); + +test('Runtime HTTP: an answer over 4 MiB is refused as it comes', async (t) => { + const chunk = Buffer.alloc(1024 * 1024, 'x'); + const { requests, url } = await server(t, (_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }); + for (let i = 0; i < 5; i++) response.write(chunk); + response.end(); + }); + const receipt = await postmark(url).send(message()); + // The provider answered 200, so it may have taken the message. + assert.strictEqual(receipt.deliveryUnknown, true); + assert.match(receipt.errorMessages[0], /4 MiB|4194304|too large/i); + assert.strictEqual(requests.length, 1); +}); + +test('Runtime HTTP: a request whose answer is lost is of unknown delivery, and goes once', async (t) => { + const { requests, url } = await server(t, (request) => request.socket.destroy()); + const receipt = await postmark(url).send(message()); + assert.strictEqual(receipt.deliveryUnknown, true); + assert.strictEqual(requests.length, 1); +}); + +test('Runtime HTTP: on a reused connection too, with the fetch the docs give for Deno', async (t) => { + const { requests, url } = await server(t, (request, response, count) => + count === 1 ? accepted(response) : request.socket.destroy(), + ); + const fetch = globalThis.Deno === undefined ? globalThis.fetch : fetchWithoutReuse(t); + const transport = postmark(url, { fetch }); + assert.strictEqual((await transport.send(message())).successful, true); + const lost = await transport.send(message()); + assert.strictEqual(lost.deliveryUnknown, true); + assert.strictEqual(requests.length, 2); +}); + +// Why the documentation gives Deno its own fetch: when this fails, Deno no +// longer sends a request twice, and the documentation can say so. +test( + "Runtime HTTP: Deno's own fetch sends a lost request again on a reused connection", + { + skip: globalThis.Deno === undefined && 'Deno only', + }, + async (t) => { + const { requests, url } = await server(t, (request, response, count) => + count === 1 ? accepted(response) : request.socket.destroy(), + ); + const transport = postmark(url); + assert.strictEqual((await transport.send(message())).successful, true); + const lost = await transport.send(message()); + assert.strictEqual(lost.deliveryUnknown, true); + assert.strictEqual(requests.length, 3); + }, +); + +test('Runtime attachments: fromUrl reads through the fetch of the runtime, checking each URL', async (t) => { + const { requests, url } = await server(t, (request, response) => { + if (request.url === '/moved') { + response.writeHead(302, { location: '/report.txt' }).end(); + return; + } + response.writeHead(200, { 'content-type': 'text/plain' }).end('report'); + }); + const allowed = []; + const allowUrl = (target) => { + allowed.push(target.pathname); + return true; + }; + const attachment = await fromUrl(`${url}/moved`, { allowUrl }); + assert.strictEqual( + new TextDecoder().decode(await readAttachmentContent(attachment.content)), + 'report', + ); + assert.strictEqual(attachment.filename, 'report.txt'); + assert.deepStrictEqual(allowed, ['/moved', '/report.txt']); + const before = requests.length; + await assert.rejects( + fromUrl(`${url}/moved`, { allowUrl: (target) => target.pathname === '/moved' }), + ); + assert.strictEqual(requests.length, before + 1); +}); diff --git a/tests/runtime/sendmail.test.js b/tests/runtime/sendmail.test.js new file mode 100644 index 0000000..c1703a4 --- /dev/null +++ b/tests/runtime/sendmail.test.js @@ -0,0 +1,66 @@ +// Not ported: the sendmail transport on the runtime that runs the test (Node, +// Bun or Deno), spawning a command of its own with no mock: a shell script +// that keeps its arguments and the message it reads, so POSIX only. +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { createMessage } = require('@alexify/mailer'); +const { SendmailTransport } = require('@alexify/mailer/sendmail'); + +const posix = { skip: process.platform === 'win32' && 'the command is a shell script' }; + +// A sendmail command that runs `body` after keeping its arguments and the +// message next to itself. +const command = (t, body = '') => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-sendmail-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const file = path.join(dir, 'sendmail'); + const script = `#!/bin/sh\nprintf '%s\\n' "$@" > "$0.args"\ncat > "$0.message"\n${body}\n`; + fs.writeFileSync(file, script, { mode: 0o755 }); + return file; +}; + +const message = () => + createMessage({ + from: 'shop@example.com', + to: 'bob@example.com', + bcc: 'audit@example.com', + subject: 'Hi', + content: { text: 'Hello\n.\nStill here' }, + }); + +test('Runtime sendmail: the message goes to stdin, the recipients after --', posix, async (t) => { + const file = command(t); + const receipt = await new SendmailTransport({ path: file }).send(message()); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + const args = fs.readFileSync(`${file}.args`, 'utf8').split('\n'); + assert.deepStrictEqual(args, [ + '-i', + '-f', + 'shop@example.com', + '--', + 'bob@example.com', + 'audit@example.com', + '', + ]); + const sent = fs.readFileSync(`${file}.message`, 'utf8'); + assert.match(sent, /^Subject: Hi$/m); + assert.match(sent, /^Still here$/m); + assert.doesNotMatch(sent, /^Bcc:/im); + assert.doesNotMatch(sent, /\r/); +}); + +test('Runtime sendmail: an exit code or a signal fails the send', posix, async (t) => { + const refused = command(t, 'echo "bob: user unknown" >&2\nexit 67'); + const receipt = await new SendmailTransport({ path: refused }).send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.errors[0].category, 'rejected'); + assert.match(receipt.errorMessages[0], /user unknown/); + // Killed after it read the whole message: it may have queued it. + const killed = command(t, 'kill -9 $$'); + const lost = await new SendmailTransport({ path: killed }).send(message()); + assert.strictEqual(lost.successful, false); + assert.strictEqual(lost.deliveryUnknown, true, lost.errorMessages.join()); +}); diff --git a/tests/runtime/smtp.test.js b/tests/runtime/smtp.test.js new file mode 100644 index 0000000..bd3a31c --- /dev/null +++ b/tests/runtime/smtp.test.js @@ -0,0 +1,125 @@ +// Not ported: the SMTP transport on the runtime that runs the test (Node, Bun +// or Deno), with no mock: its sockets, TLS and DKIM against local servers. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage } = require('@alexify/mailer'); +const { SmtpTransport, createHttpProxyConnector } = require('@alexify/mailer/smtp'); +const { ca, proxyServer, smtpServer } = require('../helpers/smtp/servers.js'); +const { verifyDkim } = require('../helpers/mime/verify-dkim.js'); +const keys = require('../helpers/mime/dkim-test-keys.js'); + +const message = (subject = 'Grüße') => + createMessage({ + from: 'shop@example.com', + to: ['bob@example.com', 'carol@bücher.de'], + subject, + content: { text: 'Line one\n.starts with a dot\n', html: '

Привіт

' }, + attachments: [new File(['report'], 'report.txt', { type: 'text/plain' })], + }); + +const sendWith = async (config, messages) => { + const transport = new SmtpTransport(config); + try { + const receipts = []; + for (const each of messages) receipts.push(await transport.send(each)); + return receipts; + } finally { + await transport.closeAllConnections(); + } +}; + +test('Runtime SMTP: signed with RSA and Ed25519, over one pooled connection with AUTH', async (t) => { + const { port, state } = await smtpServer(t); + const signatures = [ + { privateKey: keys.TEST_DKIM_PRIVATE_KEY, publicKey: keys.TEST_DKIM_PUBLIC_KEY }, + { + privateKey: keys.TEST_DKIM_ED25519_PRIVATE_KEY, + publicKey: keys.TEST_DKIM_ED25519_PUBLIC_KEY, + algorithm: 'ed25519-sha256', + }, + ]; + for (const { privateKey, publicKey, algorithm } of signatures) { + state.data.length = 0; + const receipts = await sendWith( + { + host: '127.0.0.1', + port, + auth: { user: 'user', pass: 'pässword' }, + dkim: { + signatures: [ + { + signingDomain: keys.TEST_DKIM_DOMAIN, + selector: keys.TEST_DKIM_SELECTOR, + privateKey, + algorithm, + }, + ], + }, + }, + [message('First'), message('Second')], + ); + for (const receipt of receipts) { + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + } + assert.strictEqual(state.data.length, 2); + for (const data of state.data) { + assert.match(data, new RegExp(`DKIM-Signature: v=1; a=${algorithm ?? 'rsa-sha256'};`)); + assert.match(data, /\.starts with a dot/); + verifyDkim(data, publicKey); + } + } + // Two messages per pooled connection: one greeting and one AUTH each time. + const ehlo = state.commands.filter((command) => command.startsWith('EHLO')); + assert.strictEqual(ehlo.length, 2); + assert.ok(state.commands.includes('RCPT TO:')); +}); + +test('Runtime SMTP: implicit TLS and STARTTLS trust the CA they are given', async (t) => { + for (const mode of ['implicit', 'starttls']) { + const { port, state } = await smtpServer(t, mode); + const [receipt] = await sendWith( + { + host: 'localhost', + port, + secure: mode === 'implicit', + requireTls: true, + tls: { ca }, + auth: { user: 'user', pass: 'pass' }, + }, + [message()], + ); + assert.strictEqual(receipt.successful, true, `${mode}: ${receipt.errorMessages?.join()}`); + assert.strictEqual(state.messages, 1, mode); + assert.ok( + state.commands.some((command) => command.startsWith('AUTH PLAIN')), + mode, + ); + } +}); + +test('Runtime SMTP: a certificate of an unknown CA is a failure that is not retried', async (t) => { + const { port, state } = await smtpServer(t, 'implicit'); + const [receipt] = await sendWith({ host: 'localhost', port, secure: true }, [message()]); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.retryable, false, receipt.errorMessages.join()); + assert.strictEqual(receipt.deliveryUnknown, undefined); + assert.strictEqual(state.commands.length, 0); +}); + +test('Runtime SMTP: the session goes through an HTTP CONNECT proxy', async (t) => { + const { port, state } = await smtpServer(t, 'starttls'); + const proxy = await proxyServer(t); + const [receipt] = await sendWith( + { + host: 'localhost', + port, + requireTls: true, + tls: { ca }, + connect: createHttpProxyConnector({ url: proxy.url }), + }, + [message()], + ); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(state.messages, 1); + assert.strictEqual(proxy.requests[0].url, `localhost:${port}`); +}); diff --git a/tests/unit/export-parity.test.js b/tests/unit/export-parity.test.js index e4bc645..9d77154 100644 --- a/tests/unit/export-parity.test.js +++ b/tests/unit/export-parity.test.js @@ -45,8 +45,11 @@ for (const { specifier, runtimeFile, dtsFile } of entryPoints) { assert.deepStrictEqual(phantom, [], `absent from ${runtimeFile} at runtime`); }); - test(`Exports: ${specifier} resolves to ${runtimeFile} and imports by name from ESM`, () => { + test(`Exports: ${specifier} resolves to ${runtimeFile} and imports by name from ESM`, (t) => { assert.strictEqual(require(specifier), require(path.join(ROOT, runtimeFile))); + // The import runs in a child of the node CLI, whose flags Deno does not + // take; tests/runtime imports by name in the process on every runtime. + if (globalThis.Deno !== undefined) return t.skip('spawns the node CLI'); const names = [...declaredValueExports(dtsFile)].sort(); const source = [ `import { ${names.join(', ')} } from '${specifier}';`, diff --git a/tests/unit/jmap-raw-message.integration.test.js b/tests/unit/jmap-raw-message.integration.test.js index a5a67a9..d4a8cb2 100644 --- a/tests/unit/jmap-raw-message.integration.test.js +++ b/tests/unit/jmap-raw-message.integration.test.js @@ -288,6 +288,13 @@ for (const encoding of ['7bit', undefined]) { }); } +// Deno's fetch sends a request again when a reused connection closes before +// its answer, so the server gets it twice; tests/runtime/http.test.js pins it, +// with the fetch the documentation gives for Deno. +const onDeno = (mode) => ({ + skip: mode === 'lost' && globalThis.Deno !== undefined && "Deno's fetch sends it again", +}); + for (const mode of [ 'lost', 'missing', @@ -298,7 +305,7 @@ for (const mode of [ 'partial', 'method-error', ]) { - test(`JMAP raw submission ${mode} has safe retry classification`, async () => { + test(`JMAP raw submission ${mode} has safe retry classification`, onDeno(mode), async () => { const context = await setup({ submitting: mode, timeout: 200 }); try { const receipt = await context.transport.sendRaw(raw); @@ -318,7 +325,7 @@ for (const mode of [ } for (const mode of ['lost', 'reject', 'duplicate', 'different-blob', 'partial']) { - test(`JMAP raw import ${mode} never reuses a different blob`, async () => { + test(`JMAP raw import ${mode} never reuses a different blob`, onDeno(mode), async () => { const context = await setup({ importing: mode }); try { const receipt = await context.transport.sendRaw(raw); diff --git a/tests/unit/ratelimit.test.js b/tests/unit/ratelimit.test.js index 7cf0f14..3d57e05 100644 --- a/tests/unit/ratelimit.test.js +++ b/tests/unit/ratelimit.test.js @@ -12,6 +12,8 @@ const message = (subject = 'Hello', to = 'b@example.com') => // A clock that moves the mocked timers with it. const timeline = (t) => { t.mock.timers.enable({ apis: ['setTimeout'] }); + // Node resets the timers after each test; Deno keeps them enabled. + t.after(() => t.mock.timers.reset()); let time = 0; return { now: () => time, diff --git a/tests/unit/smtp-connect.test.js b/tests/unit/smtp-connect.test.js index 422e7aa..3d77c6d 100644 --- a/tests/unit/smtp-connect.test.js +++ b/tests/unit/smtp-connect.test.js @@ -3,20 +3,10 @@ // server of plain TCP, implicit TLS or STARTTLS, and an HTTP or HTTPS proxy. const { test } = require('node:test'); const assert = require('node:assert'); -const fs = require('node:fs'); -const http = require('node:http'); -const https = require('node:https'); const net = require('node:net'); -const path = require('node:path'); -const tls = require('node:tls'); const { createMessage } = require('../../index.js'); const { SmtpTransport, createHttpProxyConnector } = require('../../src/smtp/index.js'); - -const credentials = { - key: fs.readFileSync(path.join(__dirname, '../fixtures/smtp/server-key.pem')), - cert: fs.readFileSync(path.join(__dirname, '../fixtures/smtp/server-cert.pem')), -}; -const ca = [credentials.cert.toString()]; +const { ca, close, listen, proxyServer, smtpServer } = require('../helpers/smtp/servers.js'); const message = () => createMessage({ @@ -26,107 +16,6 @@ const message = () => content: { text: 'Hi' }, }); -const listen = (server, host = '127.0.0.1') => - new Promise((resolve) => server.listen(0, host, () => resolve(server.address().port))); -const close = (server) => - new Promise((resolve) => { - server.closeAllConnections?.(); - server.close(() => resolve()); - }); - -// An SMTP server answering a whole transaction; `mode` is 'plain' (with AUTH -// offered in clear), 'implicit' or 'starttls'. -const smtpServer = async (t, mode = 'plain') => { - const state = { commands: [], messages: 0, sockets: new Set() }; - const handle = (socket, upgraded) => { - state.sockets.add(socket); - socket.on('error', () => {}); - if (!upgraded) socket.write('220 ready\r\n'); - let buffer = ''; - let inData = false; - const onData = (bytes) => { - buffer += bytes.toString('latin1'); - let end; - while ((end = buffer.indexOf('\r\n')) >= 0) { - const line = buffer.slice(0, end); - buffer = buffer.slice(end + 2); - if (inData) { - if (line === '.') { - inData = false; - state.messages++; - socket.write('250 queued\r\n'); - } - continue; - } - state.commands.push(line); - const verb = line.split(' ')[0].toUpperCase(); - if (verb === 'EHLO') { - const extra = mode === 'starttls' && !upgraded ? 'STARTTLS' : 'AUTH PLAIN'; - socket.write(`250-hello\r\n250 ${extra}\r\n`); - } else if (verb === 'STARTTLS') { - socket.off('data', onData); - socket.write('220 go\r\n'); - const secure = new tls.TLSSocket(socket, { isServer: true, ...credentials }); - handle(secure, true); - return; - } else if (verb === 'DATA') { - inData = true; - socket.write('354 go\r\n'); - } else if (verb === 'QUIT') { - socket.end('221 bye\r\n'); - } else if (verb === 'AUTH') { - socket.write('235 ok\r\n'); - } else { - socket.write('250 ok\r\n'); - } - } - }; - socket.on('data', onData); - }; - const server = - mode === 'implicit' - ? tls.createServer(credentials, (socket) => handle(socket, false)) - : net.createServer((socket) => handle(socket, false)); - const port = await listen(server); - t.after(() => { - for (const socket of state.sockets) socket.destroy(); - return close(server); - }); - return { port, state }; -}; - -// An HTTP proxy that tunnels CONNECT to 127.0.0.1, or answers as `reply` says. -const proxyServer = async (t, { secure = false, reply } = {}) => { - const requests = []; - // A socket handed over to 'connect' is no longer the server's to close, - // and an HTTP server keeps it half open after the client leaves. - const tunnels = new Set(); - const server = secure ? https.createServer(credentials) : http.createServer(); - server.on('connect', (request, client, head) => { - requests.push({ url: request.url, headers: request.headers }); - tunnels.add(client); - client.on('error', () => {}); - if (reply !== undefined) { - reply(request, client); - return; - } - const port = Number(request.url.slice(request.url.lastIndexOf(':') + 1)); - const upstream = net.connect(port, '127.0.0.1', () => { - client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); - if (head.length > 0) upstream.write(head); - upstream.pipe(client); - client.pipe(upstream); - }); - upstream.on('error', () => client.destroy()); - }); - const port = await listen(server); - t.after(() => { - for (const tunnel of tunnels) tunnel.destroy(); - return close(server); - }); - return { port, requests, url: `${secure ? 'https' : 'http'}://127.0.0.1:${port}` }; -}; - const send = async (config) => { const transport = new SmtpTransport({ pool: false, ...config }); try { diff --git a/tests/unit/smtp.test.js b/tests/unit/smtp.test.js index 3e19dcd..5020bb0 100644 --- a/tests/unit/smtp.test.js +++ b/tests/unit/smtp.test.js @@ -528,7 +528,11 @@ test('SMTP: implicit TLS and STARTTLS send the host name for SNI, and no IP addr await new Promise((resolve) => server.close(resolve)); } } - assert.deepStrictEqual(names, ['localhost', 'localhost', 'localhost', 'localhost', false, false]); + // A server reads no SNI as false on Node, undefined on Bun and null on Deno. + assert.deepStrictEqual( + names.map((name) => name || false), + ['localhost', 'localhost', 'localhost', 'localhost', false, false], + ); }); test('SMTP: implicit TLS is connected once its handshake completes, within the timeout', async () => { From b27abaea32674123406d13c2fd6c579cda143932 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 00:15:41 +0300 Subject: [PATCH 048/143] fix(smtp): a refused TLS certificate is a configuration failure A failure without a reply of the server was classified by the words of its message alone, so a certificate refused during the handshake was `unknown`. The receipt now takes classifyReceiptError of the error, by the code along its causes. classifyReceiptError took only the codes that name a certificate, which the leaf of an unknown issuer does not: UNABLE_TO_VERIFY_LEAF_SIGNATURE is what Node and Bun report for an untrusted CA. isCertificateCode, shared with isUnsentRequestError, knows it. Checked on Node, Bun and Deno: an unknown CA over implicit TLS and after STARTTLS, and a name the certificate does not hold, are each a non-retryable `configuration` failure. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 ++++++++ src/core/receipt.js | 10 +++++++++- src/internal/unsent.js | 6 +++--- src/smtp/smtp-transport.js | 8 ++++---- tests/runtime/smtp.test.js | 5 +++-- tests/unit/core.test.js | 12 +++++++----- tests/unit/internal.test.js | 3 ++- tests/unit/smtp.test.js | 31 +++++++++++++++++++++++++++++++ 8 files changed, 67 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ee6ac8d..d76c4e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -435,6 +435,14 @@ fails without it; ported tests that pinned the old behavior say "Changed after t 85. pool (known bug 7): `continueOnSuccess` was accepted and never read; honored, it would send a message that went out through one transport again through the others. It is gone from the type, and `continueOnSuccess: true` throws a `TypeError` when the pool is made. +86. smtp: a failure without a reply of the server was classified by the words of its message + alone, so a certificate refused during the TLS handshake was `unknown`; and + `classifyReceiptError` (36) took only the codes that name a certificate, which the leaf of an + unknown issuer does not (`UNABLE_TO_VERIFY_LEAF_SIGNATURE`, what Node and Bun report for an + untrusted CA). The receipt takes the classification of the error, by the code along its + causes, and `isCertificateCode` (`src/core/receipt.js`, shared with `isUnsentRequestError`) + knows that code: a refused certificate is a `configuration` failure, not retried, on Node, + Bun and Deno. ## Code ported from nodemailer, unemail and sently diff --git a/src/core/receipt.js b/src/core/receipt.js index 3dfdc72..b266cdb 100644 --- a/src/core/receipt.js +++ b/src/core/receipt.js @@ -117,6 +117,13 @@ const matchesWordRule = (rule, text, name) => { // The first `code` along the chain of causes, as `fetch failed` keeps the // socket error behind it. +// A certificate refused during the TLS handshake: the verification errors of +// OpenSSL that name one (CERT_HAS_EXPIRED, DEPTH_ZERO_SELF_SIGNED_CERT), Node's +// ERR_TLS_CERT_ALTNAME_INVALID, and the leaf of an unknown issuer, which names +// none and is the one Node and Bun report for a CA they do not know. +const isCertificateCode = (code) => + typeof code === 'string' && (code.includes('CERT') || code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE'); + const findErrorCode = (error) => { let current = error; for (let depth = 0; depth < 4 && current != null && typeof current === 'object'; depth++) { @@ -146,7 +153,7 @@ const classifyReceiptError = (error) => { return { category: 'network', retryable: true, code: 'network' }; } // A certificate refused during the TLS handshake will not pass on a retry. - if (typeof errorCode === 'string' && errorCode.includes('CERT')) { + if (isCertificateCode(errorCode)) { return { category: 'configuration', retryable: false, code: 'configuration' }; } @@ -313,5 +320,6 @@ module.exports = { classifyReceiptError, createReceiptError, createFailedReceipt, + isCertificateCode, parseRetryAfter, }; diff --git a/src/internal/unsent.js b/src/internal/unsent.js index 11b60c8..51f7048 100644 --- a/src/internal/unsent.js +++ b/src/internal/unsent.js @@ -1,6 +1,8 @@ // Not ported: tells a request that never left from one that may // have reached the server, for a request that must not be sent twice. +const { isCertificateCode } = require('../core/receipt.js'); + // System error codes of a connection that was never made, so no request went // out: a refused connection, an unknown or unreachable host. A certificate // rejected during the TLS handshake also ends the connection before it. @@ -10,11 +12,9 @@ const UNSENT_CODES = new Set([ 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', - 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', ]); -const isUnsentCode = (code) => - typeof code === 'string' && (UNSENT_CODES.has(code) || code.includes('CERT')); +const isUnsentCode = (code) => UNSENT_CODES.has(code) || isCertificateCode(code); /** * Whether a failed request certainly never reached the server: somewhere in diff --git a/src/smtp/smtp-transport.js b/src/smtp/smtp-transport.js index af3bb95..859acd3 100644 --- a/src/smtp/smtp-transport.js +++ b/src/smtp/smtp-transport.js @@ -834,10 +834,10 @@ const createSmtpFailure = (message, error) => { if (error instanceof SmtpResponseError || error instanceof SmtpAuthResponseError) { return createReplyFailure(message, error); } - return createFailedReceipt(message, { - provider: 'smtp', - attempts: 1, - }); + // No reply to read: what the error is, by the code along its causes (a + // refused certificate, a reset), and only then by its words. + const { category, retryable } = classifyReceiptError(error); + return createFailedReceipt(message, { provider: 'smtp', category, retryable, attempts: 1 }); }; // Failures found before or without a reply of the server, which no retry diff --git a/tests/runtime/smtp.test.js b/tests/runtime/smtp.test.js index bd3a31c..0c322da 100644 --- a/tests/runtime/smtp.test.js +++ b/tests/runtime/smtp.test.js @@ -97,11 +97,12 @@ test('Runtime SMTP: implicit TLS and STARTTLS trust the CA they are given', asyn } }); -test('Runtime SMTP: a certificate of an unknown CA is a failure that is not retried', async (t) => { +test('Runtime SMTP: a certificate of an unknown CA is a configuration failure', async (t) => { const { port, state } = await smtpServer(t, 'implicit'); const [receipt] = await sendWith({ host: 'localhost', port, secure: true }, [message()]); assert.strictEqual(receipt.successful, false); - assert.strictEqual(receipt.retryable, false, receipt.errorMessages.join()); + assert.strictEqual(receipt.errors[0].category, 'configuration', receipt.errorMessages.join()); + assert.strictEqual(receipt.retryable, false); assert.strictEqual(receipt.deliveryUnknown, undefined); assert.strictEqual(state.commands.length, 0); }); diff --git a/tests/unit/core.test.js b/tests/unit/core.test.js index 1dabf01..c4af9f1 100644 --- a/tests/unit/core.test.js +++ b/tests/unit/core.test.js @@ -306,11 +306,13 @@ test('Receipt: a failure is classified by its status, then its code, then its wo new TypeError('fetch failed', { cause: Object.assign(new Error(code), { code }) }); assert.strictEqual(classify(caused('ECONNREFUSED')), 'network'); assert.strictEqual(classify(caused('ETIMEDOUT')), 'timeout'); - assert.deepStrictEqual(classifyReceiptError(caused('CERT_HAS_EXPIRED')), { - category: 'configuration', - retryable: false, - code: 'configuration', - }); + for (const code of ['CERT_HAS_EXPIRED', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE']) { + assert.deepStrictEqual(classifyReceiptError(caused(code)), { + category: 'configuration', + retryable: false, + code: 'configuration', + }); + } }); test('Core: generateId makes the identifiers, and what it returns is checked', () => { diff --git a/tests/unit/internal.test.js b/tests/unit/internal.test.js index 8dda94d..982ad4e 100644 --- a/tests/unit/internal.test.js +++ b/tests/unit/internal.test.js @@ -7,7 +7,8 @@ const fetchFailed = (code) => new TypeError('fetch failed', { cause: Object.assign(new Error(code), { code }) }); test('Internal: a request whose connection was never made was not sent', () => { - for (const code of ['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'CERT_HAS_EXPIRED']) { + const codes = ['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'CERT_HAS_EXPIRED']; + for (const code of [...codes, 'UNABLE_TO_VERIFY_LEAF_SIGNATURE']) { assert.strictEqual(isUnsentRequestError(fetchFailed(code)), true, code); } const wrapped = new Error('JMAP request failed', { cause: fetchFailed('ECONNREFUSED') }); diff --git a/tests/unit/smtp.test.js b/tests/unit/smtp.test.js index 5020bb0..2139c1c 100644 --- a/tests/unit/smtp.test.js +++ b/tests/unit/smtp.test.js @@ -12,6 +12,7 @@ const { checkSmtpPath } = require('../../src/smtp/envelope.js'); const { OAuth2TokenManager } = require('../../src/smtp/oauth2.js'); const { SmtpConnection } = require('../../src/smtp/smtp-connection.js'); const { MockSmtpServer } = require('../helpers/smtp/mock-smtp-server.js'); +const { ca, smtpServer } = require('../helpers/smtp/servers.js'); const message = () => createMessage({ @@ -834,3 +835,33 @@ test('SMTP: mail sent in plain text to another host is warned of, to the logger await server.stop(); } }); + +test('SMTP: a certificate refused in the handshake is a configuration failure, not retried', async (t) => { + const { connect } = require('node:net'); + const cases = [ + // The CA is not trusted: UNABLE_TO_VERIFY_LEAF_SIGNATURE, which names no + // certificate, over implicit TLS and after STARTTLS. + { mode: 'implicit', config: { host: 'localhost', secure: true } }, + { mode: 'starttls', config: { host: 'localhost', requireTls: true } }, + // Trusted, for another name: ERR_TLS_CERT_ALTNAME_INVALID. + { mode: 'implicit', config: { host: 'mail.example.test', secure: true, tls: { ca } } }, + ]; + for (const { mode, config } of cases) { + const { port, state } = await smtpServer(t, mode); + const transport = new SmtpTransport({ + ...config, + port, + pool: false, + connect: () => connect(port, '127.0.0.1'), + }); + const receipt = await transport.send(message()); + await transport.closeAllConnections(); + // Changed after the port: the receipt was `unknown`, by the words of the + // message alone. + assert.strictEqual(receipt.errors[0].category, 'configuration', receipt.errorMessages[0]); + assert.strictEqual(receipt.errors[0].code, 'configuration'); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(receipt.deliveryUnknown, undefined); + assert.strictEqual(state.messages, 0); + } +}); From 66b3a78d5aa21120793cbed8f07b5671c3b6e816 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 00:24:19 +0300 Subject: [PATCH 049/143] docs: 1.1 in the README, the guides, the comparison and the changelog The README, the home page, why, getting started and the API page name what 1.1 adds: sendmail, eighteen HTTP providers, ./attachments, ./render, ./webhooks with the delivery events, and the breaker, rate limit, idempotency, hooks and tee decorators, with the credits of nodemailer, unemail and sently. The column of @alexify/mailer in docs/comparison.json follows the code, counted from it (scheduling in 8 providers, tracking in 10, metadata in 17, verify in 13), and the table is regenerated. CHANGELOG [Unreleased] holds the release: what changes from 1.0 (the EHLO name, maxMessages, encodings by content, capitalized field names, the DKIM h=, the classification of refusals and certificates, Deno's fetch), what is added, the security changes and the fixes. SECURITY.md completes the threat model with provider requests, attachments, sendmail and webhooks; llms.txt, package.json, CLAUDE.md (what this is, status, layout, roadmap) follow. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 129 ++++++++++++++++++++++++++++++++++ CLAUDE.md | 68 +++++++++++------- README.md | 71 +++++++++++-------- SECURITY.md | 28 +++++--- docs/api/exports.md | 13 ++-- docs/comparison.json | 98 +++++++++++++------------- docs/guide/comparison.md | 85 ++++++++++++---------- docs/guide/getting-started.md | 67 ++++++++++++------ docs/guide/why.md | 39 +++++----- docs/index.md | 33 +++++---- docs/public/llms.txt | 29 +++++--- package.json | 10 ++- 12 files changed, 449 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2bd2c..6b1a90b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,135 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +The features of three libraries of the [comparison](https://mailerat.vercel.app/guide/comparison) +where they led this package, ported onto its contracts: +[nodemailer](https://github.com/nodemailer/nodemailer) 10.0.16 (MIT-0, © 2011-2023 Andris +Reinman), [unemail](https://github.com/productdevbook/unemail) 0.7.0 (MIT, © 2025 Wind) and +[sently](https://github.com/omqkhafi/sently) 1.2.2 (MIT, © 2026 sendx contributors). What is +ported is the feature, the mapping of a provider's fields, an algorithm or a test case; the HTTP +layer, the retries and the receipts are this package's. The known bugs of the sources are fixed in +the port, each with a test that fails on the source's behavior, and listed in `CLAUDE.md`. The +release adds sendmail, ten HTTP providers, attachments from paths and URLs, templates, verified +webhooks into one model of delivery events, five decorators, and Bun and Deno in CI. Everything is +additive but the defaults listed below. + +### Upgrading from 1.0 + +- **EHLO names the machine:** the default is its host name when that is a fully qualified domain + name, its address otherwise (`[127.0.0.1]` at worst), no longer `localhost`, which servers such + as Postfix with `reject_non_fqdn_helo_hostname` refuse. Set `localName` to choose it. +- **A pooled SMTP connection carries at most 100 messages** (`maxMessages`), then is closed with + `QUIT` and replaced; `Infinity` keeps the old behavior. +- **Bodies are encoded by their content:** 7bit for short ASCII lines, quoted-printable for mostly + Latin text, base64 otherwise (Cyrillic, CJK), always with CRLF line breaks. Every body was + quoted-printable. +- **Custom header fields are written capitalized** (`X-Custom-Header`, `List-Unsubscribe-Post`) + rather than in lower case. Field names are case-insensitive, so readers and DKIM see no + difference. +- **The default DKIM `h=` signs `List-Unsubscribe` and `List-Unsubscribe-Post`** too, as + RFC 8058 asks. +- **Failures are classified more precisely:** a message a provider's limits refuse before any + request is a non-retryable `validation` failure, and a TLS certificate refused in the handshake + of `./smtp` a non-retryable `configuration` failure, where both were `unknown`. +- **On Deno, give the HTTP transports a `fetch` that keeps no idle connection** + (`Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`): Deno's own sends a request again when a + reused connection closes before its answer. See + [known issues](https://mailerat.vercel.app/guide/known-issues#deno). + +### Added + +- **Internationalized domains as A-labels:** `user@bücher.de` goes out as + `user@xn--bcher-kva.de` in header fields, SMTP envelopes, JMAP and every provider, so it needs + no SMTPUTF8; a local part beyond ASCII still does. The `Message` keeps the address as given. +- **`list` on the message:** the `List-Id` of RFC 2919, the URL fields of RFC 2369 (help, + unsubscribe, subscribe, post, owner, archive), and with `oneClick` the `List-Unsubscribe-Post` + of RFC 8058 that Gmail and Yahoo ask of bulk senders. Formatted as nodemailer does; a URL with + whitespace, controls or angle brackets is refused, and one-click needs an `https` URL. +- **`scheduledAt`, `tracking` and `metadata` on the message,** from unemail's model: when the + provider should deliver it, open and click tracking for this message, and values the provider + returns in its events. A transport that cannot schedule refuses the message with a + `validation` receipt rather than sending it at once. Resend, SendGrid, Mailgun, Maileroo, + Brevo, MailerSend, SparkPost and SMTP2GO schedule; ten providers take `tracking`, seventeen + `metadata` (each page says how). +- **`./attachments`:** `fromPath`, `fromUrl`, `fromDataUri`, `fromStream` and `inlineDataUrls`, + for what nodemailer read through `path`, `href` and data URIs. Nothing is read implicitly: + `fromUrl` goes through an injected `fetch`, checks each redirect with `allowUrl` (by default no + `localhost`, loopback, private or link-local address), and bounds time and bytes; `fromPath` + can be held to a `root` by real path. +- **`./render`:** `renderContent(renderer, input)` for the content of `createMessage`, + `createReactEmailRenderer({ render })` with the `render` of `@react-email/render` passed in, and + `htmlToText`, linear in the length of its input. +- **`./smtp` connects through what you give it:** `connect` opens each connection (a proxy, a + tunnel, a socket of another module) with implicit TLS or STARTTLS over it; + `createHttpProxyConnector` makes one for an HTTP or HTTPS proxy; `localAddress` picks the address + to connect from. +- **`./sendmail`:** the sendmail command of the system (Postfix, Exim, msmtp), one process per + message, composed by `./mime` with DKIM; `sendRaw` and `verify`. Only an exit with 0 succeeds, a + signal after the whole message is `deliveryUnknown`, and the sysexits codes are classified. +- **Raw messages and `verify()` through every decorator:** `./pool`, `./retry`, `./otel`, + `./logger` and `./mock` pass `sendRaw` and `verify` on. `./ses` and `./mailgun` send raw MIME, + and `./ses`, `./sendgrid`, `./resend` and `./mailgun` verify their key, one that may only send + included. +- **Ten HTTP providers:** `./postmark` (with `/email/batch`), `./brevo` (an idempotency key on + every request), `./mailjet` (the Send API v3.1 batch), `./mailersend`, `./sparkpost` (US or EU, + raw MIME), `./zeptomail`, `./smtp2go` (US, EU or AU), `./azure` (Azure Communication Services, + signed with the access key, repeatable requests), `./scaleway` and `./cloudflare` (the REST API + of Cloudflare Email Service, raw MIME). Each goes through the request layer of the others: no + redirect followed, a bounded body, `deliveryUnknown` when the provider may have taken the + message. +- **`./breaker`:** a circuit breaker that opens after `failureThreshold` failures of the + provider, answers with a retryable `service-unavailable` receipt while open (which `./pool` + fails over with), and lets exactly one probe through after `resetTimeoutMs`. +- **`./ratelimit`:** a token bucket per `key`, waiting first in, first out, or refusing with a + retryable `rate-limit` receipt; an aborted send leaves the queue, and a cost over `burst` is + refused. +- **`./idempotency`:** a key reserved in a store before the send, so a repeat is answered with + the first receipt, a send in flight as of unknown delivery, and the same key for another + message refused. `createMemoryIdempotencyStore` is the in-process store; a shared one implements + `reserve`, `complete` and `release`. +- **`./hooks`:** `beforeSend` may replace or refuse a message, `afterSend`, `onRetry` and + `onError` hear each send, with a context made for each. +- **`./tee`:** mirrors each message to other transports, each with a timeout of its own and not + awaited by default, the primary's receipt the one returned. +- **Delivery events and `./webhooks`:** the `DeliveryEvent` model at the root, one event per + message and recipient, and a verifier per provider for SES, Resend, SendGrid, Mailgun, + Postmark, MailerSend, Mailtrap, Brevo, Mailjet, SparkPost, Lettermint, Maileroo, SMTP2GO, Azure + and Scaleway. Nothing is read before it is verified: a signature where the provider signs (SNS + with a topic allowlist, Svix, ECDSA, HMAC), its Basic credentials or token where it does not, + within a window of time and once by its id (`createMemoryReplayStore`, or any store). +- **Bun and Deno:** CI runs the unit suite and the new `tests/runtime` (the package by its name, + with no mock: sockets, TLS, DKIM, `fetch`, a child process) on Bun 1.4.3 and Deno 2.9.6, as + well as on Node 22 and 24. `package.json` declares `"type": "commonjs"`, which Deno needs and + Node assumes. +- **Documentation** for every new subpath, a guide to webhooks and events, Bun and Deno in + getting started, and the comparison updated. + +### Security + +- **An attachment reads no file or URL it was not given by the application** (nodemailer read + any `path` or `href`, following redirects to any address, cloud metadata included), and a data + URI that does not parse is refused rather than read as a path. +- **A List-Unsubscribe URL or a metadata entry cannot add a header field:** a line break, + whitespace or angle bracket in a URL is refused, and a metadata key is held to + `[A-Za-z0-9_.-]`, its value to a string without control characters. +- **A DKIM signing domain or selector with a tab, CR or LF is refused** before it reaches the + host parser, which removed them and let the name, line break and all, into the + `DKIM-Signature` field (the fix of 1.0 missed it). +- **The webhooks of a provider are verified before they are read,** where sently parsed SES, + SendGrid, Postmark and Brevo events unverified and checked Resend and Mailgun with no window + of time or replay check. + +### Fixed + +- **A TLS certificate refused in the handshake was `unknown` in an SMTP receipt,** classified by + the words of its message alone; and `classifyReceiptError` missed + `UNABLE_TO_VERIFY_LEAF_SIGNATURE`, what Node and Bun report for an untrusted CA. Both are now a + non-retryable `configuration` failure. +- **A message refused by a provider's limits before any request was `unknown`;** it is a + non-retryable `validation` failure. +- **A pooled SMTP connection was reused for any number of messages,** while servers limit the + messages of one connection (`maxMessages`, above). + ## [1.0.0] - 2026-10-17 The first release: the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0 diff --git a/CLAUDE.md b/CLAUDE.md index d76c4e2..788ba29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,18 +10,24 @@ Nodemailer, emailjs or Upyo, made of two parts: 1. **The transport,** ported from upyo 0.7.0 (see "Code ported from upyo"): the message model, its own SMTP client, MIME builder and DKIM signing, Amazon SES v2 and seven more HTTP providers on `fetch`, JMAP, and the pool, retry, mock, OpenTelemetry and logger decorators, one subpath - each. + each. 1.1 adds what nodemailer, unemail and sently did better (see "Code ported from + nodemailer, unemail and sently"): sendmail, an injected `connect` and an HTTP CONNECT proxy for + SMTP, ten more HTTP providers, attachments from paths and URLs, List-* fields with one-click, + templates with React Email passed in, and the circuit breaker, rate limit, hooks and tee + decorators. 2. **A reliability layer on top of it,** which none of the fourteen libraries in `docs/comparison.json` has: a persistent queue and outbox, an append-only audit log, one model - of delivery events, suppression, idempotency and failover between providers. + of delivery events, suppression, idempotency and failover between providers. 1.1 has the model + of delivery events with the verified webhooks of fifteen providers (`./webhooks`) and the + idempotency decorator whose store interface the outbox is to implement (`./idempotency`). It has zero runtime dependencies and runs on Node.js ≥ 22, and on Bun and Deno through their Node compatibility (see "Platform"; no browser entry). -**Status: 1.0.0 is the ported transport, the reliability layer is being designed.** The root -entry exports the message model and the transport contract of upyo, each transport and decorator -is a subpath, and their API is under semver from 1.0.0. The reliability layer (queue, audit log, delivery events) has no API yet; its direction -is in "Roadmap" below. +**Status: 1.1 is the transport and the first parts of the reliability layer.** The root entry +exports the message model and the transport contract of upyo, with the delivery events of 1.1; +each transport, decorator and helper is a subpath, and their API is under semver from 1.0.0. The +queue, the audit log and suppression have no API yet; their direction is in "Roadmap" below. It is maintained by Alexis Technologies and set up like the sibling libraries `@alexify/metaschema` and `@alexify/protoarray` (the scaffold follows protoarray's @@ -637,10 +643,13 @@ src/ index.js public barrel (explicit named exports) version.js the package name and version (OpenTelemetry scope), kept equal to package.json / one directory per subpath: index.js + index.d.ts + its modules - internal/ private modules shared by subpaths, not in the exports map (redact.js, - redact-addresses.js, unsent.js, delivery-unknown.js, http.js, - config.js, loopback.js, body.js, sleep.js, log.js, base64.js, - headers.js, provider.js, batch.js, validate-ssl.js) + internal/ private modules shared by subpaths, not in the exports map: the request + layer (http.js, api.js for the providers of 1.1, body.js, unsent.js, + delivery-unknown.js, config.js, loopback.js, validate-ssl.js, verify.js), + receipts (provider.js, refusal.js), message fields (headers.js, raw.js, + schedule.js, tracking.js, capabilities.js, batch.js, base64.js, uuid.js), + the decorators of 1.1 (gate.js, callback.js, options.js), and redact.js, + redact-addresses.js, sleep.js, log.js tests/ unit/*.test.js node:test suites, one per area (-.test.js for a subpath) runtime/*.test.js the package by its name on Node, Bun and Deno, with no mock @@ -705,9 +714,10 @@ model of states and events. `tenantId` is in the context from the start. **Phase 1, reliable sending (P0).** - **The transport of its own** (done, ported from upyo 0.7.0; its known bugs are on the list of - "Code ported from upyo"): the MIME builder; the SMTP client (pool, STARTTLS with `requireTls`, not RFC 8689 REQUIRETLS, - AUTH PLAIN/LOGIN/XOAUTH2, DSN, PIPELINING); DKIM (RSA-SHA256, Ed25519); SES v2 over `fetch` with - SigV4; seven more HTTP providers and JMAP. + "Code ported from upyo"): the MIME builder; the SMTP client (pool, STARTTLS with `requireTls`, + not RFC 8689 REQUIRETLS, AUTH PLAIN/LOGIN/XOAUTH2, DSN, PIPELINING); DKIM (RSA-SHA256, Ed25519); + SES v2 over `fetch` with SigV4; seven more HTTP providers and JMAP. 1.1 added sendmail and ten + providers. - **A persistent queue:** the outbox in MongoDB (transactional outbox, no Redis needed) and a BullMQ driver (Valkey) for throughput and delays; priorities (auth codes before newsletters). - **The audit log** of every state, with a keyed HMAC plus an encrypted block for the recipient @@ -716,7 +726,8 @@ model of states and events. `tenantId` is in the context from the start. - **Idempotency:** a deterministic key (tenant + purpose + recipient + window) deduplicated in the outbox. At least once with deduplication, never a promise of exactly once. A provider without an idempotency key (SES) leads to `delivery_unknown` and reconciliation from events, not to a blind - retry. + retry. `./idempotency` (1.1) reserves keys in a store of `reserve`, `complete` and `release`, + which the outbox is to implement. - **Errors classified:** permanent (5xx, rejected address) are not retried; temporary ones are, with jitter and `Retry-After`, under an attempt limit. - **The correctness suite:** golden `.eml` files, DKIM checked by an independent verifier in CI, @@ -725,19 +736,20 @@ model of states and events. `tenantId` is in the context from the start. wrong result). **Phase 2, a closed loop (P1).** Webhooks of the providers into the one event model, with the -signature mandatory and replay protection, starting with SES (SNS) and one HTTP provider; a +signature mandatory and replay protection (done in 1.1: `./webhooks`, fifteen providers); a suppression list per tenant and stream, checked before a message is queued, with a bypass policy for transactional mail (password reset, MFA); failover over a pool of providers with health, a circuit breaker with a single probe in half-open, and no switch after an ambiguous "submitted" -without an idempotency key (`./pool` of the port has strategies and failover, not yet health or a -breaker); OpenTelemetry spans and metrics (time in queue, attempts, failure rate per provider, -outbox size; `./otel` of the port covers sends); token-bucket rate limits per provider and per -tenant (SES quotas). +without an idempotency key (`./pool` has strategies and failover, `./breaker` of 1.1 takes a +failing provider out of it; no health checks yet); OpenTelemetry spans and metrics (time in +queue, attempts, failure rate per provider, outbox size; `./otel` covers sends); token-bucket rate +limits per provider and per tenant (SES quotas; `./ratelimit` of 1.1, by any key). **Phase 3, multi-tenant (P1–P2).** Provider credentials per tenant (bring your own SES), domains -and DKIM per tenant, quotas; List-Unsubscribe one-click (RFC 8058) and the split between -transactional and bulk mail; scheduled sending as a queue delay; a `Renderer` interface with an -optional React Email adapter (no template engine of its own); more providers. +and DKIM per tenant, quotas; List-Unsubscribe one-click (RFC 8058, done in 1.1) and the split +between transactional and bulk mail; scheduled sending as a queue delay (1.1 schedules through the +providers that can); a `Renderer` interface with an optional React Email adapter (no template +engine of its own; done in 1.1, `./render`); more providers (done in 1.1). **Decisions:** @@ -793,9 +805,9 @@ differs on them, as of those versions: trust, as Node and Deno do, so the tests that pass `ca` use `tests/fixtures/tls`. There is no `browser.js` and no `browser` field: sending from a browser would expose the -credentials, and SMTP needs sockets. `src/` may use `node:*` modules freely. Edge runtimes are not a target for -now; if they become one, the transport over `fetch` and the core would move behind a separate -entry with no `node:*`, and a platform test would enforce it as in protoarray. +credentials, and SMTP needs sockets. `src/` may use `node:*` modules freely. Edge runtimes are not +a target for now; if they become one, the transport over `fetch` and the core would move behind a +separate entry with no `node:*`, and a platform test would enforce it as in protoarray. ## Conventions (oxlint/oxfmt + review) @@ -918,8 +930,10 @@ published to npm, and lint/format do not cover it. - **The guides of upyo are adapted, not generated:** `docs/messages/` (compose, attachments, calendar, MIME) and `docs/transports/` (one page per subpath, plus custom transports), with examples in CommonJS on `@alexify/mailer/`. `/transports/logger` is new (pino-like - logger) and `/transports/otel` passes the providers in. One sidebar (`guide` in - `config.mts`) covers the guide, the messages and the transports. + logger) and `/transports/otel` passes the providers in. The pages of 1.1 are new: + `/messages/render`, `/guide/webhooks`, a page per provider and decorator, and Bun and Deno in + getting started. One sidebar (`guide` in `config.mts`) covers the guide, the messages, the + transports and the decorators. - When behavior changes, update README and the matching docs page together. Keep example outputs in the docs true to the code. diff --git a/README.md b/README.md index c945353..46138ff 100644 --- a/README.md +++ b/README.md @@ -8,37 +8,47 @@ [![license](https://img.shields.io/npm/l/%40alexify%2Fmailer)](./LICENSE) **Email for Node.js, with a reliability layer.** Its own SMTP client, MIME builder, DKIM signing -and HTTP providers, and on top of them what the other libraries leave to the application: a -persistent queue and outbox, an audit log, one model of delivery events, suppression and -idempotency. Zero runtime dependencies. - -> **Status: 1.0 is the transport layer,** ported from [upyo](https://github.com/dahlia/upyo) 0.7.0: -> messages, MIME and DKIM, SMTP, Amazon SES and seven more HTTP providers, JMAP, and the pool, -> retry, OpenTelemetry, logger and mock transports. The reliability layer (queue, audit log, -> delivery events) is being designed and comes in later releases. +and eighteen HTTP providers, and on top of them what the other libraries leave to the +application: one model of delivery events, idempotency, and next a persistent queue and outbox, an +audit log and suppression. Zero runtime dependencies. + +> **Status: 1.1 is the transport layer and the first parts of the reliability layer.** 1.0 ported +> the transport from [upyo](https://github.com/dahlia/upyo) 0.7.0; 1.1 adds what +> [nodemailer](https://github.com/nodemailer/nodemailer), +> [unemail](https://github.com/productdevbook/unemail) and +> [sently](https://github.com/omqkhafi/sently) did better: sendmail, ten more HTTP providers, +> attachments from paths and URLs, one-click List-Unsubscribe, verified webhooks into one model of +> delivery events, and decorators for circuit breaking, rate limits, idempotency, hooks and +> mirroring. The persistent queue, the audit log and suppression are being designed. ### 📖 [Read the documentation →](https://mailerat.vercel.app/) ## Why @alexify/mailer -- **Its own transport.** An SMTP client (pool, STARTTLS, PIPELINING, DSN, SMTPUTF8, OAuth 2.0), a - MIME builder and DKIM (RSA-SHA256, Ed25519), and HTTP providers on `fetch`: Amazon SES v2, - Resend, SendGrid, Mailgun, Mailtrap, Maileroo, Lettermint, Plunk, and JMAP. Secure by default: - TLS certificates are verified, AUTH never goes over plain text, CRLF in a header is rejected. -- **What happens after `send()`** (being designed). A persistent queue and outbox, an append-only - audit log and one model of delivery events (delivered, bounced, complained, failed). None of the +- **Its own transport.** An SMTP client (pool, STARTTLS, PIPELINING, DSN, SMTPUTF8, OAuth 2.0, an + HTTP CONNECT proxy or any socket you connect), sendmail, a MIME builder (IDN domains as + punycode, 7bit, quoted-printable or base64 by content) and DKIM (RSA-SHA256, Ed25519), JMAP, and + eighteen HTTP providers on `fetch`: Amazon SES v2, Resend, SendGrid, Mailgun, Postmark, Brevo, + Mailjet, MailerSend, SparkPost, Mailtrap, Maileroo, Lettermint, ZeptoMail, SMTP2GO, Azure + Communication Services, Scaleway, Cloudflare and Plunk. Secure by default: TLS certificates are + verified, AUTH never goes over plain text, CRLF in a header is rejected, and no file or URL is + read for an attachment unless you ask. +- **What happens after `send()`.** The webhooks of fifteen providers, each verified before it is + read, give one model of delivery events (delivered, bounced, complained, …). Next: a persistent + queue and outbox, an append-only audit log and suppression. None of the [fourteen libraries compared](https://mailerat.vercel.app/guide/comparison) has the three. - **Sent once, or known to be unknown.** Errors are classified as permanent or temporary, and a failure after which the provider may have taken the message is marked `deliveryUnknown`: - `./retry` and `./pool` never send it again. Idempotency keys with deduplication come with the - outbox. -- **Correct before broad.** The tests read the composed messages back with an independent parser; - golden `.eml` messages, DKIM checked by an independent verifier and fuzzing of headers, - `multipart/related`, dot-stuffing and internationalized addresses are planned. + `./retry` and `./pool` never send it again. `./idempotency` reserves a key before the send, + `./breaker` lets one probe through a failing provider, and `./ratelimit` keeps to its quota. +- **Correct before broad.** The tests read the composed messages back with an independent parser + and check DKIM with an independent verifier, and the suites run on Node 22 and 24, Bun and Deno; + golden `.eml` messages and fuzzing of headers, `multipart/related`, dot-stuffing and + internationalized addresses are planned. - **Zero dependencies.** Built on `node:*` modules and `fetch`. What a part needs from your - application is passed in: the OpenTelemetry providers, a pino-like logger, and later the driver of - a queue or a database (BullMQ, MongoDB). CommonJS with ESM named imports and hand-written - TypeScript typings, no build step. + application is passed in: the OpenTelemetry providers, a pino-like logger, the `render` of React + Email, and later the driver of a queue or a database (BullMQ, MongoDB). CommonJS with ESM named + imports and hand-written TypeScript typings, no build step. ## Installation @@ -79,11 +89,13 @@ if (!receipt.successful) console.error(receipt.errorMessages); | Subpath | What it is | | --- | --- | -| `@alexify/mailer` | messages, attachments, calendar parts, the `Transport` contract, receipts | +| `@alexify/mailer` | messages, attachments, calendar parts, the `Transport` contract, receipts, delivery events | | `/mime` | MIME composition and DKIM signing | -| `/smtp`, `/jmap` | SMTP and JMAP transports | -| `/ses`, `/resend`, `/sendgrid`, `/mailgun`, `/mailtrap`, `/maileroo`, `/lettermint`, `/plunk` | HTTP providers | -| `/pool`, `/retry`, `/otel`, `/logger` | decorators: failover, retries, OpenTelemetry, logging | +| `/attachments`, `/render` | attachments from a path, a URL, a data URI or a stream; templates and React Email | +| `/smtp`, `/sendmail`, `/jmap` | SMTP, sendmail and JMAP transports | +| `/ses`, `/resend`, `/sendgrid`, `/mailgun`, `/postmark`, `/brevo`, `/mailjet`, `/mailersend`, `/sparkpost`, `/mailtrap`, `/maileroo`, `/lettermint`, `/zeptomail`, `/smtp2go`, `/azure`, `/scaleway`, `/cloudflare`, `/plunk` | HTTP providers | +| `/webhooks` | verified webhooks of fifteen providers, read into delivery events | +| `/pool`, `/retry`, `/breaker`, `/ratelimit`, `/idempotency`, `/hooks`, `/tee`, `/otel`, `/logger` | decorators: failover, retries, circuit breaking, rate limits, idempotency, hooks, mirroring, OpenTelemetry, logging | | `/mock` | an in-memory transport for tests | Every transport takes `generateId` for the identifiers it makes (a ULID or a cuid2 instead of a @@ -104,8 +116,11 @@ Postmark, Mailtrap and Amazon SES side by side, feature by feature, from their s ## Credits The transport layer is a port of [upyo](https://github.com/dahlia/upyo) 0.7.0 by Hong Minhee -(MIT), converted to CommonJS with hand-written types and no dependencies; its notice is in -[LICENSE](./LICENSE). +(MIT), converted to CommonJS with hand-written types and no dependencies. Features of 1.1 are +ported from [nodemailer](https://github.com/nodemailer/nodemailer) 10.0.16 by Andris Reinman +(MIT-0), [unemail](https://github.com/productdevbook/unemail) 0.7.0 (MIT) and +[sently](https://github.com/omqkhafi/sently) 1.2.2 (MIT), with their known bugs fixed in the port. +Their notices are in [LICENSE](./LICENSE). ## Changelog diff --git a/SECURITY.md b/SECURITY.md index 7003714..b7b7128 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,7 +19,7 @@ or by email to . Include, as far as you can: - what an attacker can do, and what they need in order to do it; -- the affected version and Node.js version; +- the affected version, and the runtime (Node.js, Bun or Deno) and its version; - the transport or provider involved (SMTP server, Amazon SES, another HTTP provider); - a minimal reproduction. @@ -28,8 +28,9 @@ timeline once the report is confirmed. ## Threat model -The transport layer is built and the reliability layer is being designed; this section will be -completed with it. The principles both are designed around: +The transport layer, the delivery events and their webhooks are built; the queue and the audit +log are being designed, and this section will be completed with them. The principles all of it is +designed around: - **Message content is untrusted data.** Addresses, display names, subjects, headers, file names and bodies can come from users. A CR or LF in any header field is rejected rather than encoded @@ -39,17 +40,28 @@ completed with it. The principles both are designed around: over a plain connection: when TLS is required and the server does not offer STARTTLS, or the upgrade fails, the send fails instead of falling back to plain text. Bcc recipients never appear in the headers of the message. -- **Attachments do not read the machine by default** (planned). Reading a file path or fetching a - URL for an attachment is off unless the application turns it on; fetching a URL is limited in size - and refuses private and link-local addresses. +- **Provider requests stay with their provider.** Redirects are not followed, so credentials in a + header never reach another host; a provider URL must be HTTPS (plain HTTP only to a loopback + host); and an answer is read up to 4 MiB. +- **Attachments do not read the machine by default.** No file or URL is read for an attachment + but through `fromPath` and `fromUrl` of `./attachments`, which the application calls. + `fromUrl` checks every URL, redirects included, with `allowUrl`, by default refusing + `localhost` and loopback, private and link-local addresses; it sends its headers to the first + origin only and bounds time and bytes. `fromPath` with a `root` compares real paths, so a + symbolic link cannot lead out of it. +- **The sendmail command gets arguments, not a shell line.** The recipients follow `--`, none may + start with `-`, and Bcc recipients go on the command line, never into the message. - **Credentials and personal data stay out of logs.** A transport shows `[REDACTED]` in place of its SMTP password, OAuth2 tokens, provider key and DKIM private key when it is inspected, logged or serialized. `./otel` and `./logger` record addresses only when told to (`recordSensitiveData`, `recordMessage`); otherwise the addresses that error text names, such as the recipient of an SMTP rejection, are redacted. A receipt returned to the application keeps its full error text: logging it is the application's to scrub. -- **Webhooks are verified** (planned). A delivery event from a provider is accepted only with a - valid signature and within a replay window. +- **Webhooks are verified before they are read.** A delivery event is read only from a request + whose signature verifies, or, where the provider signs nothing, whose Basic credentials or token + match, compared in constant time; within a window of time where a time is signed, and once by + its id. An SNS message (SES, Scaleway) is taken only from a listed topic, signed with the + certificate of an SNS host or of the trust chain the application pins. - **The audit log can forget a person** (planned). Recipients are stored as a keyed hash and an encrypted block whose key can be destroyed, so a request to erase someone's data does not require editing an append-only log. diff --git a/docs/api/exports.md b/docs/api/exports.md index a15a34c..9890b60 100644 --- a/docs/api/exports.md +++ b/docs/api/exports.md @@ -16,12 +16,14 @@ import { SmtpTransport } from '@alexify/mailer/smtp'; ## `@alexify/mailer` -The message model and the transport contract, ported from `@upyo/core`. +The message model and the transport contract, ported from `@upyo/core`, and the +model of delivery events. - **Messages:** `createMessage`, and the `Message`, `MessageConstructor`, - `MessageContent`, `Address`, `Attachment` and `Priority` types; `parseAddress`, - `formatAddress`, `isEmailAddress`, `isAttachment`, `comparePriority`. See - [composing messages](/messages/compose). + `MessageContent`, `Address`, `Attachment` and `Priority` types, with the + `ListFields`, `ListUrl` and `MessageTracking` of the `list` and `tracking` fields; + `parseAddress`, `formatAddress`, `isEmailAddress`, `isAttachment`, + `comparePriority`. See [composing messages](/messages/compose). - **Attachments:** `readAttachmentContent`, `iterateAttachmentContent`. See [attachments](/messages/attachments). - **Calendar:** `createCalendarAttachment`, `resolveCalendarContent`, @@ -36,6 +38,9 @@ The message model and the transport contract, ported from `@upyo/core`. `parseRetryAfter`. - **Raw messages:** the `RawMessage` type; `createRawMessagePlan`, `iterateRawMessage`, `analyzeRawMessage`, `RawMessageValidationError`. +- **Delivery events:** the `DeliveryEvent`, `DeliveryEventType` and + `DeliveryBounce` types, which `./webhooks` gives. See + [webhooks and events](/guide/webhooks). ## Subpaths diff --git a/docs/comparison.json b/docs/comparison.json index babe372..f4a2633 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -1,5 +1,5 @@ { - "legend": "✅ in the code · ⚠️ partial, or only on the provider side · ❌ none · 🔜 planned by the author, no code yet · — does not apply. Every cell was decided by reading the source of the project as of 2026-10-07, not its README; the column of @alexify/mailer is this repository as of 2026-10-17.", + "legend": "✅ in the code · ⚠️ partial, or only on the provider side · ❌ none · 🔜 planned by the author, no code yet · — does not apply. Every cell was decided by reading the source of the project as of 2026-10-07, not its README; the column of @alexify/mailer is this repository as of 2026-10-18.", "groups": [ { "title": "Runtimes (from imports and CI, not run)", @@ -281,12 +281,13 @@ "text": "≥22, CI on 22 and 24" }, "deno": { - "mark": "❌", - "text": "not a target" + "mark": "✅", + "text": "CI: unit and runtime suites", + "note": "Through `npm:@alexify/mailer`. Deno’s `fetch` sends a request again when a reused connection closes before its answer, so the docs give it a `fetch` of its own ([known issues](/guide/known-issues#deno))." }, "bun": { - "mark": "❌", - "text": "not a target" + "mark": "✅", + "text": "CI: unit and runtime suites" }, "edge": { "mark": "❌", @@ -301,23 +302,24 @@ "text": "PIPELINING, SIZE, DSN" }, "sendmail": { - "mark": "❌" + "mark": "✅", + "text": "`./sendmail`, exit codes classified" }, "jmap": { "mark": "✅", "text": "RFC 8620/8621" }, "httpProviders": { - "text": "8" + "text": "18" }, "failover": { "mark": "✅", - "text": "RR, weighted, priority; no health yet", + "text": "RR, weighted, priority, circuit breaker", "note": "A failure after which the provider may have taken the message is marked `deliveryUnknown`: `./retry` does not retry it and `./pool` does not fail over with it." }, "smtpPool": { "mark": "✅", - "text": "on by default, 5" + "text": "on by default, 5, 100 messages each" }, "retry": { "mark": "✅", @@ -325,13 +327,12 @@ "note": "A failure after which the provider may have taken the message is marked `deliveryUnknown`: `./retry` does not retry it and `./pool` does not fail over with it." }, "rateLimit": { - "mark": "⚠️", - "text": "`sendMany` concurrency; token bucket planned" + "mark": "✅", + "text": "token bucket per key, FIFO" }, "circuitBreaker": { - "mark": "🔜", - "text": "one probe in half-open", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "✅", + "text": "one probe in half-open" }, "timeout": { "mark": "✅", @@ -343,62 +344,62 @@ "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." }, "attachments": { - "mark": "⚠️", - "text": "bytes, Promise, Blob, chunk factory" + "mark": "✅", + "text": "path, URL, data URI, stream, bytes, Blob", + "note": "Nothing is read implicitly: `fromPath` and `fromUrl` are called by the application, `fromUrl` checks every redirect against `allowUrl` (no private address by default) and bounds time and bytes." }, "streaming": { "mark": "⚠️", - "text": "SMTP; HTTP providers buffer" + "text": "SMTP, sendmail; HTTP providers buffer" }, "inline": { "mark": "✅", "text": "multipart/related" }, "rawMime": { - "mark": "⚠️", - "text": "SMTP/JMAP only" + "mark": "✅", + "text": "SMTP, sendmail, JMAP, SES, Mailgun, SparkPost, Cloudflare" }, "ical": { "mark": "✅", "text": "every METHOD" }, "unicode": { - "mark": "⚠️", - "text": "SMTPUTF8, no automatic punycode" + "mark": "✅", + "text": "punycode + SMTPUTF8" }, "listUnsubscribe": { - "mark": "⚠️", - "text": "manual header; one-click planned" + "mark": "✅", + "text": "RFC 8058 one-click, DKIM-signed" }, "templates": { - "mark": "❌", - "text": "none, by design" + "mark": "⚠️", + "text": "`Renderer` interface, no engine by design" }, "reactEmail": { - "mark": "🔜", - "text": "optional adapter", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "✅", + "text": "`render` passed in" }, "batch": { "mark": "✅", - "text": "native: Resend, Mailtrap, JMAP; concurrency elsewhere" + "text": "native: Resend, Postmark, Mailjet, Mailtrap, JMAP; concurrency elsewhere" }, "scheduled": { - "mark": "🔜", - "text": "a delay in the queue", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "⚠️", + "text": "native in 8 providers; queue delay planned", + "note": "A transport that cannot schedule refuses the message before sending it, rather than sending it at once." }, "tags": { - "mark": "⚠️", - "text": "tags in 7 providers" + "mark": "✅", + "text": "metadata in 17 providers, tags in 13" }, "tracking": { - "mark": "⚠️", - "text": "config only" + "mark": "✅", + "text": "per message in 10 providers" }, "idempotency": { - "mark": "⚠️", - "text": "Resend, Lettermint, Maileroo; outbox key planned" + "mark": "✅", + "text": "decorator with reserved keys + 5 providers" }, "dkim": { "mark": "✅", @@ -415,12 +416,13 @@ "mark": "✅" }, "proxy": { - "mark": "⚠️", - "text": "HTTP through an injected `fetch`; SMTP no" + "mark": "✅", + "text": "SMTP: HTTP CONNECT or any `connect`; HTTP: `fetch`" }, "verify": { - "mark": "⚠️", - "text": "SMTP/JMAP only" + "mark": "✅", + "text": "SMTP, sendmail, JMAP, 13 HTTP providers", + "note": "The other five have no request that checks a sending key without sending a message." }, "smtpAuth": { "text": "PLAIN, LOGIN, XOAUTH2, OAUTHBEARER" @@ -430,9 +432,9 @@ "text": "`requireTls`, SNI, CA, cert, `minVersion`" }, "webhooks": { - "mark": "🔜", - "text": "SES (SNS) first, signature required", - "note": "On the [roadmap](/guide/why#the-plan) of the reliability layer: designed, with no code yet." + "mark": "✅", + "text": "15 providers, verified before parsing", + "note": "A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time and a replay store." }, "inbound": { "mark": "❌", @@ -466,8 +468,8 @@ "text": "decorator: spans + metrics" }, "hooks": { - "mark": "⚠️", - "text": "decorators, `onRetry`, a pino-like logger" + "mark": "✅", + "text": "`./hooks`, decorators, a pino-like logger" }, "mock": { "mark": "✅", @@ -498,7 +500,7 @@ "text": "2026-10 (this release)" }, "tests": { - "text": "126 / ~1440" + "text": "167 / ~2340" }, "maintainer": { "text": "1 author, Alexis Technologies" diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index 697b049..d407df6 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -55,50 +55,50 @@ it. Runtimes (from imports and CI, not run) Node.js, minimum version✅ ≥22, CI on 22 and 24✅ ≥20⚠️ ≥18, CI 20+✅ ≥20✅ ≥20.11✅ ≥18❌ Workers only⚠️ needs a connector of your own✅ no engines field✅ ≥20✅ ≥12✅ ≥18✅ ≥18✅ ≥16.20✅ ≥20 -Deno❌ not a target❌ not declared❌ not declared✅ CI: deno test⚠️ HTTP drivers, not SMTP✅ Deno.connect❌✅ edge-mailer/deno⚠️ browser-entry⚠️ fetch, not tested⚠️ through Node compatibility⚠️ not declared⚠️ fetch, not declared⚠️ not declared⚠️ through npm: -Bun❌ not a target✅ CI: test:bun❌ not declared✅ CI: bun test✅ CI: bun✅ CI: bun test❌❌ no connector⚠️ browser-entry⚠️ fetch⚠️ through Node compatibility⚠️ not declared⚠️ not declared⚠️ not declared⚠️ through Node compatibility +Deno✅ CI: unit and runtime suites1❌ not declared❌ not declared✅ CI: deno test⚠️ HTTP drivers, not SMTP✅ Deno.connect❌✅ edge-mailer/deno⚠️ browser-entry⚠️ fetch, not tested⚠️ through Node compatibility⚠️ not declared⚠️ fetch, not declared⚠️ not declared⚠️ through npm: +Bun✅ CI: unit and runtime suites✅ CI: test:bun❌ not declared✅ CI: bun test✅ CI: bun✅ CI: bun test❌❌ no connector⚠️ browser-entry⚠️ fetch⚠️ through Node compatibility⚠️ not declared⚠️ not declared⚠️ not declared⚠️ through Node compatibility Cloudflare Workers / edge❌ not a target yet⚠️ only with nodejs_compat❌⚠️ HTTP packages yes, SMTP no⚠️ HTTP + 3 Cloudflare drivers, no SMTP✅ HTTP + SMTP (cloudflare:sockets)✅ the only target✅ no node:*⚠️ browser-entry⚠️ fetch, not tested⚠️ fs, axios; not declared⚠️ fetch mode⚠️ declared in 5.0.0❌ http/https agents⚠️ explicit credentials needed Browser❌ by design❌❌⚠️ no target, no tests✅ HTTP drivers only⚠️ throws❌❌✅ browser-bundle⚠️ secret key in the client❌✅ needs a CORS proxy❌❌✅ explicit credentials Transports and delivery reliability SMTP client✅ PIPELINING, SIZE, DSN✅ SMTP + LMTP✅ no LMTP✅ PIPELINING, SIZE, DSN✅ own implementation✅ own implementation✅✅—❌❌❌❌❌❌ -Sendmail / local binary❌✅❌❌❌❌❌❌—❌❌❌❌❌❌ +Sendmail / local binary✅ ./sendmail, exit codes classified✅❌❌❌❌❌❌—❌❌❌❌❌❌ JMAP✅ RFC 8620/8621❌❌✅ RFC 8620/8621❌❌❌❌—❌❌❌❌❌❌ -HTTP providers, count81 (SES, needs the SDK)0818 + http + 2 CF14 + CF binding00—111111 -Failover between providers✅ RR, weighted, priority; no health yet1⚠️ host DNS fallback only❌✅ RR, weighted, priority✅ fallback, RR, weighted, tee✅ fallback + weighted❌❌—❌❌❌❌❌⚠️ multi-region endpoint on the AWS side -SMTP connection pool✅ on by default, 5✅ max 5, 100 msg❌ one connection✅ on by default✅ off by default✅ off by default❌✅——————— -Retry / backoff✅ jitter, Retry-After, no blind resend1⚠️ requeue in the pool only⚠️ 1 retry on 450/451✅ jitter, Retry-After✅ 5 modes, jitter✅ no jitter, retries SMTP 5xx❌❌ retryHint only—❌❌❌❌❌✅ standard / adaptive -Rate limiting⚠️ sendMany concurrency; token bucket planned✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry -Circuit breaker🔜 one probe in half-open2❌❌❌✅ half-open without a single probe⚠️ cooldownMs❌❌—❌❌❌❌❌❌ +HTTP providers, count181 (SES, needs the SDK)0818 + http + 2 CF14 + CF binding00—111111 +Failover between providers✅ RR, weighted, priority, circuit breaker2⚠️ host DNS fallback only❌✅ RR, weighted, priority✅ fallback, RR, weighted, tee✅ fallback + weighted❌❌—❌❌❌❌❌⚠️ multi-region endpoint on the AWS side +SMTP connection pool✅ on by default, 5, 100 messages each✅ max 5, 100 msg❌ one connection✅ on by default✅ off by default✅ off by default❌✅——————— +Retry / backoff✅ jitter, Retry-After, no blind resend2⚠️ requeue in the pool only⚠️ 1 retry on 450/451✅ jitter, Retry-After✅ 5 modes, jitter✅ no jitter, retries SMTP 5xx❌❌ retryHint only—❌❌❌❌❌✅ standard / adaptive +Rate limiting✅ token bucket per key, FIFO✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry +Circuit breaker✅ one probe in half-open❌❌❌✅ half-open without a single probe⚠️ cooldownMs❌❌—❌❌❌❌❌❌ Timeout / AbortSignal✅ signal + timeout over the body⚠️ timeouts, no AbortSignal⚠️ timeout, no signal✅ signal + timeout⚠️ signal for HTTP, not SMTP⚠️ SMTP connect only⚠️ socket timeouts✅ signal on connect and send—⚠️ signal, no timeout⚠️ global signal only⚠️ timeout, no signal⚠️ 180 s timeout⚠️ hard 10 s✅ abortSignal + requestTimeout -Persistent queue (survives a restart)🔜 MongoDB outbox, BullMQ driver2❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌ +Persistent queue (survives a restart)🔜 MongoDB outbox, BullMQ driver3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌ Messages -Attachments⚠️ bytes, Promise, Blob, chunk factory✅ path, URL, Buffer, stream, data-URI⚠️ path, stream, no URL⚠️ bytes, Blob, File, factory⚠️ bytes, string, URL⚠️ bytes + path⚠️ base64 only⚠️ string, bytes, Blob⚠️ base64 only⚠️ base64 or URL⚠️ base64, helper for Buffer✅ Buffer, Blob, stream, 50 MB✅ base64✅ string / Buffer✅ Uint8Array or Raw MIME -Streaming large attachments⚠️ SMTP; HTTP providers buffer⚠️ SMTP yes; DKIM and SES buffer✅⚠️ SMTP only❌❌❌❌❌❌❌⚠️ accepts a stream❌❌❌ +Attachments✅ path, URL, data URI, stream, bytes, Blob4✅ path, URL, Buffer, stream, data-URI⚠️ path, stream, no URL⚠️ bytes, Blob, File, factory⚠️ bytes, string, URL⚠️ bytes + path⚠️ base64 only⚠️ string, bytes, Blob⚠️ base64 only⚠️ base64 or URL⚠️ base64, helper for Buffer✅ Buffer, Blob, stream, 50 MB✅ base64✅ string / Buffer✅ Uint8Array or Raw MIME +Streaming large attachments⚠️ SMTP, sendmail; HTTP providers buffer⚠️ SMTP yes; DKIM and SES buffer✅⚠️ SMTP only❌❌❌❌❌❌❌⚠️ accepts a stream❌❌❌ Inline images (CID)✅ multipart/related✅ multipart/related⚠️ manual Content-ID⚠️ mixed, not related; Mailgun bug⚠️ mixed, not related⚠️ HTTP: 3 providers only❌✅✅✅✅✅✅✅✅ -Sending raw MIME⚠️ SMTP/JMAP only✅❌⚠️ SMTP/JMAP only⚠️ smtp driver only❌❌❌— builds MIME only❌❌✅ /messages.mime❌❌✅ Content.Raw +Sending raw MIME✅ SMTP, sendmail, JMAP, SES, Mailgun, SparkPost, Cloudflare✅❌⚠️ SMTP/JMAP only⚠️ smtp driver only❌❌❌— builds MIME only❌❌✅ /messages.mime❌❌✅ Content.Raw iCalendar✅ every METHOD✅ icalEvent + METHOD⚠️ manual✅ every METHOD🔜 planned❌❌✅⚠️ as an attachment❌❌⚠️ as an attachment⚠️ as an attachment⚠️ as an attachment⚠️ through Raw MIME -Unicode / IDN⚠️ SMTPUTF8, no automatic punycode✅ punycode + SMTPUTF8⚠️ Q-encoding, no punycode⚠️ SMTPUTF8, no automatic punycode⚠️ RFC 2047/2231⚠️ one RFC 2047 word⚠️ RFC 2047⚠️ SMTPUTF8, no punycode⚠️ Subject as one B-word⚠️ UTF-8 JSON⚠️ UTF-8 JSON❌❌⚠️ UTF-8 JSON⚠️ Charset, no IDN -List-Unsubscribe⚠️ manual header; one-click planned✅ list.unsubscribe❌⚠️ manual header✅ RFC 8058 oneClick❌⚠️ headers⚠️ headers❌⚠️ headers, topics⚠️ headers, asm⚠️ headers⚠️ headers⚠️ headers✅ ListManagementOptions -Built-in templates❌ none, by design❌❌❌⚠️ Renderer interface✅ {{var}} + plugins❌❌❌✅ Templates API✅ dynamic templates✅ Templates API✅ Templates API✅ Templates API✅ Templates API -React Email🔜 optional adapter2❌❌❌✅✅❌❌❌✅❌❌❌❌❌ -Batch / bulk✅ native: Resend, Mailtrap, JMAP; concurrency elsewhere❌❌✅ native: Resend, Mailtrap, JMAP✅ sendBatch⚠️ sendBulk, SendGrid bug⚠️ queue per session✅—✅ up to 100, no attachments✅ personalizations⚠️ recipient-variables✅ batch + bulk API✅ batchSend, 500✅ SendBulkEmail -Scheduled sending🔜 a delay in the queue2❌❌❌⚠️ 8 drivers❌❌❌—✅✅✅ up to 3 days❌⚠️ campaigns only❌ -Tags / metadata⚠️ tags in 7 providers⚠️ headers; tags on SES only⚠️ headers⚠️ tags in 7 providers✅⚠️ headers only⚠️ headers⚠️ headers⚠️ headers✅✅✅✅✅✅ -Open/click tracking⚠️ config only❌❌⚠️ config only✅ 10 drivers❌❌❌—⚠️ per domain✅✅✅⚠️ per domain✅ per-message override -Idempotency key⚠️ Resend, Lettermint, Maileroo; outbox key planned❌❌⚠️ Resend, Lettermint, Maileroo✅ middleware + providers⚠️ decorator; batch bypasses it❌❌—✅❌❌❌❌❌ +Unicode / IDN✅ punycode + SMTPUTF8✅ punycode + SMTPUTF8⚠️ Q-encoding, no punycode⚠️ SMTPUTF8, no automatic punycode⚠️ RFC 2047/2231⚠️ one RFC 2047 word⚠️ RFC 2047⚠️ SMTPUTF8, no punycode⚠️ Subject as one B-word⚠️ UTF-8 JSON⚠️ UTF-8 JSON❌❌⚠️ UTF-8 JSON⚠️ Charset, no IDN +List-Unsubscribe✅ RFC 8058 one-click, DKIM-signed✅ list.unsubscribe❌⚠️ manual header✅ RFC 8058 oneClick❌⚠️ headers⚠️ headers❌⚠️ headers, topics⚠️ headers, asm⚠️ headers⚠️ headers⚠️ headers✅ ListManagementOptions +Built-in templates⚠️ Renderer interface, no engine by design❌❌❌⚠️ Renderer interface✅ {{var}} + plugins❌❌❌✅ Templates API✅ dynamic templates✅ Templates API✅ Templates API✅ Templates API✅ Templates API +React Email✅ render passed in❌❌❌✅✅❌❌❌✅❌❌❌❌❌ +Batch / bulk✅ native: Resend, Postmark, Mailjet, Mailtrap, JMAP; concurrency elsewhere❌❌✅ native: Resend, Mailtrap, JMAP✅ sendBatch⚠️ sendBulk, SendGrid bug⚠️ queue per session✅—✅ up to 100, no attachments✅ personalizations⚠️ recipient-variables✅ batch + bulk API✅ batchSend, 500✅ SendBulkEmail +Scheduled sending⚠️ native in 8 providers; queue delay planned5❌❌❌⚠️ 8 drivers❌❌❌—✅✅✅ up to 3 days❌⚠️ campaigns only❌ +Tags / metadata✅ metadata in 17 providers, tags in 13⚠️ headers; tags on SES only⚠️ headers⚠️ tags in 7 providers✅⚠️ headers only⚠️ headers⚠️ headers⚠️ headers✅✅✅✅✅✅ +Open/click tracking✅ per message in 10 providers❌❌⚠️ config only✅ 10 drivers❌❌❌—⚠️ per domain✅✅✅⚠️ per domain✅ per-message override +Idempotency key✅ decorator with reserved keys + 5 providers❌❌⚠️ Resend, Lettermint, Maileroo✅ middleware + providers⚠️ decorator; batch bypasses it❌❌—✅❌❌❌❌❌ Security and authentication DKIM signing in the library✅ RSA + Ed25519, streaming, over-signing✅ RSA-SHA256❌✅ RSA + Ed25519, streaming✅ RSA + Ed25519, SMTP only⚠️ body canonicalization bug❌✅ RSA❌⚠️ keys through the API⚠️ whitelabel, generic⚠️ keys through the API⚠️ keys through the API⚠️ domain DNS records⚠️ Easy / BYO DKIM OAuth2 for SMTP✅ refresh, OAUTHBEARER✅ refresh + service account⚠️ bearer token only✅ refresh, OAUTHBEARER⚠️ callback, no refresh✅ refresh client❌⚠️ no refresh——————— ARC❌❌❌❌❌❌❌❌❌—————— DSN✅✅❌✅❌❌✅✅——————— -Proxy⚠️ HTTP through an injected fetch; SMTP no⚠️ HTTP CONNECT, SOCKS through a module❌❌⚠️ custom fetch❌❌❌—⚠️ baseUrl only⚠️ axios proxy✅⚠️ custom fetch + undici⚠️ environment only⚠️ through requestHandler -verify() / connection check⚠️ SMTP/JMAP only✅⚠️ VRFY, not a check⚠️ SMTP/JMAP only⚠️ isAvailable(), nominal✅❌⚠️ probe() without auth———⚠️ domain DNS only⚠️ getServer()⚠️ getAllAccounts()⚠️ GetAccount +Proxy✅ SMTP: HTTP CONNECT or any connect; HTTP: fetch⚠️ HTTP CONNECT, SOCKS through a module❌❌⚠️ custom fetch❌❌❌—⚠️ baseUrl only⚠️ axios proxy✅⚠️ custom fetch + undici⚠️ environment only⚠️ through requestHandler +verify() / connection check✅ SMTP, sendmail, JMAP, 13 HTTP providers6✅⚠️ VRFY, not a check⚠️ SMTP/JMAP only⚠️ isAvailable(), nominal✅❌⚠️ probe() without auth———⚠️ domain DNS only⚠️ getServer()⚠️ getAllAccounts()⚠️ GetAccount SMTP AUTH mechanismsPLAIN, LOGIN, XOAUTH2, OAUTHBEARERPLAIN, LOGIN, CRAM-MD5, XOAUTH2CRAM-MD5, LOGIN, PLAIN, XOAUTH2PLAIN, LOGIN, XOAUTH2, OAUTHBEARERPLAIN, LOGIN, CRAM-MD5, XOAUTH2PLAIN, LOGIN, CRAM-MD5, XOAUTH2PLAIN, LOGIN, CRAM-MD5PLAIN, LOGIN, CRAM-MD5, XOAUTH2——————— SMTP TLS policies✅ requireTls, SNI, CA, cert, minVersion✅ REQUIRETLS, SNI, no DANE⚠️ STARTTLS accepts any certificate✅ requireTls, CA, cert, minVersion✅ requireTLS, off by default✅ requireTLS by default with auth⚠️ falls back to plaintext + AUTH✅ tlsPolicy ×4, REQUIRETLS——————— Platform features (events, domains, provider API) -Webhooks: parsing and signature verification🔜 SES (SNS) first, signature required2❌❌❌🔜 planned✅ 10 parsers, HMAC in 5❌❌—✅ standardwebhooks✅ ECDSA⚠️ CRUD, no verification⚠️ CRUD + types, no verification✅ HMAC verify❌ SNS signature not in the SDK +Webhooks: parsing and signature verification✅ 15 providers, verified before parsing7❌❌❌🔜 planned✅ 10 parsers, HMAC in 5❌❌—✅ standardwebhooks✅ ECDSA⚠️ CRUD, no verification⚠️ CRUD + types, no verification✅ HMAC verify❌ SNS signature not in the SDK Inbound mail❌ out of scope❌❌❌🔜 planned❌❌❌—✅⚠️ parser with a bug⚠️ routes✅✅❌ legacy client-ses only -Suppression / bounce API🔜 per tenant, before the queue2❌❌❌🔜 planned❌❌❌—✅⚠️ generic client✅✅✅✅ +Suppression / bounce API🔜 per tenant, before the queue3❌❌❌🔜 planned❌❌❌—✅⚠️ generic client✅✅✅✅ Domain management (API)❌ out of scope❌❌❌❌❌❌❌—✅⚠️ generic client✅✅✅✅ identities Address validation⚠️ RFC 5322/6531 syntax⚠️ parser only⚠️ parser only⚠️ RFC 5322/6531 syntax⚠️ regex⚠️ regex❌❌ syntax assert❌❌❌✅ /v4/address/validate❌❌✅ GetEmailAddressInsights Sending log on the provider side—————————✅ logs API⚠️ generic /v3/messages✅ logs API⚠️ events, opens, clicks⚠️ emailLogs⚠️ event destinations + archiving @@ -106,9 +106,9 @@ it. Multi-channel: SMS, WhatsApp, push❌ out of scope❌❌❌❌✅ SMS, WhatsApp, webpush, FCM❌❌—❌❌❌❌❌❌ Observability and extensibility OpenTelemetry✅ decorator: spans + metrics❌❌✅ decorator: spans + metrics⚠️ withLogger⚠️ hooks + consoleObserver❌❌❌❌❌❌❌❌⚠️ middlewareStack -Hooks / middleware⚠️ decorators, onRetry, a pino-like logger✅ use(compile|stream)⚠️ logger function⚠️ decorators, LogTape✅ Middleware pipeline✅ hooks + plugins⚠️ logger✅ onEvent⚠️ dump()⚠️ console.error⚠️ setClient❌❌❌✅ logger + middlewareStack +Hooks / middleware✅ ./hooks, decorators, a pino-like logger✅ use(compile|stream)⚠️ logger function⚠️ decorators, LogTape✅ Middleware pipeline✅ hooks + plugins⚠️ logger✅ onEvent⚠️ dump()⚠️ console.error⚠️ setClient❌❌❌✅ logger + middlewareStack Mock / test mode✅ ./mock✅ jsonTransport, streamTransport, Ethereal⚠️ Message#read()✅ @upyo/mock✅ mock + Mailpit⚠️ preview, Mailpit❌❌—❌✅ sandboxMode✅ testmode⚠️ Sandbox server✅ Sandbox API⚠️ TestRenderEmailTemplate -Local audit log / event journal🔜 append-only, crypto-shredding2❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌ +Local audit log / event journal🔜 append-only, crypto-shredding3❌❌❌❌❌❌⚠️ events, not stored—❌❌❌❌❌❌ Quality, license, maturity TypeScript types✅ hand-written, tsd✅ built in✅✅✅✅✅✅✅✅✅ hand-written .d.ts✅✅✅✅ generated Runtime dependencies0000–1 (@noble/hashes)0000422+ (axios)3 (axios)02 (axios, qs)9 @@ -116,17 +116,22 @@ it. LicenseMITMIT-0MITMITMITMITMITMITMITMITMITMITMITMITApache-2.0 Version (2026-10-07)1.0.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0 Last commit2026-10 (this release)2026-10-072026-08-262026-09-092026-09-022026-08-192025-11-152026-09-082026-01-282026-10-062025-09-192026-10-072026-07-102026-10-072026-10-06 -Tests: files / cases126 / ~144061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo) +Tests: files / cases167 / ~234061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo) Owner / maintenance risk1 author, Alexis Technologies1 maintainerno SECURITY.md1 main author1 author1 author, 0.1 → 1.2 in ~80 days1 author1 author, 0.xno SECURITY.mdvendorvendor, old toolchainvendorvendorvendorvendor -✅ in the code · ⚠️ partial, or only on the provider side · ❌ none · 🔜 planned by the author, no code yet · — does not apply. Every cell was decided by reading the source of the project as of 2026-10-07, not its README; the column of @alexify/mailer is this repository as of 2026-10-17. +✅ in the code · ⚠️ partial, or only on the provider side · ❌ none · 🔜 planned by the author, no code yet · — does not apply. Every cell was decided by reading the source of the project as of 2026-10-07, not its README; the column of @alexify/mailer is this repository as of 2026-10-18. -1. A failure after which the provider may have taken the message is marked deliveryUnknown: ./retry does not retry it and ./pool does not fail over with it. -2. On the roadmap of the reliability layer: designed, with no code yet. +1. Through npm:@alexify/mailer. Deno’s fetch sends a request again when a reused connection closes before its answer, so the docs give it a fetch of its own (known issues). +2. A failure after which the provider may have taken the message is marked deliveryUnknown: ./retry does not retry it and ./pool does not fail over with it. +3. On the roadmap of the reliability layer: designed, with no code yet. +4. Nothing is read implicitly: fromPath and fromUrl are called by the application, fromUrl checks every redirect against allowUrl (no private address by default) and bounds time and bytes. +5. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. +6. The other five have no request that checks a sending key without sending a message. +7. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time and a replay store. @@ -134,15 +139,17 @@ it. - **From the code, not the README.** Every cell was decided by reading the source of a local clone of each project as of 2026-10-07. -- **The column of @alexify/mailer is this repository,** as of 2026-10-17. ✅ and ⚠️ are code that - is here: the transport ported from upyo, with the upstream bugs fixed since. 🔜 is the - reliability layer of [the plan](/guide/why#the-plan), designed but with no code yet. Its gaps - are shown as such: no Deno, Bun or edge runtime, no rate limit beyond the concurrency of - `sendMany`, and streaming only over SMTP. +- **The column of @alexify/mailer is this repository,** as of 2026-10-18. ✅ and ⚠️ are code that + is here: the transport ported from upyo, with the upstream bugs fixed since, and what 1.1 took + from nodemailer, unemail and sently, with their bugs fixed in the port. 🔜 is the reliability + layer of [the plan](/guide/why#the-plan), designed but with no code yet. Its gaps are shown as + such: no edge runtime, streaming only over SMTP and sendmail, and scheduling only where the + provider schedules. - **Defects were not reproduced,** except in emailjs, which was run against fake SMTP servers. A cell that names a bug is a finding from reading the code; reproduce it before relying on it. -- **Runtimes were not run.** Deno, Bun and Workers support is inferred from imports, `exports` - maps and CI configurations. +- **The runtimes of the others were not run.** Their Deno, Bun and Workers support is inferred + from imports, `exports` maps and CI configurations. The suites of @alexify/mailer run on Bun and + Deno in its own CI. - **The clones are shallow** (one commit), so the history, the number of maintainers and the pace of releases are not taken from git. Test cases were counted with a regular expression. - **Amazon SES** means the newest SES client of the AWS SDK for JavaScript v3, diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index fa9a614..59890d2 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -2,8 +2,9 @@ ::: tip Status Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) -0.7.0: messages, SMTP, eight HTTP providers, JMAP and the decorators. The reliability -layer (queue, audit log, delivery events) is still being designed. +0.7.0; 1.1 adds sendmail, ten more HTTP providers, verified webhooks into delivery +events, and decorators for circuit breaking, rate limits, idempotency, hooks and +mirroring. The queue and the audit log are still being designed. ::: ## Installation @@ -67,25 +68,31 @@ Each transport implements the same `Transport` contract of the root entry, so th can be swapped without changing the code that builds messages: - [SMTP](/transports/smtp) (`@alexify/mailer/smtp`): universal email protocol, works - with any SMTP server; DKIM signing, pooling, STARTTLS, DSN. -- [Amazon SES](/transports/ses) (`@alexify/mailer/ses`): cost-effective email - service for AWS users. -- [Resend](/transports/resend) (`@alexify/mailer/resend`): developer-friendly API, - batch sending. -- [SendGrid](/transports/sendgrid) (`@alexify/mailer/sendgrid`): popular email API - with a deliverability focus. -- [Mailgun](/transports/mailgun) (`@alexify/mailer/mailgun`): HTTP API with US and - EU regions. -- [Mailtrap](/transports/mailtrap) (`@alexify/mailer/mailtrap`): Email API and Email - Sandbox for production delivery and test inbox capture. -- [Maileroo](/transports/maileroo) (`@alexify/mailer/maileroo`): JSON email API with - attachments, tags, custom headers and tracking. -- [Lettermint](/transports/lettermint) (`@alexify/mailer/lettermint`): transactional - email API with routes, metadata, idempotency and batch sending. -- [Plunk](/transports/plunk) (`@alexify/mailer/plunk`): cloud or self-hosted - transactional email. + with any SMTP server; DKIM signing, pooling, STARTTLS, DSN, an HTTP CONNECT proxy. +- [Sendmail](/transports/sendmail) (`@alexify/mailer/sendmail`): the sendmail command + of the system (Postfix, Exim, msmtp). - [JMAP](/transports/jmap) (`@alexify/mailer/jmap`): RFC 8620/8621 mail servers such as Stalwart or Fastmail. +- HTTP providers, one subpath each: + [Amazon SES](/transports/ses) (`ses`, SES v2 with SigV4), + [Resend](/transports/resend) (`resend`), + [SendGrid](/transports/sendgrid) (`sendgrid`), + [Mailgun](/transports/mailgun) (`mailgun`), + [Postmark](/transports/postmark) (`postmark`), + [Brevo](/transports/brevo) (`brevo`), + [Mailjet](/transports/mailjet) (`mailjet`), + [MailerSend](/transports/mailersend) (`mailersend`), + [SparkPost](/transports/sparkpost) (`sparkpost`), + [Mailtrap](/transports/mailtrap) (`mailtrap`), + [Maileroo](/transports/maileroo) (`maileroo`), + [Lettermint](/transports/lettermint) (`lettermint`), + [ZeptoMail](/transports/zeptomail) (`zeptomail`), + [SMTP2GO](/transports/smtp2go) (`smtp2go`), + [Azure Communication Services](/transports/azure) (`azure`), + [Scaleway](/transports/scaleway) (`scaleway`), + [Cloudflare](/transports/cloudflare) (`cloudflare`) and + [Plunk](/transports/plunk) (`plunk`). Each page lists what the provider takes of a + message: tags, metadata, tracking, a schedule, raw MIME. - [Mock](/transports/mock) (`@alexify/mailer/mock`): captures emails without sending, for tests. @@ -95,11 +102,25 @@ And the decorators, which wrap any transport: a strategy, with failover. - [Retry](/transports/retry) (`@alexify/mailer/retry`): retries with backoff, jitter and `Retry-After`. +- [Circuit breaker](/transports/breaker) (`@alexify/mailer/breaker`): stops sending + to a failing provider, then lets one probe through. +- [Rate limit](/transports/ratelimit) (`@alexify/mailer/ratelimit`): a token bucket + per key, waiting or refusing. +- [Idempotency](/transports/idempotency) (`@alexify/mailer/idempotency`): a key + reserved before each send, so a repeat is answered, not sent. +- [Hooks](/transports/hooks) (`@alexify/mailer/hooks`): before and after each send, + on each retry and error. +- [Tee](/transports/tee) (`@alexify/mailer/tee`): mirrors each message to other + transports, without waiting for them. - [OpenTelemetry](/transports/otel) (`@alexify/mailer/otel`): spans and metrics, with your providers. - [Logger](/transports/logger) (`@alexify/mailer/logger`): the delivery lifecycle, logged through your pino-like logger. +What happens after a send comes back through the provider's webhooks: +[`@alexify/mailer/webhooks`](/guide/webhooks) verifies them and reads their events +into one model of delivery events. + If none of these fit your needs, you can also [create a custom transport](/transports/custom). @@ -162,8 +183,8 @@ const transport = new ResendTransport({ }); ``` -The JMAP transport streams its raw uploads, so its `fetch` must take a body -stream with `duplex: 'half'`, as undici's does. +The JMAP transport streams a raw message whose size it does not know, so its +`fetch` must take a body stream with `duplex: 'half'`, as undici's does. Every transport also takes a pino-like `logger` for what happens inside it and its receipts do not tell: the retries of an HTTP request at `debug` (with the @@ -178,8 +199,8 @@ and the message ID of a receipt the provider gave none for. See [identifiers of your own](/messages/compose#identifiers-of-your-own). From here, see [composing messages](/messages/compose) for HTML content, multiple -recipients and threading, [attachments](/messages/attachments), and -[calendar invitations](/messages/calendar). +recipients, threading and List-Unsubscribe, [attachments](/messages/attachments), +[rendering templates](/messages/render) and [calendar invitations](/messages/calendar). See [Why @alexify/mailer?](/guide/why) for what the library is built to do, and the [API](/api/exports) for every entry point. diff --git a/docs/guide/why.md b/docs/guide/why.md index 2f5bdd9..cb68c36 100644 --- a/docs/guide/why.md +++ b/docs/guide/why.md @@ -1,8 +1,10 @@ # Why @alexify/mailer? ::: tip Status -Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0. The -reliability layer is being designed: the direction below is settled, its API is not. +Version 1.0 is the transport layer, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0, and +1.1 adds what nodemailer, unemail and sently did better, delivery events and the decorators of a +closed loop. The queue and the audit log are being designed: the direction below is settled, their +API is not. ::: ## Two parts @@ -10,13 +12,15 @@ reliability layer is being designed: the direction below is settled, its API is @alexify/mailer is an email library of its own, in the place of Nodemailer, emailjs or Upyo, made of two parts: -1. **The transport,** here today, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0: an - SMTP client, a MIME builder and DKIM signing, Amazon SES v2 (signed with SigV4, without the AWS - SDK) and seven more HTTP providers on `fetch`, JMAP, and decorators for failover, retries, - telemetry and logs. -2. **A reliability layer on top of it,** being designed. A persistent queue and outbox, an - append-only audit log, one model of delivery events, suppression, idempotency and failover - between providers. +1. **The transport,** here today, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0 and, + in 1.1, from [nodemailer](https://github.com/nodemailer/nodemailer), + [unemail](https://github.com/productdevbook/unemail) and + [sently](https://github.com/omqkhafi/sently): an SMTP client, sendmail, a MIME builder and DKIM + signing, Amazon SES v2 (signed with SigV4, without the AWS SDK) and seventeen more HTTP + providers on `fetch`, JMAP, and decorators for failover, retries, telemetry and logs. +2. **A reliability layer on top of it,** in part here. One model of delivery events read from + verified webhooks, idempotency keys, a circuit breaker and rate limits came in 1.1; a persistent + queue and outbox, an append-only audit log and suppression are being designed. ## The gap it fills @@ -28,14 +32,16 @@ That work is left to every application: a job that dies between "submitted" and retry after a timeout that sends the same password reset twice, a bounce that nobody records. The breadth of providers is not where the gap is. Each provider added is one more webhook format -and one more API that drifts, so beyond the eight that came with the port, a new provider waits -for a user who needs it, and the effort goes into what happens to a message after `send()`. +and one more API that drifts, so beyond the eighteen of 1.1 (the eight that came with the port and +the ten most used of unemail and sently), a new provider waits for a user who needs it, and the +effort goes into what happens to a message after `send()`. ## Design The transport is built; the rest of this list is the design of the reliability layer, which is being built. A failure after which a provider may have taken the message is already marked -`deliveryUnknown`, and `./retry` and `./pool` do not send it again. +`deliveryUnknown`, and `./retry` and `./pool` do not send it again; `./idempotency` reserves the +key of a message before it is sent, and `./webhooks` reads what happened to it afterwards. - **At least once, with deduplication.** Exactly-once delivery is not possible over SMTP or HTTP. A message carries an idempotency key, the outbox deduplicates by it, and a provider that has no @@ -65,10 +71,11 @@ being built. A failure after which a provider may have taken the message is alre 2. **Reliable sending:** the MIME builder, the SMTP client and DKIM, SES v2 and one more HTTP provider, an outbox in MongoDB and a BullMQ driver, idempotency, the classification of errors, the audit log and the correctness suite. -3. **A closed loop:** webhooks into the event model, a suppression gate, failover with health and a - circuit breaker, OpenTelemetry. -4. **Multi-tenant:** credentials, domains and quotas per tenant, List-Unsubscribe one-click, - scheduled sending, more providers. +3. **A closed loop:** webhooks into the event model, a circuit breaker and rate limits (all three + in 1.1), a suppression gate, failover with health, OpenTelemetry. +4. **Multi-tenant:** credentials, domains and quotas per tenant, scheduled sending as a delay in + the queue (providers that schedule do so in 1.1), List-Unsubscribe one-click and more providers + (both in 1.1). Out of scope: SMS, push and WhatsApp, inbound mail, and the management APIs of the providers (domains, templates, statistics), which their official SDKs already cover. diff --git a/docs/index.md b/docs/index.md index bd2afa2..9e23f7e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,7 +7,7 @@ titleTemplate: Email for Node.js, with a reliability layer hero: name: mailer text: Email for Node.js, with a reliability layer - tagline: SMTP, MIME and DKIM of its own, eight HTTP providers and JMAP, with failover, retries and telemetry around them. Zero dependencies. Next on top, what other libraries leave to you, a persistent queue, an audit log and delivery events. + tagline: SMTP, sendmail, MIME and DKIM of its own, eighteen HTTP providers and JMAP, with failover, retries, rate limits and idempotency around them, and verified webhooks into one model of delivery events. Zero dependencies. Next on top, a persistent queue and an audit log. image: src: /logo-mark.svg alt: mailer @@ -25,40 +25,43 @@ hero: features: - icon: ✉️ title: SMTP of its own - details: "Pooling, STARTTLS, AUTH PLAIN, LOGIN and OAuth 2.0, PIPELINING, SIZE, DSN and SMTPUTF8, on node:net and node:tls. Certificates are verified, and credentials never cross plain text." + details: "Pooling, STARTTLS, AUTH PLAIN, LOGIN and OAuth 2.0, PIPELINING, SIZE, DSN and SMTPUTF8, on node:net and node:tls, through an HTTP CONNECT proxy if you need one; and sendmail. Certificates are verified, and credentials never cross plain text." link: /transports/smtp linkText: SMTP transport - icon: 🔏 title: MIME and DKIM - details: "createMessage validates addresses, headers and attachments. The MIME keeps the Bcc out of the headers, carries inline images and iCalendar invitations, and is DKIM-signed with RSA or Ed25519." + details: "createMessage validates addresses, headers and attachments, and takes List-Unsubscribe with one-click. The MIME writes IDN domains as punycode, keeps the Bcc out of the headers, carries inline images and iCalendar invitations, and is DKIM-signed with RSA or Ed25519." link: /messages/compose linkText: Composing messages - icon: 🌐 - title: Eight HTTP providers and JMAP - details: "Amazon SES v2, Resend, SendGrid, Mailgun, Mailtrap, Maileroo, Lettermint and Plunk on fetch, and JMAP. One Transport contract and one Receipt with classified errors: swap the provider, keep the messages." + title: Eighteen HTTP providers and JMAP + details: "Amazon SES v2, Resend, SendGrid, Mailgun, Postmark, Brevo, Mailjet, MailerSend, SparkPost and nine more on fetch, and JMAP. One Transport contract and one Receipt with classified errors: swap the provider, keep the messages." link: /guide/getting-started#choosing-a-transport linkText: Choose a transport - icon: 🧩 title: Decorators around any transport - details: "A pool with strategies and failover, retries with backoff, jitter and Retry-After, OpenTelemetry spans and metrics, logs through your pino logger, and a mock for tests." + details: "A pool with strategies and failover, retries with backoff, jitter and Retry-After, a circuit breaker, token-bucket rate limits, idempotency keys, hooks, mirroring, OpenTelemetry spans and metrics, logs through your pino logger, and a mock for tests." link: /transports/pool - linkText: Pool, retry, telemetry + linkText: Pool, retry, breaker - icon: 🪶 title: Zero dependencies - details: "Built on node:* and fetch. The OpenTelemetry providers, the logger and, later, the queue driver come from your application. Node.js 22+, CommonJS with ESM imports, hand-written types." + details: "Built on node:* and fetch. The OpenTelemetry providers, the logger, React Email's render and, later, the queue driver come from your application. Node.js 22+, tested on Bun and Deno, CommonJS with ESM imports, hand-written types." link: /api/exports linkText: API - icon: 🧾 - title: "Next: the reliability layer" - details: "In design: a persistent queue and outbox, an append-only audit log, one model of delivery events, suppression and idempotency. None of the fourteen libraries compared has the three." - link: /guide/why - linkText: The plan + title: Delivery events, then the reliability layer + details: "The webhooks of fifteen providers, each verified before it is read, give one model of delivery events. In design: a persistent queue and outbox, an append-only audit log and suppression. None of the fourteen libraries compared has the three." + link: /guide/webhooks + linkText: Webhooks and events --- ::: tip Status -The transport layer is here, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0 (MIT): one -package, with a subpath per transport and decorator. The reliability layer is being designed; see -[Why @alexify/mailer?](/guide/why) for the plan. +The transport layer is here, ported from [upyo](https://github.com/dahlia/upyo) 0.7.0 (MIT), with +what [nodemailer](https://github.com/nodemailer/nodemailer), +[unemail](https://github.com/productdevbook/unemail) and +[sently](https://github.com/omqkhafi/sently) did better ported in 1.1: one package, with a subpath +per transport and decorator. Delivery events are here too; the queue and the audit log are being +designed. See [Why @alexify/mailer?](/guide/why) for the plan. ::: ## A quick look diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 0e2da44..9e271a8 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -1,21 +1,28 @@ # @alexify/mailer -> Zero-dependency email for Node.js, maintained by Alexis Technologies. Its own SMTP client, MIME -> builder, DKIM signing and HTTP providers (starting with Amazon SES v2), with a reliability layer -> on top that none of the common Node.js email libraries has: a persistent queue and outbox, an -> audit log, one model of delivery events, suppression and idempotency. +> Zero-dependency email for Node.js, Bun and Deno, maintained by Alexis Technologies. Its own SMTP +> client, sendmail, MIME builder, DKIM signing, eighteen HTTP providers and JMAP, with a +> reliability layer on top that none of the common Node.js email libraries has: one model of +> delivery events from verified webhooks and idempotency today, a persistent queue and outbox, an +> audit log and suppression next. Key facts: -- Status: 1.0 is the transport layer, ported from upyo 0.7.0 (MIT); the reliability layer - (queue, audit log, delivery events) is being designed and is not released yet +- Status: 1.0 is the transport layer, ported from upyo 0.7.0 (MIT); 1.1 ports what nodemailer, + unemail and sently did better and adds delivery events with verified webhooks; the queue and + the audit log are being designed and are not released yet - Package (npm): `@alexify/mailer`, installed with `npm install @alexify/mailer` -- Node.js >= 22 only; CommonJS with ESM named-import interop; hand-written d.ts per entry +- Node.js >= 22, tested on Bun and Deno too; CommonJS with ESM named-import interop; + hand-written d.ts per entry - Zero runtime dependencies: the transports are built on `node:*` modules and `fetch`; the - OpenTelemetry providers, the logger and the drivers of a queue or a database are passed in + OpenTelemetry providers, the logger, React Email's `render` and the drivers of a queue or a + database are passed in - Root entry `@alexify/mailer`: `createMessage`, the `Transport` contract, receipts, - attachments, calendar parts, Message-ID helpers -- Subpaths: `/mime` (MIME, DKIM), `/smtp`, `/jmap`, `/ses`, `/resend`, `/sendgrid`, `/mailgun`, - `/mailtrap`, `/maileroo`, `/lettermint`, `/plunk`, `/pool`, `/retry`, `/otel`, `/logger`, `/mock` + attachments, calendar parts, Message-ID helpers, the `DeliveryEvent` types +- Subpaths: `/mime` (MIME, DKIM), `/attachments`, `/render`, `/webhooks`; transports `/smtp`, + `/sendmail`, `/jmap`, `/ses`, `/resend`, `/sendgrid`, `/mailgun`, `/postmark`, `/brevo`, + `/mailjet`, `/mailersend`, `/sparkpost`, `/mailtrap`, `/maileroo`, `/lettermint`, + `/zeptomail`, `/smtp2go`, `/azure`, `/scaleway`, `/cloudflare`, `/plunk`, `/mock`; decorators + `/pool`, `/retry`, `/breaker`, `/ratelimit`, `/idempotency`, `/hooks`, `/tee`, `/otel`, `/logger` - Guide: https://mailerat.vercel.app/guide/getting-started - API: https://mailerat.vercel.app/api/exports - Comparison with 14 alternatives (Nodemailer, emailjs, Upyo, unemail, sently, provider SDKs): diff --git a/package.json b/package.json index 8059fdb..427e860 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@alexify/mailer", "version": "1.0.0", - "description": "Zero-dependency email for Node.js: its own SMTP client, MIME builder, DKIM, Amazon SES and seven more HTTP providers, JMAP, failover, retries and OpenTelemetry; a reliability layer (persistent queue, audit log, delivery events) is next", + "description": "Zero-dependency email for Node.js, Bun and Deno: its own SMTP client, sendmail, MIME builder and DKIM, eighteen HTTP providers and JMAP, verified webhooks into one model of delivery events, and decorators for failover, retries, circuit breaking, rate limits, idempotency and OpenTelemetry; a persistent queue and audit log are next", "license": "MIT", "type": "commonjs", "author": "Alex Dolid ", @@ -227,7 +227,13 @@ "suppression", "bounce", "deliverability", - "zero-dependency" + "zero-dependency", + "sendmail", + "circuit-breaker", + "rate-limit", + "react-email", + "bun", + "deno" ], "devDependencies": { "@opentelemetry/api": "^1.9.1", From c37df8f760e318ba0a42cf7e68d443634af36d7d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 00:35:28 +0300 Subject: [PATCH 050/143] fix: a refused TLS certificate fails every HTTP transport as configuration A certificate refused in the handshake was retried, as requestWithRetry retries any failure without a status, and the receipt of every HTTP transport read the words of its message, `fetch failed`: a retryable `network` failure. On Deno, whose fetch gives it no code, isUnsentRequestError found no sign of an unsent request and it was deliveryUnknown. isCertificateFailure knows it by the code along the causes (Node, Bun) or by Deno's text, `invalid peer certificate`, for classifyReceiptError and isUnsentRequestError; requestWithRetry throws it at the first attempt; and certificateFailure, first in the failure of every HTTP transport and of JMAP, makes the receipt a non-retryable `configuration` failure. The budgets of ./jmap, ./plunk and ./smtp2go go up half a kilobyte. Checked with each runtime's error on the eighteen providers and JMAP, raw messages included, and against an untrusted HTTPS server on Node, Bun and Deno: one connection, a `configuration` receipt. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++- CLAUDE.md | 14 ++- scripts/size-budgets.json | 6 +- src/core/receipt.js | 33 ++++- src/internal/api.js | 4 +- src/internal/http.js | 5 + src/internal/refusal.js | 36 +++++- src/internal/unsent.js | 14 +-- src/jmap/jmap-transport.js | 3 + src/jmap/raw-delivery.js | 3 + src/lettermint/lettermint-transport.js | 3 + src/maileroo/maileroo-transport.js | 6 +- src/mailgun/mailgun-transport.js | 3 + src/mailtrap/mailtrap-transport.js | 3 + src/plunk/plunk-transport.js | 3 + src/resend/resend-transport.js | 6 +- src/sendgrid/sendgrid-transport.js | 6 +- src/ses/ses-transport.js | 6 +- tests/runtime/http.test.js | 23 ++++ tests/unit/certificates.test.js | 165 +++++++++++++++++++++++++ 20 files changed, 317 insertions(+), 41 deletions(-) create mode 100644 tests/unit/certificates.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b1a90b..0de18b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,8 +35,10 @@ additive but the defaults listed below. - **The default DKIM `h=` signs `List-Unsubscribe` and `List-Unsubscribe-Post`** too, as RFC 8058 asks. - **Failures are classified more precisely:** a message a provider's limits refuse before any - request is a non-retryable `validation` failure, and a TLS certificate refused in the handshake - of `./smtp` a non-retryable `configuration` failure, where both were `unknown`. + request is a non-retryable `validation` failure, where it was `unknown`; and a TLS certificate + refused in the handshake is a non-retryable `configuration` failure at the first attempt, where + SMTP reported it as `unknown` and the HTTP transports retried it and reported a retryable + `network` failure (`deliveryUnknown` on Deno). - **On Deno, give the HTTP transports a `fetch` that keeps no idle connection** (`Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`): Deno's own sends a request again when a reused connection closes before its answer. See @@ -127,10 +129,12 @@ additive but the defaults listed below. ### Fixed -- **A TLS certificate refused in the handshake was `unknown` in an SMTP receipt,** classified by - the words of its message alone; and `classifyReceiptError` missed - `UNABLE_TO_VERIFY_LEAF_SIGNATURE`, what Node and Bun report for an untrusted CA. Both are now a - non-retryable `configuration` failure. +- **A TLS certificate refused in the handshake was misclassified everywhere:** `unknown` in an + SMTP receipt, a retried and retryable `network` failure on the HTTP transports (their receipts + read the words `fetch failed`), and `deliveryUnknown` on Deno, whose `fetch` gives it no code; + `classifyReceiptError` also missed `UNABLE_TO_VERIFY_LEAF_SIGNATURE`, what Node and Bun report + for an untrusted CA. It is now a non-retryable `configuration` failure at the first attempt, on + every transport and runtime. - **A message refused by a provider's limits before any request was `unknown`;** it is a non-retryable `validation` failure. - **A pooled SMTP connection was reused for any number of messages,** while servers limit the diff --git a/CLAUDE.md b/CLAUDE.md index 788ba29..3eceb7d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -446,9 +446,17 @@ fails without it; ported tests that pinned the old behavior say "Changed after t `classifyReceiptError` (36) took only the codes that name a certificate, which the leaf of an unknown issuer does not (`UNABLE_TO_VERIFY_LEAF_SIGNATURE`, what Node and Bun report for an untrusted CA). The receipt takes the classification of the error, by the code along its - causes, and `isCertificateCode` (`src/core/receipt.js`, shared with `isUnsentRequestError`) - knows that code: a refused certificate is a `configuration` failure, not retried, on Node, - Bun and Deno. + causes, and `isCertificateCode` (`src/core/receipt.js`) knows that code: a refused + certificate is a `configuration` failure, not retried, on Node, Bun and Deno. +87. every HTTP provider and jmap: a TLS certificate refused in the handshake was retried, as + `requestWithRetry` retries any failure without a status, and its receipt was classified by + the words of its message (`fetch failed`), a retryable `network` failure; on Deno, whose + `fetch` gives it no code, `isUnsentRequestError` found no sign of an unsent request and it was + `deliveryUnknown`. `isCertificateFailure` (`src/core/receipt.js`) knows it by the code along + the causes or by Deno's text (`invalid peer certificate`), for `classifyReceiptError` and + `isUnsentRequestError`; `requestWithRetry` throws it at the first attempt; and + `certificateFailure` (`internal/refusal.js`), first in the failure of every HTTP transport, + makes its receipt a non-retryable `configuration` failure. ## Code ported from nodemailer, unemail and sently diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d6ad948..89aca13 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -6,7 +6,7 @@ "./webhooks": 12.5, "./smtp": 23, "./sendmail": 10.5, - "./jmap": 12.5, + "./jmap": 13, "./ses": 8, "./resend": 6.5, "./sendgrid": 6.5, @@ -14,14 +14,14 @@ "./mailtrap": 6.5, "./maileroo": 6.5, "./lettermint": 6, - "./plunk": 5.5, + "./plunk": 6, "./postmark": 7, "./brevo": 7, "./mailjet": 8, "./mailersend": 7, "./sparkpost": 8, "./zeptomail": 7, - "./smtp2go": 7, + "./smtp2go": 7.5, "./azure": 8, "./scaleway": 7, "./cloudflare": 8, diff --git a/src/core/receipt.js b/src/core/receipt.js index b266cdb..297c1ee 100644 --- a/src/core/receipt.js +++ b/src/core/receipt.js @@ -115,8 +115,6 @@ const matchesWordRule = (rule, text, name) => { return rule.name !== undefined && rule.name === name; }; -// The first `code` along the chain of causes, as `fetch failed` keeps the -// socket error behind it. // A certificate refused during the TLS handshake: the verification errors of // OpenSSL that name one (CERT_HAS_EXPIRED, DEPTH_ZERO_SELF_SIGNED_CERT), Node's // ERR_TLS_CERT_ALTNAME_INVALID, and the leaf of an unknown issuer, which names @@ -124,6 +122,33 @@ const matchesWordRule = (rule, text, name) => { const isCertificateCode = (code) => typeof code === 'string' && (code.includes('CERT') || code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE'); +// Deno's fetch, on rustls, gives a refused certificate no code, only the text +// of its cause: "… client error (Connect): invalid peer certificate: …". +const CERTIFICATE_TEXT = /\binvalid peer certificate\b/i; + +/** + * Whether a failure is a TLS certificate refused in the handshake: by the code + * along its causes on Node and Bun, by their text on Deno. No retry passes + * it, and no request went out. + * + * @param error The failure. + * @returns `true` for a refused certificate. + * @internal + */ +const isCertificateFailure = (error) => { + let current = error; + for (let depth = 0; depth < 4 && current != null && typeof current === 'object'; depth++) { + if (isCertificateCode(current.code)) return true; + if (typeof current.message === 'string' && CERTIFICATE_TEXT.test(current.message)) { + return true; + } + current = current.cause; + } + return false; +}; + +// The first `code` along the chain of causes, as `fetch failed` keeps the +// socket error behind it. const findErrorCode = (error) => { let current = error; for (let depth = 0; depth < 4 && current != null && typeof current === 'object'; depth++) { @@ -153,7 +178,7 @@ const classifyReceiptError = (error) => { return { category: 'network', retryable: true, code: 'network' }; } // A certificate refused during the TLS handshake will not pass on a retry. - if (isCertificateCode(errorCode)) { + if (isCertificateFailure(error)) { return { category: 'configuration', retryable: false, code: 'configuration' }; } @@ -320,6 +345,6 @@ module.exports = { classifyReceiptError, createReceiptError, createFailedReceipt, - isCertificateCode, + isCertificateFailure, parseRetryAfter, }; diff --git a/src/internal/api.js b/src/internal/api.js index e5b3a87..25c7957 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -11,7 +11,7 @@ const { canonicalHeaderName } = require('./headers.js'); const { convertAttachments, visitCustomHeaders, withCalendarAttachment } = require('./provider.js'); const { isDeliveryUnknown } = require('./delivery-unknown.js'); const { requestWithRetry } = require('./http.js'); -const { refusalFailure } = require('./refusal.js'); +const { certificateFailure, refusalFailure } = require('./refusal.js'); const { createLog } = require('./log.js'); const MAX_ERROR_TEXT = 500; @@ -170,6 +170,8 @@ const createApiClient = (config, provider) => { */ const providerFailure = (provider, error) => { const message = error instanceof Error ? error.message : String(error); + const certificate = certificateFailure(provider, message, error); + if (certificate !== undefined) return certificate; if (error instanceof ProviderApiError) { return createFailedReceipt(message, { provider, diff --git a/src/internal/http.js b/src/internal/http.js index 1ff78f3..5b92a64 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -3,6 +3,7 @@ // retries and how long it waits) and passes it in. const { combineSignals } = require('../core/abort-signal.js'); +const { isCertificateFailure } = require('../core/receipt.js'); const { markDeliveryUnknown } = require('./delivery-unknown.js'); const { isUnsentRequestError } = require('./unsent.js'); const { abortReason, sleep } = require('./sleep.js'); @@ -152,6 +153,10 @@ const requestWithRetry = async (request) => { throw markDeliveryUnknown(wrapped ?? request.finalError?.(failure, attempt + 1) ?? failure); } } + // A certificate refused in the handshake refuses every attempt alike. + if (isCertificateFailure(failure)) { + throw settle(request.finalError?.(failure, attempt + 1) ?? failure); + } const isRetryable = isAccepted || (request.isRetryable ?? isRetryableFailure)(failure); if (!isRetryable) throw settle(failure); const retryAfterMs = Number.isFinite(failure.retryAfterMilliseconds) diff --git a/src/internal/refusal.js b/src/internal/refusal.js index c33e63a..3468a1b 100644 --- a/src/internal/refusal.js +++ b/src/internal/refusal.js @@ -1,8 +1,10 @@ -// Not ported: the receipt of a message a provider cannot take as it is, which -// its converter refuses before any request: over a limit of the provider, or -// with a field it cannot express. +// Not ported: the receipts of what fails before a provider could take a +// message: a message its converter refuses before any request (over a limit of +// the provider, or with a field it cannot express), and a TLS certificate +// refused in the handshake. -const { createFailedReceipt } = require('../core/receipt.js'); +const { createFailedReceipt, isCertificateFailure } = require('../core/receipt.js'); +const { isDeliveryUnknown } = require('./delivery-unknown.js'); /** * The refusal of a converter, a RangeError that only a converter throws, so @@ -27,4 +29,28 @@ const refusalFailure = (provider, error) => ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) : undefined; -module.exports = { RefusalError, refusalFailure }; +/** + * The receipt of a TLS certificate refused in the handshake, wrapped or not by + * the provider's own error: a non-retryable `configuration` failure, as no + * retry passes it; or `undefined` for another error. An earlier attempt of + * unknown delivery keeps the receipt so. + * + * @param provider The transport. + * @param message The text of the receipt. + * @param error What was thrown. + * @returns The failed receipt, or `undefined`. + * @internal + */ +const certificateFailure = (provider, message, error) => + isCertificateFailure(error) + ? createFailedReceipt(message, { + provider, + category: 'configuration', + code: 'configuration', + retryable: false, + deliveryUnknown: isDeliveryUnknown(error), + attempts: Number.isInteger(error.attempts) ? error.attempts : undefined, + }) + : undefined; + +module.exports = { RefusalError, certificateFailure, refusalFailure }; diff --git a/src/internal/unsent.js b/src/internal/unsent.js index 51f7048..1b33406 100644 --- a/src/internal/unsent.js +++ b/src/internal/unsent.js @@ -1,11 +1,10 @@ // Not ported: tells a request that never left from one that may // have reached the server, for a request that must not be sent twice. -const { isCertificateCode } = require('../core/receipt.js'); +const { isCertificateFailure } = require('../core/receipt.js'); // System error codes of a connection that was never made, so no request went -// out: a refused connection, an unknown or unreachable host. A certificate -// rejected during the TLS handshake also ends the connection before it. +// out: a refused connection, an unknown or unreachable host. const UNSENT_CODES = new Set([ 'ECONNREFUSED', 'ENOTFOUND', @@ -14,12 +13,12 @@ const UNSENT_CODES = new Set([ 'ENETUNREACH', ]); -const isUnsentCode = (code) => UNSENT_CODES.has(code) || isCertificateCode(code); - /** * Whether a failed request certainly never reached the server: somewhere in * its chain of causes is the code of a connection that was never made, as - * `fetch` reports it (`TypeError: fetch failed`, caused by the socket error). + * `fetch` reports it (`TypeError: fetch failed`, caused by the socket error), + * or a certificate refused in the TLS handshake, which ends the connection + * before any request. * Any other failure without a response, such as a timeout or a reset, may * have come after the server received the request. * @@ -28,9 +27,10 @@ const isUnsentCode = (code) => UNSENT_CODES.has(code) || isCertificateCode(code) * @internal */ const isUnsentRequestError = (error) => { + if (isCertificateFailure(error)) return true; let current = error; for (let depth = 0; depth < 4 && current != null; depth++) { - if (isUnsentCode(current.code)) return true; + if (UNSENT_CODES.has(current.code)) return true; current = current.cause; } return false; diff --git a/src/jmap/jmap-transport.js b/src/jmap/jmap-transport.js index 98ec824..3025917 100644 --- a/src/jmap/jmap-transport.js +++ b/src/jmap/jmap-transport.js @@ -19,6 +19,7 @@ const { findMailAccount } = require('./session.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { chunks, readItems } = require('../internal/batch.js'); const { schedulingRefusal } = require('../internal/schedule.js'); +const { certificateFailure } = require('../internal/refusal.js'); /** * JMAP capabilities required for email sending. @@ -992,6 +993,8 @@ const refuseScheduled = () => }); const createJmapFailure = (message, error, override = {}) => { + const certificate = certificateFailure('jmap', message, error); + if (certificate !== undefined) return certificate; const attempts = getAttemptCount(error); if (error instanceof JmapApiError) { diff --git a/src/jmap/raw-delivery.js b/src/jmap/raw-delivery.js index 5f6752b..8d8e2e1 100644 --- a/src/jmap/raw-delivery.js +++ b/src/jmap/raw-delivery.js @@ -7,6 +7,7 @@ const { RawMessageValidationError } = require('../core/raw-message.js'); const { JMAP_ERROR_TYPES, JmapApiError } = require('./errors.js'); const { JmapHttpClient } = require('./http-client.js'); const { findOnlyResponse, isRecord, rawOperation, uploadRawMessage } = require('./raw-upload.js'); +const { certificateFailure } = require('../internal/refusal.js'); const using = [ 'urn:ietf:params:jmap:core', @@ -277,6 +278,8 @@ const submitRawEmail = async (execute, accountId, emailId, identityId, envelope) // be retried, and a submission without a definite answer may have gone. const rawDeliveryFailure = (error, stage) => { const message = error instanceof Error ? error.message : String(error); + const certificate = certificateFailure('jmap', message, error); + if (certificate !== undefined) return certificate; if (error instanceof RawMessageValidationError) { return createFailedReceipt(message, { provider: 'jmap', diff --git a/src/lettermint/lettermint-transport.js b/src/lettermint/lettermint-transport.js index d7f40fe..d147fb8 100644 --- a/src/lettermint/lettermint-transport.js +++ b/src/lettermint/lettermint-transport.js @@ -11,6 +11,7 @@ const { const { convertMessage, generateIdempotencyKey } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); +const { certificateFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -244,6 +245,8 @@ const responseToReceipt = (response) => { }; const createLettermintFailure = (message, error) => { + const certificate = certificateFailure('lettermint', message, error); + if (certificate !== undefined) return certificate; if (error instanceof LettermintApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/maileroo/maileroo-transport.js b/src/maileroo/maileroo-transport.js index d8afe3c..c7b06c5 100644 --- a/src/maileroo/maileroo-transport.js +++ b/src/maileroo/maileroo-transport.js @@ -8,7 +8,7 @@ const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); -const { refusalFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Maileroo transport implementation for sending emails via Maileroo API. @@ -144,8 +144,8 @@ const responseToReceipt = (response) => { }; const createMailerooFailure = (message, error) => { - const refusal = refusalFailure('maileroo', error); - if (refusal !== undefined) return refusal; + const known = refusalFailure('maileroo', error) ?? certificateFailure('maileroo', message, error); + if (known !== undefined) return known; if (error instanceof MailerooApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index 3d6faec..a4c3dcf 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -10,6 +10,7 @@ const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); +const { certificateFailure } = require('../internal/refusal.js'); /** * Mailgun transport implementation for sending emails via Mailgun API. @@ -246,6 +247,8 @@ class MailgunTransport { } const createMailgunFailure = (message, error) => { + const certificate = certificateFailure('mailgun', message, error); + if (certificate !== undefined) return certificate; if (error instanceof MailgunApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/mailtrap/mailtrap-transport.js b/src/mailtrap/mailtrap-transport.js index 2c9823c..0e4ac8d 100644 --- a/src/mailtrap/mailtrap-transport.js +++ b/src/mailtrap/mailtrap-transport.js @@ -12,6 +12,7 @@ const { const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); +const { certificateFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -250,6 +251,8 @@ const toReceipt = (response, options) => { }; const createMailtrapFailure = (message, error) => { + const certificate = certificateFailure('mailtrap', message, error); + if (certificate !== undefined) return certificate; if (error instanceof MailtrapResponseError) { return createFailedReceipt(message, { deliveryUnknown: true, diff --git a/src/plunk/plunk-transport.js b/src/plunk/plunk-transport.js index 07e16ae..53770eb 100644 --- a/src/plunk/plunk-transport.js +++ b/src/plunk/plunk-transport.js @@ -9,6 +9,7 @@ const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); +const { certificateFailure } = require('../internal/refusal.js'); /** * Plunk transport implementation for sending emails via Plunk API. @@ -225,6 +226,8 @@ class PlunkTransport { } const createPlunkFailure = (message, error) => { + const certificate = certificateFailure('plunk', message, error); + if (certificate !== undefined) return certificate; if (error instanceof PlunkApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/resend/resend-transport.js b/src/resend/resend-transport.js index 7f11d66..ceb408c 100644 --- a/src/resend/resend-transport.js +++ b/src/resend/resend-transport.js @@ -13,7 +13,7 @@ const { const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); -const { refusalFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Resend transport implementation for sending emails via Resend API. @@ -343,8 +343,8 @@ class ResendTransport { } const createResendFailure = (message, error) => { - const refusal = refusalFailure('resend', error); - if (refusal !== undefined) return refusal; + const known = refusalFailure('resend', error) ?? certificateFailure('resend', message, error); + if (known !== undefined) return known; if (error instanceof ResendApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/sendgrid/sendgrid-transport.js b/src/sendgrid/sendgrid-transport.js index 7d05acd..e93be12 100644 --- a/src/sendgrid/sendgrid-transport.js +++ b/src/sendgrid/sendgrid-transport.js @@ -7,7 +7,7 @@ const { createSendGridConfig } = require('./config.js'); const { verifyByRequest } = require('../internal/verify.js'); const { SendGridApiError, SendGridHttpClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); -const { refusalFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); @@ -259,8 +259,8 @@ class SendGridTransport { } const createSendGridFailure = (message, error) => { - const refusal = refusalFailure('sendgrid', error); - if (refusal !== undefined) return refusal; + const known = refusalFailure('sendgrid', error) ?? certificateFailure('sendgrid', message, error); + if (known !== undefined) return known; if (error instanceof SendGridApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index fb48bbe..9e828cf 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -10,7 +10,7 @@ const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); -const { refusalFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const { verifyByRequest } = require('../internal/verify.js'); /** @@ -279,8 +279,8 @@ const checkAuthentication = (authentication) => { }; const createSesFailure = (message, error) => { - const refusal = refusalFailure('ses', error); - if (refusal !== undefined) return refusal; + const known = refusalFailure('ses', error) ?? certificateFailure('ses', message, error); + if (known !== undefined) return known; if (error instanceof SesApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/tests/runtime/http.test.js b/tests/runtime/http.test.js index e96d44a..cbaf751 100644 --- a/tests/runtime/http.test.js +++ b/tests/runtime/http.test.js @@ -3,9 +3,11 @@ const { test } = require('node:test'); const assert = require('node:assert'); const http = require('node:http'); +const https = require('node:https'); const { createMessage, readAttachmentContent } = require('@alexify/mailer'); const { PostmarkTransport } = require('@alexify/mailer/postmark'); const { fromUrl } = require('@alexify/mailer/attachments'); +const { credentials } = require('../helpers/smtp/servers.js'); // A server that records each request with its body, then answers as // `answer(request, response, count)` says. @@ -143,6 +145,27 @@ test( }, ); +test('Runtime HTTP: a certificate of an unknown CA fails at once, as configuration', async (t) => { + // An HTTPS server with the certificate of the test CA, which the runtime + // does not trust; each TCP connection is an attempt. + let connections = 0; + const listener = https.createServer(credentials, (_request, response) => accepted(response)); + listener.on('connection', (socket) => { + connections++; + socket.on('error', () => {}); + }); + listener.on('tlsClientError', () => {}); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => listener.close(() => resolve()))); + const url = `https://localhost:${listener.address().port}`; + const receipt = await postmark(url, { retries: 2 }).send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.errors[0].category, 'configuration', receipt.errorMessages.join()); + assert.strictEqual(receipt.retryable, false); + assert.strictEqual(receipt.deliveryUnknown, undefined); + assert.strictEqual(connections, 1); +}); + test('Runtime attachments: fromUrl reads through the fetch of the runtime, checking each URL', async (t) => { const { requests, url } = await server(t, (request, response) => { if (request.url === '/moved') { diff --git a/tests/unit/certificates.test.js b/tests/unit/certificates.test.js new file mode 100644 index 0000000..bbdd32b --- /dev/null +++ b/tests/unit/certificates.test.js @@ -0,0 +1,165 @@ +// Not ported: a TLS certificate refused in the handshake, as the fetch of +// Node, Bun and Deno report it, on every HTTP transport. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createMessage, classifyReceiptError } = require('../../index.js'); +const { isUnsentRequestError } = require('../../src/internal/unsent.js'); +const { AzureTransport } = require('../../src/azure/index.js'); +const { BrevoTransport } = require('../../src/brevo/index.js'); +const { CloudflareTransport } = require('../../src/cloudflare/index.js'); +const { JmapTransport } = require('../../src/jmap/index.js'); +const { LettermintTransport } = require('../../src/lettermint/index.js'); +const { MailerooTransport } = require('../../src/maileroo/index.js'); +const { MailerSendTransport } = require('../../src/mailersend/index.js'); +const { MailgunTransport } = require('../../src/mailgun/index.js'); +const { MailjetTransport } = require('../../src/mailjet/index.js'); +const { MailtrapTransport } = require('../../src/mailtrap/index.js'); +const { PlunkTransport } = require('../../src/plunk/index.js'); +const { PostmarkTransport } = require('../../src/postmark/index.js'); +const { ResendTransport } = require('../../src/resend/index.js'); +const { ScalewayTransport } = require('../../src/scaleway/index.js'); +const { SendGridTransport } = require('../../src/sendgrid/index.js'); +const { SesTransport } = require('../../src/ses/index.js'); +const { Smtp2goTransport } = require('../../src/smtp2go/index.js'); +const { SparkPostTransport } = require('../../src/sparkpost/index.js'); +const { ZeptoMailTransport } = require('../../src/zeptomail/index.js'); + +const message = () => + createMessage({ + from: 'alex@example.com', + to: 'bob@example.com', + subject: 'Hello', + content: { text: 'Hi' }, + }); + +// What each runtime's fetch rejects with when the server's certificate is not +// trusted: Node a cause with the OpenSSL code, Bun the code on the error +// itself, Deno (rustls) only the text of its cause. +const refusals = { + node: () => + new TypeError('fetch failed', { + cause: Object.assign(new Error('unable to verify the first certificate'), { + code: 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + }), + }), + bun: () => + Object.assign(new TypeError('unable to verify the first certificate'), { + code: 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + }), + deno: () => + new TypeError('fetch failed', { + cause: new Error( + 'error sending request for url (https://api.example.com/): client error (Connect): ' + + 'invalid peer certificate: UnknownIssuer', + ), + }), +}; + +// Retries allowed, so that one taken shows as a second request. +const options = { retries: 2 }; +const transports = { + SES: (fetch) => + new SesTransport({ + ...options, + fetch, + authentication: { type: 'credentials', accessKeyId: 'a', secretAccessKey: 'b' }, + }), + Resend: (fetch) => new ResendTransport({ ...options, fetch, apiKey: 'key' }), + SendGrid: (fetch) => new SendGridTransport({ ...options, fetch, apiKey: 'key' }), + Mailgun: (fetch) => + new MailgunTransport({ ...options, fetch, apiKey: 'key', domain: 'mg.example.com' }), + Mailtrap: (fetch) => new MailtrapTransport({ ...options, fetch, apiToken: 'token' }), + Maileroo: (fetch) => new MailerooTransport({ ...options, fetch, apiKey: 'key' }), + Lettermint: (fetch) => new LettermintTransport({ ...options, fetch, apiToken: 'token' }), + Plunk: (fetch) => new PlunkTransport({ ...options, fetch, apiKey: 'key' }), + Postmark: (fetch) => new PostmarkTransport({ ...options, fetch, serverToken: 'token' }), + Brevo: (fetch) => new BrevoTransport({ ...options, fetch, apiKey: 'key' }), + Mailjet: (fetch) => new MailjetTransport({ ...options, fetch, apiKey: 'a', apiSecret: 'b' }), + MailerSend: (fetch) => new MailerSendTransport({ ...options, fetch, apiKey: 'key' }), + SparkPost: (fetch) => new SparkPostTransport({ ...options, fetch, apiKey: 'key' }), + ZeptoMail: (fetch) => new ZeptoMailTransport({ ...options, fetch, token: 'token' }), + SMTP2GO: (fetch) => new Smtp2goTransport({ ...options, fetch, apiKey: 'key' }), + Azure: (fetch) => + new AzureTransport({ + ...options, + fetch, + connectionString: `endpoint=https://contoso.communication.azure.com/;accesskey=${btoa('k')}`, + }), + Scaleway: (fetch) => + new ScalewayTransport({ + ...options, + fetch, + secretKey: 'key', + projectId: '11111111-2222-4333-8444-555555555555', + }), + Cloudflare: (fetch) => + new CloudflareTransport({ + ...options, + fetch, + accountId: '023e105f4ecef8ad9ca31a8372d0c353', + apiToken: 'token', + }), + JMAP: (fetch) => + new JmapTransport({ + ...options, + fetch, + sessionUrl: 'https://jmap.example.com/session', + bearerToken: 'token', + }), +}; + +test('Certificates: a refused certificate is a configuration failure, sent nothing', () => { + for (const [runtime, refusal] of Object.entries(refusals)) { + assert.deepStrictEqual( + classifyReceiptError(refusal()), + { category: 'configuration', retryable: false, code: 'configuration' }, + runtime, + ); + assert.strictEqual(isUnsentRequestError(refusal()), true, runtime); + } + const reset = new TypeError('fetch failed', { cause: new Error('connection reset by peer') }); + assert.strictEqual(classifyReceiptError(reset).category, 'network'); + assert.strictEqual(isUnsentRequestError(reset), false); +}); + +test('Certificates: every HTTP transport fails at once, as configuration, not retried', async () => { + for (const [runtime, refusal] of Object.entries(refusals)) { + for (const [name, create] of Object.entries(transports)) { + let requests = 0; + const fetch = async () => { + requests++; + throw refusal(); + }; + const receipt = await create(fetch).send(message()); + const label = `${name} on ${runtime}`; + // Changed in this fix: Node's and Bun's were retried as network + // failures and Deno's taken for an unknown delivery. + assert.strictEqual(requests, 1, label); + assert.strictEqual(receipt.successful, false, label); + assert.strictEqual(receipt.errors[0].category, 'configuration', label); + assert.strictEqual(receipt.retryable, false, label); + assert.strictEqual(receipt.deliveryUnknown, undefined, label); + } + } +}); + +test('Certificates: so are the raw messages of JMAP and the providers that send them', async () => { + const raw = { + envelope: { from: 'alex@example.com', to: ['bob@example.com'] }, + content: new TextEncoder().encode( + 'From: alex@example.com\r\nTo: bob@example.com\r\nSubject: Hi\r\n\r\nHi\r\n', + ), + }; + for (const name of ['SES', 'Mailgun', 'SparkPost', 'Cloudflare', 'JMAP']) { + let requests = 0; + const fetch = async () => { + requests++; + throw refusals.deno(); + }; + const receipt = await transports[name](fetch).sendRaw(raw); + assert.strictEqual(requests, 1, name); + assert.strictEqual(receipt.errors[0].category, 'configuration', name); + assert.strictEqual(receipt.retryable, false, name); + assert.strictEqual(receipt.deliveryUnknown, undefined, name); + } +}); From 89c5279afdf7b405fff5a6d00f39d2d6dab96653 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 00:53:33 +0300 Subject: [PATCH 051/143] chore(release): 1.1.0 package.json and src/version.js go to 1.1.0, CHANGELOG [Unreleased] becomes [1.1.0] - 2026-10-18 with its compare link, and the version of @alexify/mailer in the comparison follows. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++++- docs/comparison.json | 2 +- docs/guide/comparison.md | 2 +- package.json | 2 +- src/version.js | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0de18b6..abbae17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2026-10-18 + The features of three libraries of the [comparison](https://mailerat.vercel.app/guide/comparison) where they led this package, ported onto its contracts: [nodemailer](https://github.com/nodemailer/nodemailer) 10.0.16 (MIT-0, © 2011-2023 Andris @@ -560,5 +562,6 @@ for code written against upyo 0.7: `./resend`, `./sendgrid`, `./mailgun`, `./plunk` and `./jmap` used to retry no 4xx at all, and waited without jitter, the first three without a cap. -[unreleased]: https://github.com/Alexis-Technologies/mailer/compare/v1.0.0...HEAD +[unreleased]: https://github.com/Alexis-Technologies/mailer/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/Alexis-Technologies/mailer/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/Alexis-Technologies/mailer/releases/tag/v1.0.0 diff --git a/docs/comparison.json b/docs/comparison.json index f4a2633..9855560 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -494,7 +494,7 @@ "text": "MIT" }, "version": { - "text": "1.0.0" + "text": "1.1.0" }, "lastCommit": { "text": "2026-10 (this release)" diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index d407df6..91f944a 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -114,7 +114,7 @@ it. Runtime dependencies0000–1 (@noble/hashes)0000422+ (axios)3 (axios)02 (axios, qs)9 Module formatCJS, ESM importESM + CJSESM onlyESM + CJS + JSRESM onlyESM onlyESM + CJSESM + CJS + JSRESM + CJSESM + CJSCJS onlyESM + CJSCJS onlyCJS onlyESM + CJS LicenseMITMIT-0MITMITMITMITMITMITMITMITMITMITMITMITApache-2.0 -Version (2026-10-07)1.0.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0 +Version (2026-10-07)1.1.010.0.165.0.30.7.0 (unreleased; release 0.6.0)0.7.01.2.21.2.10.8.53.0.24 (tag 3.0.28)6.32.18.1.614.0.25.1.04.10.0 (HEAD ahead)3.1147.0 Last commit2026-10 (this release)2026-10-072026-08-262026-09-092026-09-022026-08-192025-11-152026-09-082026-01-282026-10-062025-09-192026-10-072026-07-102026-10-072026-10-06 Tests: files / cases167 / ~234061 / ~133015 / ~153109 / ~130040 / ~77366 / ~5324 / ~18914 / ~2586 / ~3039 / ~44143 / ~45049 files23 (16 integration)55 files2 smoke (the rest in the monorepo) Owner / maintenance risk1 author, Alexis Technologies1 maintainerno SECURITY.md1 main author1 author1 author, 0.1 → 1.2 in ~80 days1 author1 author, 0.xno SECURITY.mdvendorvendor, old toolchainvendorvendorvendorvendor diff --git a/package.json b/package.json index 427e860..be9cb64 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alexify/mailer", - "version": "1.0.0", + "version": "1.1.0", "description": "Zero-dependency email for Node.js, Bun and Deno: its own SMTP client, sendmail, MIME builder and DKIM, eighteen HTTP providers and JMAP, verified webhooks into one model of delivery events, and decorators for failover, retries, circuit breaking, rate limits, idempotency and OpenTelemetry; a persistent queue and audit log are next", "license": "MIT", "type": "commonjs", diff --git a/src/version.js b/src/version.js index 3914683..f3e9eba 100644 --- a/src/version.js +++ b/src/version.js @@ -3,4 +3,4 @@ // require of package.json would bundle the whole manifest into ./otel. // tests/unit/version.test.js keeps it equal to package.json, and the release // process bumps both. -module.exports = { name: '@alexify/mailer', version: '1.0.0' }; +module.exports = { name: '@alexify/mailer', version: '1.1.0' }; From 4130c3baf7fdbe41d467d5dd579907c5ddd3cf72 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:05:45 +0300 Subject: [PATCH 052/143] fix(webhooks): keep an id as taken once its events are handled verify() kept the id of a request as taken before the application handled its events, and the docs answered a replay with 200, so a provider's redelivery after a handler that failed was refused and the event lost. A replay store now reserves an id for leaseMs and keeps it once committed (reserve, commit, release); verify() gives the events with commit() and release(), commit(events) of a batch keeps the events handled, an id that several events share is kept once all are, and a failure releases what was reserved. Another delivery while an id is held is refused as pending. replayStore is required where a verifier keeps ids, as an implicit in-process store is shared by neither several processes nor a verifier made per request. A refusal carries the HTTP status to answer with, 503 for a certificate that could not be read. The HMAC, timestamp, secret, recipient and bounce helpers the verifiers repeated live in request.js. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +- docs/guide/webhooks.md | 150 +++++++--- scripts/size-budgets.json | 2 +- src/webhooks/azure.js | 35 +-- src/webhooks/brevo.js | 10 +- src/webhooks/index.d.ts | 106 ++++++-- src/webhooks/lettermint.js | 35 ++- src/webhooks/maileroo.js | 36 +-- src/webhooks/mailersend.js | 23 +- src/webhooks/mailgun.js | 49 ++-- src/webhooks/mailjet.js | 10 +- src/webhooks/mailtrap.js | 36 +-- src/webhooks/postmark.js | 10 +- src/webhooks/request.js | 391 +++++++++++++++++++++++---- src/webhooks/resend.js | 46 +--- src/webhooks/scaleway.js | 35 ++- src/webhooks/sendgrid.js | 45 +-- src/webhooks/ses.js | 65 ++--- src/webhooks/smtp2go.js | 10 +- src/webhooks/sns.js | 40 ++- src/webhooks/sparkpost.js | 10 +- tests/types/webhooks.test-d.ts | 39 ++- tests/unit/webhooks-mailgun.test.js | 10 +- tests/unit/webhooks-more.test.js | 19 +- tests/unit/webhooks-others.test.js | 48 +++- tests/unit/webhooks-request.test.js | 263 +++++++++++++++++- tests/unit/webhooks-resend.test.js | 18 +- tests/unit/webhooks-sendgrid.test.js | 22 +- tests/unit/webhooks-ses.test.js | 59 +++- 29 files changed, 1182 insertions(+), 447 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abbae17..98f32b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,7 +106,12 @@ additive but the defaults listed below. Postmark, MailerSend, Mailtrap, Brevo, Mailjet, SparkPost, Lettermint, Maileroo, SMTP2GO, Azure and Scaleway. Nothing is read before it is verified: a signature where the provider signs (SNS with a topic allowlist, Svix, ECDSA, HMAC), its Basic credentials or token where it does not, - within a window of time and once by its id (`createMemoryReplayStore`, or any store). + within a window of time and once by its id. `verify()` reserves the id for `leaseMs` and gives + the events with `commit()` and `release()`: an id is kept as taken once the application has + handled its events, so a delivery whose handler failed is taken again, and another delivery of + it meanwhile is refused as `pending`. `replayStore` is required where a verifier keeps ids + (`createMemoryReplayStore()` in one process, a shared store of `reserve`, `commit` and + `release`, or `null`), and a refusal carries the `status` to answer the provider with. - **Bun and Deno:** CI runs the unit suite and the new `tests/runtime` (the package by its name, with no mock: sockets, TLS, DKIM, `fetch`, a child process) on Bun 1.4.3 and Deno 2.9.6, as well as on Node 22 and 24. `package.json` declares `"type": "commonjs"`, which Deno needs and diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 97329fb..3ac7d85 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -14,29 +14,50 @@ keeps the body as a buffer. ```js const express = require('express'); -const { createSesWebhookVerifier, WebhookVerificationError } = require('@alexify/mailer/webhooks'); - +const { + createMemoryReplayStore, + createSesWebhookVerifier, + WebhookVerificationError, +} = require('@alexify/mailer/webhooks'); + +// One process; several share a store of their own (see Time and replays). +const replayStore = createMemoryReplayStore(); const ses = createSesWebhookVerifier({ topicArns: ['arn:aws:sns:eu-west-1:123456789012:ses-events'], + replayStore, }); app.post('/webhooks/ses', express.raw({ type: '*/*' }), async (req, res) => { + let delivery; try { - const { events, confirmation } = await ses.verify({ body: req.body, headers: req.headers }); - if (confirmation) await subscriptions.review(confirmation); - for (const event of events) await deliveries.record(event); - res.sendStatus(200); + delivery = await ses.verify({ body: req.body, headers: req.headers }); } catch (error) { if (!(error instanceof WebhookVerificationError)) throw error; - // A replay was delivered before: tell the provider it arrived. - res.sendStatus(error.reason === 'replay' ? 200 : 401); + return res.sendStatus(error.status); + } + try { + if (delivery.confirmation) await subscriptions.review(delivery.confirmation); + for (const event of delivery.events) await deliveries.record(event); + await delivery.commit(); + res.sendStatus(200); + } catch (error) { + // The provider delivers the request again, and it is taken again. + await delivery.release(); + throw error; } }); ``` -A refused request throws a `WebhookVerificationError` whose `reason` says why: -`signature`, `timestamp`, `replay`, `payload`, `certificate`, `topic` or -`credentials`. +A refused request throws a `WebhookVerificationError` whose `reason` says why, +and whose `status` is what to answer the provider with: + +| `reason` | `status` | | +| --- | --- | --- | +| `replay` | 200 | Taken before: the provider may stop. | +| `pending` | 503 | Being handled by another delivery of it: the provider tries again. | +| `payload` | 400 | Verified, or too long, but not a body of the provider. | +| `certificate` | 401, or 503 | A signing certificate refused, or one that could not be read. | +| `signature`, `timestamp`, `topic`, `credentials` | 401 | Not the provider's, or too old. | ## Events @@ -63,18 +84,57 @@ has no name for, kept with its `raw` record. ## Time and replays -A signed request is taken only within `toleranceMs` of now, and only once: its -id goes into a replay store, in this process by default. A request taken before -is refused as `replay`; answer it with a 2xx, as the provider delivered it -before. Several processes share a store of their own, such as one on Redis, -with `add(id, ttlMs)` that answers whether the id is new; `replayStore: null` -turns the check off, for an application that deduplicates by the `id` of the -events itself. +A signed request is taken only within `toleranceMs` of now, and only once, by +the id the provider gives it, in two steps: + +1. `verify()` **reserves** the id in the replay store for `leaseMs`, and gives + the events with `commit()` and `release()`. +2. The application **commits** the events once it has handled them: the id is + then kept as taken, and a request taken before is refused as `replay`. If + handling them failed, it **releases** them, and the provider's next delivery + is taken again. A handler that died releases nothing, but its lease runs + out, and the delivery after it is taken. + +While an id is reserved, another delivery of the same request is refused as +`pending`, with 503, so that the provider tries again later rather than lose an +event whose first handler might still fail. Delivery is at least once: an +application whose handler fails half-way handles the first events again, and +deduplicates them by their `id`. + +`commit(events)` keeps only the events given: in a batch (SendGrid, Mailtrap, +Azure), the events handled are taken, and the others come back on the next +delivery. An SNS message of several recipients is kept once all its events +are committed. + +`replayStore` is required where a verifier keeps ids: +`createMemoryReplayStore()` in one process; for several processes, or a +function that makes its verifier for each request, a store they share; `null` +for an application that deduplicates the events by their `id` itself. A shared +store implements three methods: + +```js +// Redis, with node-redis: an atomic reservation, a commit, and a release +// that frees a reservation only, never a committed id. +const RELEASE = "if redis.call('get', KEYS[1]) == 'pending' then return redis.call('del', KEYS[1]) end"; +const replayStore = { + async reserve(id, leaseMs) { + if (await redis.set(id, 'pending', { NX: true, PX: leaseMs })) return 'reserved'; + return (await redis.get(id)) === 'done' ? 'done' : 'pending'; + }, + async commit(id, ttlMs) { + await redis.set(id, 'done', { PX: ttlMs }); + }, + async release(id) { + await redis.eval(RELEASE, { keys: [id] }); + }, +}; +``` | Option | Default | | | --- | --- | --- | +| `replayStore` | required | The ids taken, or `null`. | +| `leaseMs` | 30 seconds | How long an id stays reserved while it is handled: longer than a handler takes, shorter than the provider's first retry (20 seconds for SNS). | | `toleranceMs` | 5 minutes (an hour for SES) | How far from now a request may be signed. | -| `replayStore` | in this process | The ids already taken, or `null`. | | `maxBodyBytes` | 5 MiB | The longest body taken. | | `now` | `Date.now` | The clock. | @@ -97,7 +157,10 @@ of the message are its `tags`, as `./ses` sends them, and its other tags its ## Resend ```js -const resend = createResendWebhookVerifier({ secret: process.env.RESEND_WEBHOOK_SECRET }); +const resend = createResendWebhookVerifier({ + secret: process.env.RESEND_WEBHOOK_SECRET, + replayStore, +}); ``` Resend signs its webhooks with Svix: the HMAC-SHA256 of `svix-id`, @@ -113,14 +176,15 @@ its metadata, as `./resend` sends them. ```js const sendgrid = createSendGridWebhookVerifier({ publicKey: process.env.SENDGRID_WEBHOOK_PUBLIC_KEY, + replayStore, }); ``` Turn on the Signed Event Webhook in SendGrid and pass its verification key (base64, as SendGrid shows it, or PEM). The signature is ECDSA P-256 over the timestamp and the body. SendGrid posts events in batches: each event is taken -once by its `sg_event_id`, so a batch sent again keeps only what is new, and is -refused as `replay` when nothing is. A `bounce` is hard, a `blocked` one soft; a +once by its `sg_event_id`, so a batch sent again keeps only what is not +committed, and is refused as `replay` when nothing is left. A `bounce` is hard, a `blocked` one soft; a `dropped` message is `rejected`; an unsubscribe is `unsubscribed`. The categories are the tags of the message and its custom arguments its metadata; `messageId` is the `X-Message-Id` of the receipt, without the filter SendGrid @@ -131,6 +195,7 @@ adds after it. ```js const mailgun = createMailgunWebhookVerifier({ signingKey: process.env.MAILGUN_WEBHOOK_SIGNING_KEY, + replayStore, }); ``` @@ -149,16 +214,22 @@ const postmark = createPostmarkWebhookVerifier({ username: 'hook', password: 's3 Postmark signs nothing, so the URL of the webhook carries credentials, which the verifier requires and compares in constant time. With no time or id -signed, nothing holds a request to a window or takes it once: deduplicate by -the `id` of the events. A bounce is read by its type: `HardBounce` is hard, +signed, nothing holds a request to a window or takes it once, and it needs no +`replayStore`: deduplicate by the `id` of the events. A bounce is read by its type: `HardBounce` is hard, `SoftBounce` soft, `Transient` deferred, `SpamComplaint` a complaint, `ManuallyDeactivated` a rejection. ## MailerSend and Mailtrap ```js -const mailersend = createMailerSendWebhookVerifier({ secret: process.env.MAILERSEND_WEBHOOK_SECRET }); -const mailtrap = createMailtrapWebhookVerifier({ secret: process.env.MAILTRAP_WEBHOOK_SECRET }); +const mailersend = createMailerSendWebhookVerifier({ + secret: process.env.MAILERSEND_WEBHOOK_SECRET, + replayStore, +}); +const mailtrap = createMailtrapWebhookVerifier({ + secret: process.env.MAILTRAP_WEBHOOK_SECRET, + replayStore, +}); ``` Both sign the raw body with HMAC-SHA256, in hexadecimal in `Signature` @@ -200,8 +271,14 @@ request to a window or takes it once. ## Lettermint and Maileroo ```js -const lettermint = createLettermintWebhookVerifier({ secret: process.env.LETTERMINT_WEBHOOK_SECRET }); -const maileroo = createMailerooWebhookVerifier({ secret: process.env.MAILEROO_WEBHOOK_SECRET }); +const lettermint = createLettermintWebhookVerifier({ + secret: process.env.LETTERMINT_WEBHOOK_SECRET, + replayStore, +}); +const maileroo = createMailerooWebhookVerifier({ + secret: process.env.MAILEROO_WEBHOOK_SECRET, + replayStore, +}); ``` Lettermint signs the time and the body: `X-Lettermint-Signature: t=…,v1=…`, the @@ -228,12 +305,20 @@ payload. ## Azure Communication Services ```js -const azure = createAzureWebhookVerifier({ token: process.env.EVENT_GRID_TOKEN }); +const azure = createAzureWebhookVerifier({ token: process.env.EVENT_GRID_TOKEN, replayStore }); app.post('/webhooks/azure', express.raw({ type: '*/*' }), async (req, res) => { - const { events, validation } = await azure.verify({ body: req.body, headers: req.headers }); - if (validation) return res.json({ validationResponse: validation.validationCode }); - for (const event of events) await deliveries.record(event); + const delivery = await azure.verify({ body: req.body, headers: req.headers }); + if (delivery.validation) { + return res.json({ validationResponse: delivery.validation.validationCode }); + } + try { + for (const event of delivery.events) await deliveries.record(event); + await delivery.commit(); + } catch (error) { + await delivery.release(); + throw error; + } res.sendStatus(200); }); ``` @@ -256,6 +341,7 @@ const trustChain = fs.readFileSync('scaleway-sns-trust-chain.pem', 'utf8'); const scaleway = createScalewayWebhookVerifier({ topicArns: ['arn:scw:sns:fr-par:project-…:tem-events'], trustChain, + replayStore, }); ``` diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 89aca13..19ad2bb 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4, "./render": 3, - "./webhooks": 12.5, + "./webhooks": 13, "./smtp": 23, "./sendmail": 10.5, "./jmap": 13, diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index 09312b8..5ea17a9 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -3,11 +3,13 @@ // sently and unemail have no Azure webhook. const { + DAY_REPLAY_TTL_MS, WebhookVerificationError, bodyBytesOf, - checkReplay, + claimDelivery, createEvent, dateOf, + deliveryOf, headerOf, isRecord, parseJsonOf, @@ -18,7 +20,6 @@ const { // Event Grid retries a delivery for a day: an event id is kept a day and an // hour. -const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; const VALIDATION = 'Microsoft.EventGrid.SubscriptionValidationEvent'; const DELIVERY = 'Microsoft.Communication.EmailDeliveryReportReceived'; const ENGAGEMENT = 'Microsoft.Communication.EmailEngagementTrackingReportReceived'; @@ -75,13 +76,15 @@ const eventOf = (record) => { * taken once by its id. The validation of a new subscription is handed back, * for the application to answer `{ validationResponse: validationCode }`. * - * @param options `token`, `header`; `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events, validation? }`. + * @param options `token`, `header`; `replayStore`; `leaseMs` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, validation?, commit, + * release }`. * @throws {TypeError} If an option is invalid. */ const createAzureWebhookVerifier = (options = {}) => { const checkToken = tokenCheckOf(options, 'Event Grid'); - const common = resolveCommonOptions(options, REPLAY_TTL_MS); + const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -93,13 +96,12 @@ const createAzureWebhookVerifier = (options = {}) => { if (!isRecord(first) || first.eventType !== VALIDATION || !isRecord(first.data)) { throw new WebhookVerificationError('payload', 'The body is not a validation event.'); } - return { - events: [], + return deliveryOf([], { validation: { validationCode: first.data.validationCode, validationUrl: first.data.validationUrl, }, - }; + }); } checkToken(request); const payload = parseJsonOf(bytes); @@ -109,23 +111,12 @@ const createAzureWebhookVerifier = (options = {}) => { 'The body is not a list of Event Grid events.', ); } - const events = []; - let replayed = 0; + const entries = []; for (const record of payload) { if (!isRecord(record) || typeof record.id !== 'string') continue; - try { - await checkReplay(common, `azure:${record.id}`, REPLAY_TTL_MS); - } catch (error) { - if (error.reason !== 'replay') throw error; - replayed++; - continue; - } - events.push(eventOf(record)); - } - if (events.length === 0 && replayed > 0) { - throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + entries.push({ key: `azure:${record.id}`, events: [eventOf(record)] }); } - return { events }; + return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); }, }; }; diff --git a/src/webhooks/brevo.js b/src/webhooks/brevo.js index fedc366..bcbb2ee 100644 --- a/src/webhooks/brevo.js +++ b/src/webhooks/brevo.js @@ -13,10 +13,11 @@ const { bodyBytesOf, createEvent, dateOf, + deliveryOf, isRecord, jsonRecordOf, parseJsonOf, - resolveCommonOptions, + resolveCredentialOptions, stringOf, stringsOf, tokenCheckOf, @@ -83,12 +84,13 @@ const eventOf = (record) => { * or the value of a custom `header`. * * @param options `token`, `header` (`authorization`) and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`; + * committing does nothing, as nothing is kept. * @throws {TypeError} If an option is invalid. */ const createBrevoWebhookVerifier = (options = {}) => { const checkToken = tokenCheckOf(options, 'Brevo'); - const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + const common = resolveCredentialOptions(options); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -102,7 +104,7 @@ const createBrevoWebhookVerifier = (options = {}) => { if (events.length === 0) { throw new WebhookVerificationError('payload', 'The body is not a Brevo event.'); } - return { events }; + return deliveryOf(events); }, }; }; diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 8cf8299..09230f1 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -14,32 +14,69 @@ export interface WebhookRequest { } /** - * Why a request was refused. + * Why a request was refused: `replay`, a request taken before; `pending`, one + * another delivery of it is handling now. */ export type WebhookVerificationReason = | 'signature' | 'timestamp' | 'replay' + | 'pending' | 'payload' | 'certificate' | 'topic' | 'credentials'; /** - * A request a verifier refused. Answer `replay` with a 2xx: the provider - * delivered it before. + * The options of a {@link WebhookVerificationError}. + */ +export interface WebhookVerificationErrorOptions extends ErrorOptions { + /** The HTTP status to answer with, when it is not that of the reason. */ + readonly status?: number; +} + +/** + * A request a verifier refused. Answer the provider with `status`: 200 for + * a `replay`, which arrived before; 503 for a `pending` request, and for a + * signing certificate that could not be read, so that the provider tries + * again; 400 for a `payload` that cannot be read; 401 otherwise. */ export declare class WebhookVerificationError extends Error { readonly reason: WebhookVerificationReason; - constructor(reason: WebhookVerificationReason, message: string, options?: ErrorOptions); + readonly status: number; + constructor( + reason: WebhookVerificationReason, + message: string, + options?: WebhookVerificationErrorOptions, + ); } /** - * Where the ids of the requests already taken are kept, for `ttlMs`. + * What a replay store answers when an id is reserved: `reserved` for this + * request, `pending` while another request holds it, `done` once one was + * committed. + */ +export type ReplayState = 'reserved' | 'pending' | 'done'; + +/** + * Where the ids of the requests taken are kept. An id is reserved while its + * request is handled, for `leaseMs`, and kept for `ttlMs` once the + * application commits it, so that a request whose handler failed is taken + * again on the provider's next delivery. A store shared by several + * processes, on Redis or a database, makes `reserve` one atomic operation + * (`SET NX PX`, an insert under a unique index). */ export interface ReplayStore { - /** Keeps an id: true when it is new, false when it was seen before. */ - add(id: string, ttlMs: number): boolean | Promise; + /** + * Reserves an id for `leaseMs`, unless it is held: `reserved` when it is + * new, its lease ran out or it was released; `pending` while another + * request holds it; `done` once one was committed and until its time ends. + */ + reserve(id: string, leaseMs: number): ReplayState | Promise; + /** Keeps a reserved id as taken for `ttlMs`. */ + commit(id: string, ttlMs: number): void | Promise; + /** Frees a reserved id that is not committed; a committed one stays. */ + release(id: string): void | Promise; } /** @@ -47,7 +84,8 @@ export interface ReplayStore { */ export interface MemoryReplayStoreOptions { /** - * The most ids kept, the oldest dropped first. + * The most ids kept, the oldest committed ones dropped first; an id still + * reserved is never dropped before its lease ends. * @default 10000 */ readonly maxEntries?: number; @@ -56,7 +94,9 @@ export interface MemoryReplayStoreOptions { } /** - * Makes a store of the ids of the requests already taken, in this process. + * Makes a store of the ids of the requests taken, in this process: for an + * application of one process. Several processes, or a function that makes + * its verifier for each request, share a store of their own. * * @throws {TypeError} If an option is invalid. */ @@ -71,9 +111,18 @@ export interface WebhookVerifierOptions { /** How far from now a request may be signed, in milliseconds. */ readonly toleranceMs?: number; /** - * The ids already taken; an in-process store by default, `null` for none. + * The ids taken: {@link createMemoryReplayStore} in one process, a store + * the processes share for several, or `null` for an application that + * deduplicates the events by their `id` itself. */ - readonly replayStore?: ReplayStore | null; + readonly replayStore: ReplayStore | null; + /** + * How long an id stays reserved while its request is handled, in + * milliseconds: longer than a handler takes, shorter than the provider's + * first retry. + * @default 30000 + */ + readonly leaseMs?: number; /** * The longest body taken, in bytes. * @default 5242880 @@ -83,10 +132,22 @@ export interface WebhookVerifierOptions { /** * What a verifier reads from a request: its events, one per message and - * recipient. + * recipient, whose ids stay reserved until the application commits them, + * once handled, or releases them, when handling them failed, so that the + * provider's next delivery is taken again. A lease that runs out releases + * them too. */ export interface WebhookResult { readonly events: readonly DeliveryEvent[]; + /** + * Keeps the ids of the events handled as taken, so that the provider's + * next delivery of them is a replay: of every event, or of those given. + * An id several events share, such as the SNS message of several + * recipients, is kept once all of them are committed. + */ + commit(events?: Iterable>): Promise; + /** Frees the ids of the events not committed. */ + release(): Promise; } /** @@ -218,7 +279,8 @@ export declare function createMailgunWebhookVerifier( /** * The options of a verifier of a webhook whose URL carries Basic credentials, - * as its provider signs nothing (Postmark, Mailjet, SparkPost). + * as its provider signs nothing (Postmark, Mailjet, SparkPost, SMTP2GO). No + * id is kept: committing its delivery does nothing. */ export interface BasicWebhookVerifierOptions { readonly username: string; @@ -249,8 +311,13 @@ export declare function createPostmarkWebhookVerifier( export interface SignedBodyWebhookVerifierOptions { /** The signing secret of the webhook. */ readonly secret: string; - /** The ids already taken; an in-process store by default, `null` for none. */ - readonly replayStore?: ReplayStore | null; + /** The ids taken, as {@link WebhookVerifierOptions.replayStore}. */ + readonly replayStore: ReplayStore | null; + /** + * How long an id stays reserved while its request is handled. + * @default 30000 + */ + readonly leaseMs?: number; /** * The longest body taken, in bytes. * @default 5242880 @@ -428,8 +495,13 @@ export interface AzureWebhookVerifierOptions { * @default "authorization" */ readonly header?: string; - /** The ids already taken; an in-process store by default, `null` for none. */ - readonly replayStore?: ReplayStore | null; + /** The ids taken, as {@link WebhookVerifierOptions.replayStore}. */ + readonly replayStore: ReplayStore | null; + /** + * How long an id stays reserved while its request is handled. + * @default 30000 + */ + readonly leaseMs?: number; /** * The longest body taken, in bytes. * @default 5242880 diff --git a/src/webhooks/lettermint.js b/src/webhooks/lettermint.js index 033231c..b4c9ec7 100644 --- a/src/webhooks/lettermint.js +++ b/src/webhooks/lettermint.js @@ -2,25 +2,26 @@ // their time and raw body (https://lettermint.co/docs/platform/webhooks/signing). // sently and unemail have no Lettermint webhook. -const { createHmac } = require('node:crypto'); const { + DEFAULT_TOLERANCE_MS, WebhookVerificationError, bodyBytesOf, - checkReplay, - checkTime, + checkHexSignature, + checkSignedSeconds, + claimDelivery, createEvent, dateOf, - equalBytes, headerOf, + hmacOf, isRecord, parseJsonOf, + requireSecret, resolveCommonOptions, stringOf, stringRecordOf, stringsOf, } = require('./request.js'); -const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; const TYPES = new Map([ ['message.created', 'accepted'], ['message.sent', 'accepted'], @@ -44,11 +45,8 @@ const checkSignature = (request, bytes, secret, common) => { if (match === null) { throw new WebhookVerificationError('signature', 'The request has no Lettermint signature.'); } - const expected = createHmac('sha256', secret).update(`${match[1]}.`).update(bytes).digest(); - if (!equalBytes(expected, Buffer.from(match[2], 'hex'))) { - throw new WebhookVerificationError('signature', 'The Lettermint signature does not match.'); - } - checkTime(Number(match[1]) * 1000, common); + checkHexSignature(hmacOf(secret, [`${match[1]}.`, bytes]), match[2], 'Lettermint'); + checkSignedSeconds(match[1], common, 'Lettermint'); }; const bounceOf = (event, data) => { @@ -86,16 +84,16 @@ const eventOf = (payload) => { * and body in `X-Lettermint-Signature` (`t=…,v1=…`), with the signing secret * as it is given, within five minutes, its delivery id taken once. * - * @param options `secret`; `now`, `toleranceMs`, `replayStore` and + * @param options `secret`; `replayStore`; `now`, `toleranceMs`, `leaseMs` and * `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createLettermintWebhookVerifier = (options = {}) => { - if (typeof options.secret !== 'string' || options.secret === '') { - throw new TypeError('secret is the signing secret of the Lettermint webhook.'); - } - const secret = options.secret; + const secret = requireSecret( + options.secret, + 'secret is the signing secret of the Lettermint webhook.', + ); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { async verify(request) { @@ -105,8 +103,9 @@ const createLettermintWebhookVerifier = (options = {}) => { if (!isRecord(payload) || typeof payload.id !== 'string') { throw new WebhookVerificationError('payload', 'The body is not a Lettermint event.'); } - await checkReplay(common, `lettermint:${payload.id}`); - return { events: [eventOf(payload)] }; + return claimDelivery(common, [ + { key: `lettermint:${payload.id}`, events: [eventOf(payload)] }, + ]); }, }; }; diff --git a/src/webhooks/maileroo.js b/src/webhooks/maileroo.js index 9b1a91f..2359803 100644 --- a/src/webhooks/maileroo.js +++ b/src/webhooks/maileroo.js @@ -3,21 +3,23 @@ // sently and unemail have no Maileroo webhook. const { + DAY_REPLAY_TTL_MS, WebhookVerificationError, bodyBytesOf, checkBodyHmac, - checkReplay, + claimDelivery, createEvent, dateOf, isRecord, parseJsonOf, + requireSecret, resolveCommonOptions, splitNamedTags, + stringListOf, stringOf, } = require('./request.js'); // Maileroo signs no time: an event id is kept a day and an hour. -const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; const TYPES = new Map([ ['accepted', 'accepted'], ['delivered', 'delivered'], @@ -29,16 +31,6 @@ const TYPES = new Map([ ['clicked', 'clicked'], ]); -const recipientsOf = (to) => { - if (typeof to === 'string') return [to]; - const recipients = []; - if (!Array.isArray(to)) return recipients; - for (const address of to) { - if (typeof address === 'string') recipients.push(address); - } - return recipients; -}; - const eventsOf = (payload) => { const data = isRecord(payload.event_data) ? payload.event_data : {}; const type = TYPES.get(payload.event_type) ?? 'unknown'; @@ -49,7 +41,7 @@ const eventsOf = (payload) => { if (reason !== undefined) bounce.diagnostic = reason; } const { tags, metadata } = splitNamedTags(payload.tags); - const recipients = recipientsOf(data.to); + const recipients = stringListOf(data.to); const list = recipients.length > 0 ? recipients : [undefined]; const events = []; for (let index = 0; index < list.length; index++) { @@ -79,16 +71,16 @@ const eventsOf = (payload) => { * hexadecimal in `x-maileroo-signature`. Maileroo signs no time, so an event * is taken once by its id, kept a day and an hour. * - * @param options `secret`; `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @param options `secret`; `replayStore`; `leaseMs` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If the secret is missing. */ const createMailerooWebhookVerifier = (options = {}) => { - if (typeof options.secret !== 'string' || options.secret === '') { - throw new TypeError('secret is the signing secret of the Maileroo webhook.'); - } - const secret = options.secret; - const common = resolveCommonOptions(options, REPLAY_TTL_MS); + const secret = requireSecret( + options.secret, + 'secret is the signing secret of the Maileroo webhook.', + ); + const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -97,8 +89,8 @@ const createMailerooWebhookVerifier = (options = {}) => { if (!isRecord(payload) || typeof payload.event_id !== 'string') { throw new WebhookVerificationError('payload', 'The body is not a Maileroo event.'); } - await checkReplay(common, `maileroo:${payload.event_id}`, REPLAY_TTL_MS); - return { events: eventsOf(payload) }; + const entries = [{ key: `maileroo:${payload.event_id}`, events: eventsOf(payload) }]; + return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); }, }; }; diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js index 80a0480..141e578 100644 --- a/src/webhooks/mailersend.js +++ b/src/webhooks/mailersend.js @@ -3,15 +3,17 @@ // sently and unemail have no MailerSend webhook. const { + DAY_REPLAY_TTL_MS, WebhookVerificationError, bodyBytesOf, checkBodyHmac, - checkReplay, + claimDelivery, createEvent, dateOf, isRecord, metadataFromHeaders, parseJsonOf, + requireSecret, resolveCommonOptions, stringOf, stringsOf, @@ -19,7 +21,6 @@ const { // MailerSend signs no time: an event id is kept a day and an hour, past the // retries of a delivery. -const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; const TYPES = new Map([ ['activity.sent', 'accepted'], ['activity.delivered', 'delivered'], @@ -79,16 +80,16 @@ const eventOf = (payload) => { * MailerSend signs no time, so an event is taken once by its id, kept a day * and an hour. * - * @param options `secret`; `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @param options `secret`; `replayStore`; `leaseMs` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createMailerSendWebhookVerifier = (options = {}) => { - if (typeof options.secret !== 'string' || options.secret === '') { - throw new TypeError('secret is the signing secret of the MailerSend webhook.'); - } - const secret = options.secret; - const common = resolveCommonOptions(options, REPLAY_TTL_MS); + const secret = requireSecret( + options.secret, + 'secret is the signing secret of the MailerSend webhook.', + ); + const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -98,8 +99,8 @@ const createMailerSendWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not a MailerSend event.'); } const event = eventOf(payload); - await checkReplay(common, `mailersend:${event.id}`, REPLAY_TTL_MS); - return { events: [event] }; + const entries = [{ key: `mailersend:${event.id}`, events: [event] }]; + return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); }, }; }; diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js index bd44417..e98ef1a 100644 --- a/src/webhooks/mailgun.js +++ b/src/webhooks/mailgun.js @@ -8,24 +8,25 @@ // `unsubscribed`; a temporary failure is a deferred soft bounce; and the // message id is that of the receipt, in angle brackets. -const { createHmac } = require('node:crypto'); const { + DEFAULT_TOLERANCE_MS, WebhookVerificationError, bodyBytesOf, - checkReplay, - checkTime, + checkHexSignature, + checkSignedSeconds, + claimDelivery, createEvent, dateOf, - equalBytes, + hmacOf, isRecord, parseJsonOf, + requireSecret, resolveCommonOptions, stringOf, stringRecordOf, stringsOf, } = require('./request.js'); -const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; const TYPES = new Map([ ['accepted', 'accepted'], ['delivered', 'delivered'], @@ -35,7 +36,6 @@ const TYPES = new Map([ ['opened', 'opened'], ['clicked', 'clicked'], ]); -const HEX = /^[0-9a-f]+$/i; // What a failure is: a suppression or a rejection Mailgun made before any // attempt, a temporary failure it retries, or a bounce. @@ -78,15 +78,16 @@ const eventOf = (data, token) => { * Makes the verifier of Mailgun's webhooks. * * @param options `signingKey`, the HTTP webhook signing key of the domain; - * `now`, `toleranceMs` (five minutes), `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * `replayStore`; `now`, `toleranceMs` (five minutes), `leaseMs` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createMailgunWebhookVerifier = (options = {}) => { - if (typeof options.signingKey !== 'string' || options.signingKey === '') { - throw new TypeError('signingKey is the HTTP webhook signing key of Mailgun.'); - } - const signingKey = options.signingKey; + const signingKey = requireSecret( + options.signingKey, + 'signingKey is the HTTP webhook signing key of Mailgun.', + ); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { async verify(request) { @@ -96,28 +97,20 @@ const createMailgunWebhookVerifier = (options = {}) => { const signature = isRecord(payload) ? payload.signature : undefined; const timestamp = stringOf(signature?.timestamp); const token = stringOf(signature?.token); - const given = stringOf(signature?.signature); - if ( - timestamp === undefined || - token === undefined || - given === undefined || - !HEX.test(given) - ) { + if (timestamp === undefined || token === undefined) { throw new WebhookVerificationError('signature', 'The request has no Mailgun signature.'); } - const expected = createHmac('sha256', signingKey) - .update(timestamp + token) - .digest(); - if (!equalBytes(expected, Buffer.from(given, 'hex'))) { - throw new WebhookVerificationError('signature', 'The Mailgun signature does not match.'); - } - checkTime(Number(timestamp) * 1000, common); - await checkReplay(common, `mailgun:${token}`); + checkHexSignature( + hmacOf(signingKey, [timestamp + token]), + stringOf(signature?.signature), + 'Mailgun', + ); + checkSignedSeconds(timestamp, common, 'Mailgun'); const data = payload['event-data']; if (!isRecord(data)) { throw new WebhookVerificationError('payload', 'The body has no Mailgun event.'); } - return { events: [eventOf(data, token)] }; + return claimDelivery(common, [{ key: `mailgun:${token}`, events: [eventOf(data, token)] }]); }, }; }; diff --git a/src/webhooks/mailjet.js b/src/webhooks/mailjet.js index 70ae72c..8f7c2a2 100644 --- a/src/webhooks/mailjet.js +++ b/src/webhooks/mailjet.js @@ -9,10 +9,11 @@ const { checkBasicCredentials, createEvent, dateOf, + deliveryOf, isRecord, jsonRecordOf, parseJsonOf, - resolveCommonOptions, + resolveCredentialOptions, stringOf, } = require('./request.js'); @@ -62,12 +63,13 @@ const eventOf = (record) => { * tag, the event payload the metadata, as `./mailjet` sends them. * * @param options `username`, `password` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`; + * committing does nothing, as nothing is kept. * @throws {TypeError} If a credential is missing. */ const createMailjetWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'Mailjet'); - const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + const common = resolveCredentialOptions(options); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -81,7 +83,7 @@ const createMailjetWebhookVerifier = (options = {}) => { if (events.length === 0) { throw new WebhookVerificationError('payload', 'The body is not a Mailjet event.'); } - return { events }; + return deliveryOf(events); }, }; }; diff --git a/src/webhooks/mailtrap.js b/src/webhooks/mailtrap.js index d46857c..5aa215e 100644 --- a/src/webhooks/mailtrap.js +++ b/src/webhooks/mailtrap.js @@ -3,14 +3,16 @@ // sently and unemail have no Mailtrap webhook. const { + DAY_REPLAY_TTL_MS, WebhookVerificationError, bodyBytesOf, checkBodyHmac, - checkReplay, + claimDelivery, createEvent, dateOf, isRecord, parseJsonOf, + requireSecret, resolveCommonOptions, stringOf, stringRecordOf, @@ -18,7 +20,6 @@ const { // Mailtrap signs no time and retries for up to a day: an event id is kept a // day and an hour. -const REPLAY_TTL_MS = 25 * 60 * 60 * 1000; const TYPES = new Map([ ['delivery', 'delivered'], ['soft bounce', 'bounced'], @@ -98,38 +99,27 @@ const eventOf = (record) => { * the webhook; a body of JSON or JSON Lines. Mailtrap signs no time, so each * event is taken once by its `event_id`, kept a day and an hour. * - * @param options `secret`; `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @param options `secret`; `replayStore`; `leaseMs` and `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createMailtrapWebhookVerifier = (options = {}) => { - if (typeof options.secret !== 'string' || options.secret === '') { - throw new TypeError('secret is the signing secret of the Mailtrap webhook.'); - } - const secret = options.secret; - const common = resolveCommonOptions(options, REPLAY_TTL_MS); + const secret = requireSecret( + options.secret, + 'secret is the signing secret of the Mailtrap webhook.', + ); + const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { async verify(request) { const bytes = bodyBytesOf(request, common); checkBodyHmac(request, bytes, 'mailtrap-signature', secret, 'Mailtrap'); - const events = []; - let replayed = 0; + const entries = []; for (const record of recordsOf(bytes)) { if (!isRecord(record)) continue; const event = eventOf(record); - try { - await checkReplay(common, `mailtrap:${event.id}`, REPLAY_TTL_MS); - } catch (error) { - if (error.reason !== 'replay') throw error; - replayed++; - continue; - } - events.push(event); - } - if (events.length === 0 && replayed > 0) { - throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + entries.push({ key: `mailtrap:${event.id}`, events: [event] }); } - return { events }; + return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); }, }; }; diff --git a/src/webhooks/postmark.js b/src/webhooks/postmark.js index bd864fe..5986f1d 100644 --- a/src/webhooks/postmark.js +++ b/src/webhooks/postmark.js @@ -15,9 +15,10 @@ const { checkBasicCredentials, createEvent, dateOf, + deliveryOf, isRecord, parseJsonOf, - resolveCommonOptions, + resolveCredentialOptions, stringOf, stringRecordOf, } = require('./request.js'); @@ -107,13 +108,14 @@ const eventOf = (record) => { * * @param options `username` and `password` of the webhook's URL, and * `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`; + * committing does nothing, as nothing is kept. * @throws {TypeError} If an option is invalid. */ const createPostmarkWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'Postmark'); // No time or id to hold a request to: the credentials alone stand for it. - const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + const common = resolveCredentialOptions(options); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -122,7 +124,7 @@ const createPostmarkWebhookVerifier = (options = {}) => { if (!isRecord(record) || typeof record.RecordType !== 'string') { throw new WebhookVerificationError('payload', 'The body is not a Postmark event.'); } - return { events: [eventOf(record)] }; + return deliveryOf([eventOf(record)]); }, }; }; diff --git a/src/webhooks/request.js b/src/webhooks/request.js index bb404d9..cdb3dbd 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -1,82 +1,300 @@ // Not ported: what the webhook verifiers share — the raw body, the header // fields of any kind of request, the error of a refusal, the window of time a -// signed request is taken in, the store of the ids already seen, and the -// shape of an event. sently parsed a payload apart from its check; here -// nothing is read before it is verified. +// signed request is taken in, the store of the ids already taken, the +// delivery of the events of a request, and the shape of an event. sently +// parsed a payload apart from its check; here nothing is read before it is +// verified, and an id is kept as taken only once the application has handled +// its events. const { createHmac, timingSafeEqual } = require('node:crypto'); -const { checkOptionalFunction, checkPositiveNumber } = require('../internal/options.js'); +const { + checkOptionalFunction, + checkPositiveInteger, + checkPositiveNumber, +} = require('../internal/options.js'); const DEFAULT_MAX_BODY_BYTES = 5 * 1024 * 1024; +// Longer than an application takes to handle a request, shorter than the +// first retry of a provider (20 s for SNS), so that a request whose handler +// died is taken again. +const DEFAULT_LEASE_MS = 30 * 1000; +/** + * How far from now a signed request is taken unless a verifier says + * otherwise. + * + * @internal + */ +const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; +/** + * How long the id of an event is kept when its provider signs no time and + * retries for a day. + * + * @internal + */ +const DAY_REPLAY_TTL_MS = 25 * 60 * 60 * 1000; const UTF8 = new TextDecoder('utf-8', { fatal: true }); +const HEX = /^[0-9a-f]+$/i; +const REPLAY_METHODS = ['reserve', 'commit', 'release']; + +// The answer a refusal asks for: a replay arrived before, so the provider +// may stop; a request still being handled may pass later, so it should try +// again; a body that cannot be read is the sender's; the rest is refused. +const STATUSES = new Map([ + ['replay', 200], + ['pending', 503], + ['payload', 400], +]); /** - * A request a verifier refused: `reason` says why. + * A request a verifier refused: `reason` says why, and `status` what to + * answer the provider with. */ class WebhookVerificationError extends Error { /** - * @param reason `signature`, `timestamp`, `replay`, `payload`, + * @param reason `signature`, `timestamp`, `replay`, `pending`, `payload`, * `certificate`, `topic` or `credentials`. * @param message What failed. - * @param options The cause. + * @param options The cause, and `status` when it is not that of the + * reason (200 for `replay`, 503 for `pending`, 400 for `payload`, 401 + * otherwise). */ constructor(reason, message, options) { super(message, options); this.name = 'WebhookVerificationError'; this.reason = reason; + this.status = options?.status ?? STATUSES.get(reason) ?? 401; } } /** - * Makes a store of the ids of the requests already taken, in this process, - * which holds at most `maxEntries`, the oldest dropped first. + * Makes a store of the ids of the requests taken, in this process: an id is + * `pending` for the lease of the request that reserved it, and `done` for + * its time once that request is committed. It holds at most `maxEntries` + * ids, the oldest done ones dropped first; a pending id is never dropped + * before its lease ends. * * @param options `maxEntries` (10000) and `now`. - * @returns `{ add(id, ttlMs) }`, true when the id is new. + * @returns `{ reserve(id, leaseMs), commit(id, ttlMs), release(id) }`. + * @throws {TypeError} If an option is invalid. */ const createMemoryReplayStore = (options = {}) => { - const maxEntries = options.maxEntries ?? 10000; - if (!Number.isInteger(maxEntries) || maxEntries < 1) { - throw new TypeError(`maxEntries must be a positive integer: ${String(maxEntries)}`); - } + const maxEntries = checkPositiveInteger(options.maxEntries ?? 10000, 'maxEntries'); const now = checkOptionalFunction(options.now, 'now') ?? Date.now; - const seen = new Map(); + const entries = new Map(); + + // A Map iterates in the order of insertion, and an entry is written anew + // when it changes, so the first ones are the oldest. + const trim = (at) => { + for (const [id, entry] of entries) { + const live = entry.expiresAt > at; + if (live && entries.size <= maxEntries) break; + if (live && !entry.done) continue; + entries.delete(id); + } + }; + + const write = (id, entry, at) => { + entries.delete(id); + entries.set(id, entry); + trim(at); + }; + return { - add(id, ttlMs) { + reserve(id, leaseMs) { const at = now(); - const expiresAt = seen.get(id); - if (expiresAt !== undefined && expiresAt > at) return false; - seen.delete(id); - seen.set(id, at + ttlMs); - for (const [key, until] of seen) { - if (seen.size <= maxEntries && until > at) break; - seen.delete(key); - } - return true; + const entry = entries.get(id); + if (entry !== undefined && entry.expiresAt > at) return entry.done ? 'done' : 'pending'; + write(id, { done: false, expiresAt: at + leaseMs }, at); + return 'reserved'; + }, + commit(id, ttlMs) { + const at = now(); + write(id, { done: true, expiresAt: at + ttlMs }, at); + }, + release(id) { + if (entries.get(id)?.done === false) entries.delete(id); }, }; }; +const checkReplayStore = (store) => { + if (store === undefined) { + throw new TypeError( + 'replayStore is required: createMemoryReplayStore() for one process, a store the ' + + 'processes share for several, or null for an application that deduplicates the ' + + 'events by their id itself.', + ); + } + if (store === null) return null; + for (const method of REPLAY_METHODS) { + if (typeof store?.[method] !== 'function') { + throw new TypeError(`replayStore must have a ${method} method, or be null.`); + } + } + return store; +}; + /** * Resolves the options every verifier takes. * * @internal */ -const resolveCommonOptions = (options, defaultToleranceMs) => { - const replayStore = - options.replayStore === undefined ? createMemoryReplayStore() : options.replayStore; - if (replayStore !== null && typeof replayStore?.add !== 'function') { - throw new TypeError('replayStore must have an add method, or be null.'); +const resolveCommonOptions = (options, defaultToleranceMs) => ({ + now: checkOptionalFunction(options.now, 'now') ?? Date.now, + toleranceMs: checkPositiveNumber(options.toleranceMs ?? defaultToleranceMs, 'toleranceMs'), + replayStore: checkReplayStore(options.replayStore), + leaseMs: checkPositiveNumber(options.leaseMs ?? DEFAULT_LEASE_MS, 'leaseMs'), + maxBodyBytes: checkPositiveNumber(options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES, 'maxBodyBytes'), +}); + +/** + * Resolves the options of a verifier of a provider that signs neither a time + * nor an id: its credentials stand for a request, and nothing is kept of it. + * + * @internal + */ +const resolveCredentialOptions = (options) => + resolveCommonOptions({ ...options, replayStore: null }, DEFAULT_TOLERANCE_MS); + +/** + * The events of a verified request, whose ids stay reserved in the replay + * store until the application commits them, or releases them so that the + * provider's next delivery is taken again. A lease that runs out releases + * them too. + */ +class WebhookDelivery { + events = []; + #store; + #ttlMs; + // The reserved ids, each with the number of its events not committed. + #left = new Map(); + #keyOf = new WeakMap(); + + constructor(fields, store, ttlMs) { + for (const name in fields) { + if (Object.hasOwn(fields, name)) this[name] = fields[name]; + } + this.#store = store; + this.#ttlMs = ttlMs; } - return { - now: checkOptionalFunction(options.now, 'now') ?? Date.now, - toleranceMs: checkPositiveNumber(options.toleranceMs ?? defaultToleranceMs, 'toleranceMs'), - replayStore, - maxBodyBytes: checkPositiveNumber( - options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES, - 'maxBodyBytes', - ), - }; + + add(key, events) { + for (const event of events) this.events.push(event); + if (this.#store === null) return; + this.#left.set(key, events.length); + for (const event of events) this.#keyOf.set(event, key); + } + + get reserved() { + return this.#left.size; + } + + /** + * Keeps the ids of the events handled as taken, so that the provider's + * next delivery of them is a replay: of every event of the request, or of + * those given. An id that several events share is kept once all of them + * are committed. + * + * @param events The events handled; all of them when left out. + */ + async commit(events) { + if (this.#store === null) return; + if (events === undefined) { + for (const key of this.#left.keys()) { + await this.#store.commit(key, this.#ttlMs); + this.#left.delete(key); + } + return; + } + for (const event of events) { + const key = this.#keyOf.get(event); + const left = key === undefined ? undefined : this.#left.get(key); + if (left === undefined) continue; + this.#keyOf.delete(event); + if (left > 1) { + this.#left.set(key, left - 1); + continue; + } + await this.#store.commit(key, this.#ttlMs); + this.#left.delete(key); + } + } + + /** + * Frees the ids of the events not committed, so that the provider's next + * delivery of them is taken: call it when handling the events failed. + */ + async release() { + if (this.#store === null) return; + for (const key of this.#left.keys()) { + await this.#store.release(key); + this.#left.delete(key); + } + } +} + +/** + * Reserves the ids of the records of a verified request and makes the + * delivery of the events of those that are new. A record whose id was taken + * before, or is being taken by another request, is left out; a request with + * records and nothing new is refused, as `pending` while another request + * holds one of them and as `replay` otherwise. A failure releases what was + * reserved. + * + * @param common The options of the verifier. + * @param entries `{ key, events }` for each record, `key` its id in the + * store. + * @param fields The other fields of the result, such as `confirmation`. + * @param ttlMs How long a committed id is kept: twice the window of time by + * default, so that a request cannot come back in it. + * @returns The delivery. + * @internal + */ +const claimDelivery = async (common, entries, fields = {}, ttlMs = common.toleranceMs * 2) => { + const store = common.replayStore; + const delivery = new WebhookDelivery(fields, store, ttlMs); + let done = 0; + let pending = 0; + try { + for (const entry of entries) { + const state = store === null ? 'reserved' : await store.reserve(entry.key, common.leaseMs); + if (state === 'reserved') delivery.add(entry.key, entry.events); + else if (state === 'pending') pending++; + else if (state === 'done') done++; + else throw new TypeError(`replayStore.reserve answered ${String(state)}.`); + } + } catch (error) { + await delivery.release(); + throw error; + } + if (store === null || delivery.reserved > 0 || done + pending === 0) return delivery; + const single = entries.length === 1; + if (pending > 0) { + throw new WebhookVerificationError( + 'pending', + single + ? `The request ${entries[0].key} is being handled by another delivery.` + : 'The events of the batch are being handled by another delivery.', + ); + } + throw new WebhookVerificationError( + 'replay', + single + ? `The request ${entries[0].key} was taken before.` + : 'Every event of the batch was taken before.', + ); +}; + +/** + * The delivery of the events of a request whose provider gives no id to + * take it by: nothing is kept, and committing it does nothing. + * + * @internal + */ +const deliveryOf = (events, fields) => { + const delivery = new WebhookDelivery(fields, null, 0); + delivery.add(undefined, events); + return delivery; }; /** @@ -148,16 +366,16 @@ const checkTime = (atMs, common) => { }; /** - * Refuses a request whose id was taken before. + * Refuses a time a provider signs in whole seconds that is not one, or that + * is too far from now. * * @internal */ -const checkReplay = async (common, id, ttlMs = common.toleranceMs * 2) => { - if (common.replayStore === null) return; - // Kept for twice the window by default, so a request cannot come back in it. - if ((await common.replayStore.add(id, ttlMs)) !== true) { - throw new WebhookVerificationError('replay', `The request ${id} was taken before.`); +const checkSignedSeconds = (value, common, label) => { + if (typeof value !== 'string' || !/^\d+$/.test(value)) { + throw new WebhookVerificationError('timestamp', `The ${label} timestamp is not a number.`); } + checkTime(Number(value) * 1000, common); }; /** @@ -167,6 +385,42 @@ const checkReplay = async (common, id, ttlMs = common.toleranceMs * 2) => { */ const equalBytes = (a, b) => a.length === b.length && timingSafeEqual(a, b); +/** + * The HMAC-SHA256 of the parts given, in order. + * + * @internal + */ +const hmacOf = (secret, parts) => { + const hmac = createHmac('sha256', secret); + for (const part of parts) hmac.update(part); + return hmac.digest(); +}; + +/** + * Checks a signature given in hexadecimal against the digest expected. + * + * @internal + */ +const checkHexSignature = (expected, given, label) => { + const hex = given?.trim(); + if (hex === undefined || !HEX.test(hex)) { + throw new WebhookVerificationError('signature', `The request has no ${label} signature.`); + } + if (!equalBytes(expected, Buffer.from(hex, 'hex'))) { + throw new WebhookVerificationError('signature', `The ${label} signature does not match.`); + } +}; + +/** + * A secret an option must hold: a string that is not empty. + * + * @internal + */ +const requireSecret = (value, message) => { + if (typeof value !== 'string' || value === '') throw new TypeError(message); + return value; +}; + /** * A date from seconds, milliseconds or an ISO string, or `undefined`. * @@ -213,23 +467,13 @@ const isRecord = (value) => typeof value === 'object' && value !== null && !Arra */ const stringOf = (value) => (typeof value === 'string' && value !== '' ? value : undefined); -const HEX = /^[0-9a-f]+$/i; - /** * Checks the HMAC-SHA256 of the raw body, in hexadecimal in a header field. * * @internal */ -const checkBodyHmac = (request, bytes, header, secret, label) => { - const given = headerOf(request, header); - if (given === undefined || !HEX.test(given.trim())) { - throw new WebhookVerificationError('signature', `The request has no ${label} signature.`); - } - const expected = createHmac('sha256', secret).update(bytes).digest(); - if (!equalBytes(expected, Buffer.from(given.trim(), 'hex'))) { - throw new WebhookVerificationError('signature', `The ${label} signature does not match.`); - } -}; +const checkBodyHmac = (request, bytes, header, secret, label) => + checkHexSignature(hmacOf(secret, [bytes]), headerOf(request, header), label); /** * The credentials a webhook URL carries (`https://user:password@…`), for a @@ -295,6 +539,26 @@ const stringsOf = (list) => { return strings; }; +/** + * The strings of a value that is one string or a list of them. + * + * @internal + */ +const stringListOf = (value) => + typeof value === 'string' && value !== '' ? [value] : stringsOf(value); + +const SES_BOUNCE_KINDS = new Map([ + ['Permanent', 'hard'], + ['Transient', 'soft'], +]); + +/** + * How a bounce of one of the types of SES went, which Resend passes on too. + * + * @internal + */ +const bounceKindOf = (type) => SES_BOUNCE_KINDS.get(type) ?? 'undetermined'; + /** * A record of JSON text, such as metadata a provider keeps as one string. * @@ -383,24 +647,35 @@ const splitNamedTags = (named) => { }; module.exports = { + DAY_REPLAY_TTL_MS, + DEFAULT_TOLERANCE_MS, + WebhookDelivery, WebhookVerificationError, basicCredentialsOf, bodyBytesOf, + bounceKindOf, checkBasicCredentials, checkBodyHmac, - checkReplay, + checkHexSignature, + checkSignedSeconds, checkTime, + claimDelivery, createEvent, createMemoryReplayStore, dateOf, + deliveryOf, equalBytes, headerOf, + hmacOf, isRecord, jsonRecordOf, metadataFromHeaders, parseJsonOf, + requireSecret, resolveCommonOptions, + resolveCredentialOptions, splitNamedTags, + stringListOf, stringOf, stringRecordOf, stringsOf, diff --git a/src/webhooks/resend.js b/src/webhooks/resend.js index 5776356..5813aea 100644 --- a/src/webhooks/resend.js +++ b/src/webhooks/resend.js @@ -10,24 +10,26 @@ // or soft; and a failed or suppressed email is `rejected`, not `bounced` or // `unknown`. -const { createHmac } = require('node:crypto'); const { + DEFAULT_TOLERANCE_MS, WebhookVerificationError, bodyBytesOf, - checkReplay, - checkTime, + bounceKindOf, + checkSignedSeconds, + claimDelivery, createEvent, dateOf, equalBytes, headerOf, + hmacOf, isRecord, parseJsonOf, resolveCommonOptions, splitNamedTags, + stringListOf, stringOf, } = require('./request.js'); -const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; const TYPES = new Map([ ['email.sent', 'accepted'], @@ -41,10 +43,6 @@ const TYPES = new Map([ ['email.opened', 'opened'], ['email.clicked', 'clicked'], ]); -const BOUNCE_KINDS = new Map([ - ['Permanent', 'hard'], - ['Transient', 'soft'], -]); /** * The key of a Svix secret, `whsec_` and base64. @@ -74,10 +72,7 @@ const checkSvixSignature = (request, bytes, key, common) => { if (id === undefined || timestamp === undefined || header === undefined) { throw new WebhookVerificationError('signature', 'The request has no Svix signature.'); } - const expected = createHmac('sha256', key) - .update(`${id}.${timestamp}.`, 'utf8') - .update(bytes) - .digest(); + const expected = hmacOf(key, [`${id}.${timestamp}.`, bytes]); let matched = false; for (const part of header.split(' ')) { if (!part.startsWith('v1,')) continue; @@ -86,37 +81,24 @@ const checkSvixSignature = (request, bytes, key, common) => { if (!matched) { throw new WebhookVerificationError('signature', 'The Svix signature does not match.'); } - if (!/^\d+$/.test(timestamp)) { - throw new WebhookVerificationError('timestamp', 'The Svix timestamp is not a number.'); - } - checkTime(Number(timestamp) * 1000, common); + checkSignedSeconds(timestamp, common, 'Svix'); return id; }; -const recipientsOf = (to) => { - if (typeof to === 'string') return [to]; - const recipients = []; - if (!Array.isArray(to)) return recipients; - for (const address of to) { - if (typeof address === 'string') recipients.push(address); - } - return recipients; -}; - const eventsOf = (payload, id) => { const data = isRecord(payload.data) ? payload.data : {}; const type = TYPES.get(payload.type) ?? 'unknown'; const { tags, metadata } = splitNamedTags(data.tags); let bounce; if (type === 'bounced' && isRecord(data.bounce)) { - bounce = { kind: BOUNCE_KINDS.get(data.bounce.type) ?? 'undetermined' }; + bounce = { kind: bounceKindOf(data.bounce.type) }; if (stringOf(data.bounce.message) !== undefined) bounce.diagnostic = data.bounce.message; } const detail = data.bounce ?? data.failed ?? data.suppressed ?? data.delivery_delayed; const reason = isRecord(detail) ? (stringOf(detail.subType) ?? stringOf(detail.reason) ?? stringOf(detail.message)) : undefined; - const recipients = recipientsOf(data.to); + const recipients = stringListOf(data.to); const list = recipients.length > 0 ? recipients : [undefined]; const events = []; for (let index = 0; index < list.length; index++) { @@ -144,8 +126,9 @@ const eventsOf = (payload, id) => { * Makes the verifier of the webhooks of Resend, which Svix signs. * * @param options `secret`, the signing secret of the webhook (`whsec_…`); - * `now`, `toleranceMs` (five minutes), `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * `replayStore`; `now`, `toleranceMs` (five minutes), `leaseMs` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createResendWebhookVerifier = (options = {}) => { @@ -155,12 +138,11 @@ const createResendWebhookVerifier = (options = {}) => { async verify(request) { const bytes = bodyBytesOf(request, common); const id = checkSvixSignature(request, bytes, key, common); - await checkReplay(common, `resend:${id}`); const payload = parseJsonOf(bytes); if (!isRecord(payload)) { throw new WebhookVerificationError('payload', 'The body is not a Resend event.'); } - return { events: eventsOf(payload, id) }; + return claimDelivery(common, [{ key: `resend:${id}`, events: eventsOf(payload, id) }]); }, }; }; diff --git a/src/webhooks/scaleway.js b/src/webhooks/scaleway.js index ae3a8e4..8f9a98e 100644 --- a/src/webhooks/scaleway.js +++ b/src/webhooks/scaleway.js @@ -66,13 +66,15 @@ const eventOf = (record, snsMessageId) => { * issued, published within an hour, taken once. * * @param options `topicArns`; `trustChain`, the PEM of Scaleway's SNS trust - * chain; `fetch`, `now`, `toleranceMs`, `replayStore` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events, confirmation? }`. + * chain; `replayStore`; `fetch`, `now`, `toleranceMs`, `leaseMs` and + * `maxBodyBytes`. + * @returns `{ verify(request) }`, which gives `{ events, confirmation?, + * commit, release }`. * @throws {TypeError} If an option is invalid. */ const createScalewayWebhookVerifier = (options = {}) => { const authorities = trustChainOf(options.trustChain); - const verifySns = createSnsVerifier(options, { + const verify = createSnsVerifier(options, { provider: 'scaleway', defaultToleranceMs: DEFAULT_TOLERANCE_MS, isCertificateUrl: (value) => isHttpsUrlOf(value, SCALEWAY_HOST), @@ -82,23 +84,18 @@ const createScalewayWebhookVerifier = (options = {}) => { } throw certificateError('The signing certificate was not issued by the trust chain.'); }, + eventsOf: (message, record) => [ + record === undefined + ? createEvent({ + provider: 'scaleway', + type: 'unknown', + id: message.MessageId, + raw: message, + }) + : eventOf(record, message.MessageId), + ], }); - return { - async verify(request) { - const { message, record, confirmation } = await verifySns(request); - if (confirmation !== undefined) return { events: [], confirmation }; - if (record === undefined) { - const event = createEvent({ - provider: 'scaleway', - type: 'unknown', - id: message.MessageId, - raw: message, - }); - return { events: [event] }; - } - return { events: [eventOf(record, message.MessageId)] }; - }, - }; + return { verify }; }; module.exports = { createScalewayWebhookVerifier }; diff --git a/src/webhooks/sendgrid.js b/src/webhooks/sendgrid.js index 430e449..6fffe00 100644 --- a/src/webhooks/sendgrid.js +++ b/src/webhooks/sendgrid.js @@ -10,20 +10,21 @@ const { createPublicKey, verify: verifySignature } = require('node:crypto'); const { + DEFAULT_TOLERANCE_MS, WebhookVerificationError, bodyBytesOf, - checkReplay, - checkTime, + checkSignedSeconds, + claimDelivery, createEvent, dateOf, headerOf, isRecord, parseJsonOf, resolveCommonOptions, + stringListOf, stringOf, } = require('./request.js'); -const DEFAULT_TOLERANCE_MS = 5 * 60 * 1000; const SIGNATURE = 'x-twilio-email-event-webhook-signature'; const TIMESTAMP = 'x-twilio-email-event-webhook-timestamp'; const TYPES = new Map([ @@ -93,16 +94,6 @@ const publicKeyOf = (value) => { } }; -const tagsOf = (category) => { - if (typeof category === 'string') return [category]; - const tags = []; - if (!Array.isArray(category)) return tags; - for (const value of category) { - if (typeof value === 'string') tags.push(value); - } - return tags; -}; - const metadataOf = (record) => { const metadata = Object.create(null); for (const name in record) { @@ -141,7 +132,7 @@ const eventOf = (record, index, timestamp) => { stringOf(record.reason) ?? stringOf(record.response), url: stringOf(record.url), - tags: tagsOf(record.category), + tags: stringListOf(record.category), metadata: metadataOf(record), raw: record, }); @@ -151,9 +142,9 @@ const eventOf = (record, index, timestamp) => { * Makes the verifier of SendGrid's Signed Event Webhook. * * @param options `publicKey`, its verification key (base64 of the key, or - * PEM); `now`, `toleranceMs` (five minutes), `replayStore` and + * PEM); `replayStore`; `now`, `toleranceMs` (five minutes), `leaseMs` and * `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`. * @throws {TypeError} If an option is invalid. */ const createSendGridWebhookVerifier = (options = {}) => { @@ -171,34 +162,20 @@ const createSendGridWebhookVerifier = (options = {}) => { if (!verifySignature('sha256', signed, key, Buffer.from(signature, 'base64'))) { throw new WebhookVerificationError('signature', 'The SendGrid signature does not match.'); } - if (!/^\d+$/.test(timestamp)) { - throw new WebhookVerificationError('timestamp', 'The SendGrid timestamp is not a number.'); - } - checkTime(Number(timestamp) * 1000, common); + checkSignedSeconds(timestamp, common, 'SendGrid'); const payload = parseJsonOf(bytes); if (!Array.isArray(payload)) { throw new WebhookVerificationError('payload', 'The body is not a list of SendGrid events.'); } // A batch SendGrid sends again keeps the ids of its events: those taken // before are left out, and a batch of nothing new is a replay. - const events = []; - let replayed = 0; + const entries = []; for (let index = 0; index < payload.length; index++) { if (!isRecord(payload[index])) continue; const event = eventOf(payload[index], index, timestamp); - try { - await checkReplay(common, `sendgrid:${event.id}`); - } catch (error) { - if (error.reason !== 'replay') throw error; - replayed++; - continue; - } - events.push(event); - } - if (events.length === 0 && replayed > 0) { - throw new WebhookVerificationError('replay', 'Every event of the batch was taken before.'); + entries.push({ key: `sendgrid:${event.id}`, events: [event] }); } - return { events }; + return claimDelivery(common, entries); }, }; }; diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index 4c50833..892e71e 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -10,17 +10,13 @@ // hard or soft; and a reject or a rendering failure is `rejected`, not // `bounced`. -const { createEvent, dateOf, isRecord, stringOf } = require('./request.js'); +const { bounceKindOf, createEvent, dateOf, isRecord, stringOf } = require('./request.js'); const { createSnsVerifier, isHttpsUrlOf } = require('./sns.js'); // SNS retries a delivery with the time it was published, for up to an hour // by default. const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; const SNS_HOST = /^sns\.[a-z0-9-]+\.amazonaws\.com(\.cn)?$/; -const BOUNCE_KINDS = new Map([ - ['Permanent', 'hard'], - ['Transient', 'soft'], -]); // The tags SES writes itself, and the one `./ses` writes for the priority. const OWN_TAG = /^(ses|aws):/; @@ -70,7 +66,7 @@ const onlyDestinationOf = (mail) => { }; const bounceOf = (bounce) => { - const kind = BOUNCE_KINDS.get(bounce.bounceType) ?? 'undetermined'; + const kind = bounceKindOf(bounce.bounceType); return recipientsOf(bounce.bouncedRecipients, (item) => { const recipient = stringOf(item?.emailAddress); if (recipient === undefined) return undefined; @@ -188,41 +184,38 @@ const eventsOf = (record, snsMessageId) => { /** * Makes the verifier of the SNS messages of Amazon SES. * - * @param options `topicArns`, the topics taken; `fetch`, for the signing - * certificates; `now`, `toleranceMs` (an hour), `replayStore` and + * @param options `topicArns`, the topics taken; `replayStore`; `fetch`, for + * the signing certificates; `now`, `toleranceMs` (an hour), `leaseMs` and * `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events, confirmation? }`. + * @returns `{ verify(request) }`, which gives `{ events, confirmation?, + * commit, release }`. * @throws {TypeError} If an option is invalid. */ -const createSesWebhookVerifier = (options = {}) => { - const verifySns = createSnsVerifier(options, { +const createSesWebhookVerifier = (options = {}) => ({ + /** + * Verifies an SNS message and reads its events. + * + * @param request `body`, the raw body, and `headers`. + * @returns The delivery of its events, and `confirmation` for a + * subscription message. + * @throws {WebhookVerificationError} If the message is refused. + */ + verify: createSnsVerifier(options, { provider: 'ses', defaultToleranceMs: DEFAULT_TOLERANCE_MS, isCertificateUrl: isSnsUrl, - }); - return { - /** - * Verifies an SNS message and reads its events. - * - * @param request `body`, the raw body, and `headers`. - * @returns `{ events }`, and `confirmation` for a subscription message. - * @throws {WebhookVerificationError} If the message is refused. - */ - async verify(request) { - const { message, record, confirmation } = await verifySns(request); - if (confirmation !== undefined) return { events: [], confirmation }; - if (record === undefined) { - const event = createEvent({ - provider: 'ses', - type: 'unknown', - id: `${message.MessageId}:0`, - raw: message, - }); - return { events: [event] }; - } - return { events: eventsOf(record, message.MessageId) }; - }, - }; -}; + eventsOf: (message, record) => + record === undefined + ? [ + createEvent({ + provider: 'ses', + type: 'unknown', + id: `${message.MessageId}:0`, + raw: message, + }), + ] + : eventsOf(record, message.MessageId), + }), +}); module.exports = { createSesWebhookVerifier }; diff --git a/src/webhooks/smtp2go.js b/src/webhooks/smtp2go.js index 4f56d92..35597ab 100644 --- a/src/webhooks/smtp2go.js +++ b/src/webhooks/smtp2go.js @@ -9,9 +9,10 @@ const { checkBasicCredentials, createEvent, dateOf, + deliveryOf, isRecord, metadataFromHeaders, - resolveCommonOptions, + resolveCredentialOptions, stringOf, } = require('./request.js'); @@ -81,12 +82,13 @@ const eventOf = (record) => { * `X-Metadata-` fields when the webhook adds them to its payload. * * @param options `username`, `password` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`; + * committing does nothing, as nothing is kept. * @throws {TypeError} If a credential is missing. */ const createSmtp2goWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'SMTP2GO'); - const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + const common = resolveCredentialOptions(options); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -95,7 +97,7 @@ const createSmtp2goWebhookVerifier = (options = {}) => { if (!isRecord(record) || typeof record.event !== 'string') { throw new WebhookVerificationError('payload', 'The body is not an SMTP2GO event.'); } - return { events: [eventOf(record)] }; + return deliveryOf([eventOf(record)]); }, }; }; diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index eb59d9f..c2be0f4 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -10,8 +10,8 @@ const { checkOptionalFunction } = require('../internal/options.js'); const { WebhookVerificationError, bodyBytesOf, - checkReplay, checkTime, + claimDelivery, isRecord, parseJsonOf, resolveCommonOptions, @@ -74,8 +74,12 @@ const stringToSign = (message, fields) => { return text; }; -const certificateError = (message, cause) => - new WebhookVerificationError('certificate', message, cause === undefined ? undefined : { cause }); +const certificateError = (message, cause, status) => + new WebhookVerificationError('certificate', message, { cause, status }); + +// A certificate that could not be read may be read on the provider's next +// delivery, so the answer asks for one. +const UNREAD = 503; const loadCertificate = async (fetchOption, url) => { const fetch = fetchOption ?? globalThis.fetch; @@ -83,10 +87,14 @@ const loadCertificate = async (fetchOption, url) => { try { response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(10_000) }); } catch (error) { - throw certificateError('The signing certificate could not be read.', error); + throw certificateError('The signing certificate could not be read.', error, UNREAD); } if (!response.ok) { - throw certificateError(`The signing certificate answered HTTP ${response.status}.`); + throw certificateError( + `The signing certificate answered HTTP ${response.status}.`, + undefined, + UNREAD, + ); } try { return new X509Certificate(await readText(response, MAX_CERTIFICATE_BYTES)); @@ -156,14 +164,16 @@ const topicsOf = (topicArns) => { }; /** - * Makes the verification of SNS messages: `verify(request)` gives `{ message, - * record }` for a notification, `record` its parsed message when it is a JSON - * object, or `{ message, confirmation }` for a subscription message. + * Makes the verification of SNS messages: `verify(request)` gives the + * delivery of the events of a notification, or of the `confirmation` of a + * subscription message, under the id of the SNS message. * * @param options `topicArns`, `fetch`, and the options every verifier takes. * @param service `provider`, `defaultToleranceMs`, `isCertificateUrl` (the - * hosts of certificates and subscriptions), and `trust(certificate)`, which - * throws for a certificate the service did not issue. + * hosts of certificates and subscriptions), `trust(certificate)`, which + * throws for a certificate the service did not issue, and `eventsOf(message, + * record)`, the events of a notification, `record` its parsed message when it + * is a JSON object. * @internal */ const createSnsVerifier = (options, service) => { @@ -183,7 +193,7 @@ const createSnsVerifier = (options, service) => { } await checkSignature(message, certificateOf, service.isCertificateUrl, common); checkTime(Date.parse(message.Timestamp), common); - await checkReplay(common, `${service.provider}:${message.MessageId}`); + const key = `${service.provider}:${message.MessageId}`; if (message.Type !== 'Notification') { if (!service.isCertificateUrl(message.SubscribeURL)) { throw new WebhookVerificationError( @@ -191,8 +201,7 @@ const createSnsVerifier = (options, service) => { 'The subscription URL is not of the service.', ); } - return { - message, + return claimDelivery(common, [{ key, events: [] }], { confirmation: { type: message.Type, topicArn: message.TopicArn, @@ -200,7 +209,7 @@ const createSnsVerifier = (options, service) => { token: message.Token, message: message.Message, }, - }; + }); } let record; try { @@ -208,7 +217,8 @@ const createSnsVerifier = (options, service) => { } catch { record = undefined; } - return { message, record: isRecord(record) ? record : undefined }; + const events = service.eventsOf(message, isRecord(record) ? record : undefined); + return claimDelivery(common, [{ key, events }]); }; }; diff --git a/src/webhooks/sparkpost.js b/src/webhooks/sparkpost.js index 94a93ab..b7a99a2 100644 --- a/src/webhooks/sparkpost.js +++ b/src/webhooks/sparkpost.js @@ -9,9 +9,10 @@ const { checkBasicCredentials, createEvent, dateOf, + deliveryOf, isRecord, parseJsonOf, - resolveCommonOptions, + resolveCredentialOptions, stringOf, stringRecordOf, stringsOf, @@ -89,12 +90,13 @@ const eventOf = (inner, index) => { * recipient. * * @param options `username`, `password` and `maxBodyBytes`. - * @returns `{ verify(request) }`, which gives `{ events }`. + * @returns `{ verify(request) }`, which gives `{ events, commit, release }`; + * committing does nothing, as nothing is kept. * @throws {TypeError} If a credential is missing. */ const createSparkPostWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'SparkPost'); - const common = resolveCommonOptions({ ...options, replayStore: null }, 5 * 60 * 1000); + const common = resolveCredentialOptions(options); return { async verify(request) { const bytes = bodyBytesOf(request, common); @@ -111,7 +113,7 @@ const createSparkPostWebhookVerifier = (options = {}) => { const inner = isRecord(payload[index]) ? innerOf(payload[index]) : undefined; if (inner !== undefined) events.push(eventOf(inner, index)); } - return { events }; + return deliveryOf(events); }, }; }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index 154258c..5336773 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -18,6 +18,7 @@ import { createSesWebhookVerifier, createSmtp2goWebhookVerifier, createSparkPostWebhookVerifier, + type ReplayState, type ReplayStore, type SesWebhookResult, type WebhookVerificationReason, @@ -41,10 +42,28 @@ expectError(ses.verify({ body: {} })); expectError(createSesWebhookVerifier({})); const error = new WebhookVerificationError('replay', 'seen'); expectType(error.reason); +expectType(error.status); +new WebhookVerificationError('pending', 'held', { status: 503, cause: error }); +expectType>(result.commit()); +expectType>(result.commit([result.events[0]])); +expectType>(result.release()); +expectType>(store.reserve('id', 30_000)); +const shared: ReplayStore = { + reserve: async () => 'reserved' as const, + commit: async () => {}, + release: async () => {}, +}; +createSesWebhookVerifier({ topicArns: ['arn'], replayStore: shared, leaseMs: 10_000 }); +expectError(createSesWebhookVerifier({ topicArns: ['arn'] })); +expectError({ add: () => true }); -const resend = createResendWebhookVerifier({ secret: 'whsec_abc=', toleranceMs: 1000 }); +const resend = createResendWebhookVerifier({ + secret: 'whsec_abc=', + toleranceMs: 1000, + replayStore: store, +}); expectAssignable[]>((await resend.verify({ body: '' })).events); -const sendgrid = createSendGridWebhookVerifier({ publicKey: 'MFkw…' }); +const sendgrid = createSendGridWebhookVerifier({ publicKey: 'MFkw…', replayStore: store }); expectAssignable[]>( (await sendgrid.verify({ body: '' })).events, ); @@ -60,7 +79,8 @@ const mailersend = createMailerSendWebhookVerifier({ secret: 's', replayStore: n expectAssignable[]>( (await mailersend.verify({ body: '' })).events, ); -createMailtrapWebhookVerifier({ secret: 's', maxBodyBytes: 1 }); +createMailtrapWebhookVerifier({ secret: 's', maxBodyBytes: 1, replayStore: store }); +expectError(createMailtrapWebhookVerifier({ secret: 's' })); createBrevoWebhookVerifier({ token: 't', header: 'x-webhook-token' }); createMailjetWebhookVerifier({ username: 'u', password: 'p' }); const sparkpost = createSparkPostWebhookVerifier({ username: 'u', password: 'p' }); @@ -68,17 +88,22 @@ expectAssignable[]>( (await sparkpost.verify({ body: '' })).events, ); expectError(createBrevoWebhookVerifier({})); -expectError(createMailtrapWebhookVerifier({ secret: 's', toleranceMs: 1 })); +expectError(createMailtrapWebhookVerifier({ secret: 's', replayStore: null, toleranceMs: 1 })); const scaleway = createScalewayWebhookVerifier({ topicArns: ['arn:scw:sns:fr-par:p:t'], trustChain: '-----BEGIN CERTIFICATE-----', + replayStore: store, }); expectType((await scaleway.verify({ body: '' })).confirmation?.token); -createLettermintWebhookVerifier({ secret: 'whsec_x', toleranceMs: 60_000 }); -createMailerooWebhookVerifier({ secret: 's' }); +createLettermintWebhookVerifier({ secret: 'whsec_x', toleranceMs: 60_000, replayStore: null }); +createMailerooWebhookVerifier({ secret: 's', replayStore: store }); createSmtp2goWebhookVerifier({ username: 'u', password: 'p' }); -const azure = createAzureWebhookVerifier({ token: 't', header: 'x-webhook-token' }); +const azure = createAzureWebhookVerifier({ + token: 't', + header: 'x-webhook-token', + replayStore: store, +}); expectType((await azure.verify({ body: '' })).validation?.validationCode); expectError(createScalewayWebhookVerifier({ topicArns: ['arn'] })); expectError(createAzureWebhookVerifier({})); diff --git a/tests/unit/webhooks-mailgun.test.js b/tests/unit/webhooks-mailgun.test.js index 695d326..e4c4a36 100644 --- a/tests/unit/webhooks-mailgun.test.js +++ b/tests/unit/webhooks-mailgun.test.js @@ -5,6 +5,7 @@ const assert = require('node:assert'); const { createHmac } = require('node:crypto'); const { createMailgunWebhookVerifier, + createMemoryReplayStore, createPostmarkWebhookVerifier, WebhookVerificationError, } = require('../../src/webhooks/index.js'); @@ -12,7 +13,12 @@ const { const KEY = 'key-mailgun-signing'; const NOW = 1_800_000_000_000; const mailgun = (options = {}) => - createMailgunWebhookVerifier({ signingKey: KEY, now: () => NOW, ...options }); + createMailgunWebhookVerifier({ + signingKey: KEY, + now: () => NOW, + replayStore: createMemoryReplayStore({ now: () => NOW }), + ...options, + }); const signedMailgun = (data, token = 'token-1', timestamp = '1800000000') => ({ body: JSON.stringify({ signature: { @@ -137,7 +143,7 @@ test('Webhooks Mailgun: a forged, old or repeated request is refused', async () ); // Changed in the port: sently took any time and any replay. await rejects(verifier.verify(signedMailgun(eventData(), 'old', '1799990000')), 'timestamp'); - await verifier.verify(good); + await (await verifier.verify(good)).commit(); await rejects(verifier.verify(good), 'replay'); await rejects(verifier.verify(signedMailgun('nothing', 'no-data')), 'payload'); await rejects(verifier.verify({ body: '"text"' }), 'signature'); diff --git a/tests/unit/webhooks-more.test.js b/tests/unit/webhooks-more.test.js index 4d3472e..7868dd9 100644 --- a/tests/unit/webhooks-more.test.js +++ b/tests/unit/webhooks-more.test.js @@ -5,6 +5,7 @@ const assert = require('node:assert'); const { createHmac } = require('node:crypto'); const { createBrevoWebhookVerifier, + createMemoryReplayStore, createMailerSendWebhookVerifier, createMailjetWebhookVerifier, createMailtrapWebhookVerifier, @@ -24,7 +25,10 @@ const basic = (user, password) => `Basic ${Buffer.from(`${user}:${password}`).to const plain = (event) => ({ ...event, metadata: { ...event.metadata } }); test('Webhooks MailerSend: the HMAC of the body, an event taken once', async () => { - const verifier = createMailerSendWebhookVerifier({ secret: 'ms-secret' }); + const verifier = createMailerSendWebhookVerifier({ + secret: 'ms-secret', + replayStore: createMemoryReplayStore(), + }); const payload = { type: 'activity.hard_bounced', created_at: '2027-01-15T08:00:00.000000Z', @@ -42,10 +46,11 @@ test('Webhooks MailerSend: the HMAC of the body, an event taken once', async () }, }; const body = JSON.stringify(payload); - const { events } = await verifier.verify({ + const delivery = await verifier.verify({ body, headers: { Signature: hmac('ms-secret', body) }, }); + const { events } = delivery; assert.deepStrictEqual(plain(events[0]), { provider: 'mailersend', type: 'bounced', @@ -59,6 +64,7 @@ test('Webhooks MailerSend: the HMAC of the body, an event taken once', async () metadata: { orderId: '42' }, raw: payload, }); + await delivery.commit(); await rejects( verifier.verify({ body, headers: { Signature: hmac('ms-secret', body) } }), 'replay', @@ -123,7 +129,10 @@ test('Webhooks MailerSend: the HMAC of the body, an event taken once', async () }); test('Webhooks Mailtrap: a signed batch, in JSON or JSON Lines, each event once', async () => { - const verifier = createMailtrapWebhookVerifier({ secret: 'mt-secret' }); + const verifier = createMailtrapWebhookVerifier({ + secret: 'mt-secret', + replayStore: createMemoryReplayStore(), + }); const delivery = { event: 'delivery', message_id: 'mt-1', @@ -143,10 +152,11 @@ test('Webhooks Mailtrap: a signed batch, in JSON or JSON Lines, each event once' bounce_category: 'invalid', }; const body = JSON.stringify({ events: [delivery, bounce] }); - const { events } = await verifier.verify({ + const taken = await verifier.verify({ body, headers: { 'Mailtrap-Signature': hmac('mt-secret', body) }, }); + const { events } = taken; assert.deepStrictEqual(plain(events[0]), { provider: 'mailtrap', type: 'delivered', @@ -164,6 +174,7 @@ test('Webhooks Mailtrap: a signed batch, in JSON or JSON Lines, each event once' diagnostic: '550 unknown', }); assert.strictEqual(events[1].reason, 'invalid'); + await taken.commit(); await rejects( verifier.verify({ body, headers: { 'Mailtrap-Signature': hmac('mt-secret', body) } }), 'replay', diff --git a/tests/unit/webhooks-others.test.js b/tests/unit/webhooks-others.test.js index 6ede4f4..90be1dd 100644 --- a/tests/unit/webhooks-others.test.js +++ b/tests/unit/webhooks-others.test.js @@ -7,6 +7,7 @@ const { createAzureWebhookVerifier, createLettermintWebhookVerifier, createMailerooWebhookVerifier, + createMemoryReplayStore, createScalewayWebhookVerifier, createSmtp2goWebhookVerifier, WebhookVerificationError, @@ -47,6 +48,7 @@ test('Webhooks Scaleway: an SNS message signed by a certificate the trust chain trustChain: scaleway.trustChain, fetch: served.fetch, now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: createMemoryReplayStore(), }); const record = { id: 'ev-1', @@ -118,6 +120,7 @@ test('Webhooks Scaleway: a certificate the trust chain did not issue, or off Sca trustChain: scaleway.trustChain, fetch: certificateFetch(certificate).fetch, now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: null, }); await rejects( foreign.verify({ body: JSON.stringify(scwMessage({ type: 'email_spam' })) }), @@ -144,7 +147,11 @@ test('Webhooks Scaleway: a certificate the trust chain did not issue, or off Sca test('Webhooks Lettermint: the HMAC of the time and body, within a window, once', async () => { const secret = 'whsec_lettermint'; - const verifier = createLettermintWebhookVerifier({ secret, now: () => 1_800_000_000_000 }); + const verifier = createLettermintWebhookVerifier({ + secret, + now: () => 1_800_000_000_000, + replayStore: createMemoryReplayStore(), + }); const sign = (body, t = '1800000000') => ({ body, headers: { @@ -165,7 +172,8 @@ test('Webhooks Lettermint: the HMAC of the time and body, within a window, once' }, }; const body = JSON.stringify(payload); - const { events } = await verifier.verify(sign(body)); + const taken = await verifier.verify(sign(body)); + const { events } = taken; assert.deepStrictEqual(plain(events[0]), { provider: 'lettermint', type: 'bounced', @@ -178,6 +186,7 @@ test('Webhooks Lettermint: the HMAC of the time and body, within a window, once' metadata: { orderId: '42' }, raw: payload, }); + await taken.commit(); await rejects(verifier.verify(sign(body)), 'replay'); await rejects(verifier.verify({ body, headers: {} }), 'signature'); await rejects(verifier.verify({ ...sign(body), body: body.replace('bob', 'eve') }), 'signature'); @@ -220,7 +229,10 @@ test('Webhooks Lettermint: the HMAC of the time and body, within a window, once' }); test('Webhooks Maileroo: the HMAC of the body, an event per recipient, once', async () => { - const verifier = createMailerooWebhookVerifier({ secret: 'mr-secret' }); + const verifier = createMailerooWebhookVerifier({ + secret: 'mr-secret', + replayStore: createMemoryReplayStore(), + }); const sign = (body) => ({ body, headers: { @@ -241,7 +253,8 @@ test('Webhooks Maileroo: the HMAC of the body, an event per recipient, once', as }, }; const body = JSON.stringify(payload); - const { events } = await verifier.verify(sign(body)); + const taken = await verifier.verify(sign(body)); + const { events } = taken; assert.strictEqual(events.length, 2); assert.deepStrictEqual(plain(events[1]), { provider: 'maileroo', @@ -255,6 +268,7 @@ test('Webhooks Maileroo: the HMAC of the body, an event per recipient, once', as metadata: { orderId: '42' }, raw: payload, }); + await taken.commit(); await rejects(verifier.verify(sign(body)), 'replay'); const cases = [ ['accepted', 'accepted'], @@ -391,7 +405,10 @@ test('Webhooks SMTP2GO: Basic credentials, a body of JSON or of a form', async ( }); test('Webhooks Azure: Event Grid events with the token, and the validation handed back', async () => { - const verifier = createAzureWebhookVerifier({ token: 'eg-token' }); + const verifier = createAzureWebhookVerifier({ + token: 'eg-token', + replayStore: createMemoryReplayStore(), + }); const validation = [ { id: 'v-1', @@ -406,10 +423,13 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande body: JSON.stringify(validation), headers: { 'aeg-event-type': 'SubscriptionValidation' }, }); - assert.deepStrictEqual(validated, { - events: [], - validation: { validationCode: 'code-1', validationUrl: validation[0].data.validationUrl }, - }); + assert.deepStrictEqual( + { events: validated.events, validation: validated.validation }, + { + events: [], + validation: { validationCode: 'code-1', validationUrl: validation[0].data.validationUrl }, + }, + ); await rejects( verifier.verify({ body: '[{"eventType":"Other"}]', @@ -443,7 +463,8 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande }; const body = JSON.stringify([report, click, { eventType: 'no id' }]); const headers = { 'aeg-event-type': 'Notification', Authorization: 'Bearer eg-token' }; - const { events } = await verifier.verify({ body, headers }); + const taken = await verifier.verify({ body, headers }); + const { events } = taken; assert.deepStrictEqual(plain(events[0]), { provider: 'azure', type: 'bounced', @@ -459,6 +480,7 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande }); assert.strictEqual(events[1].type, 'clicked'); assert.strictEqual(events[1].url, 'https://e.com/a'); + await taken.commit(); await rejects(verifier.verify({ body, headers }), 'replay'); await rejects( verifier.verify({ body, headers: { 'aeg-event-type': 'Notification' } }), @@ -485,7 +507,11 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande ['delivered', 'bounced', 'bounced', 'rejected', 'deferred', 'unknown', 'opened', 'unknown'], ); await rejects(verifier.verify({ body: '{}', headers }), 'payload'); - const custom = createAzureWebhookVerifier({ token: 't', header: 'x-webhook-token' }); + const custom = createAzureWebhookVerifier({ + token: 't', + header: 'x-webhook-token', + replayStore: null, + }); const ok = await custom.verify({ body: '[]', headers: { 'x-webhook-token': 't' } }); assert.deepStrictEqual(ok.events, []); assert.throws(() => createAzureWebhookVerifier({}), /token is required/); diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 7d3a615..84c89a6 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -36,19 +36,260 @@ test('Webhooks: dates come from seconds, milliseconds or ISO text', () => { assert.strictEqual(dateOf(Number.NaN), undefined); }); -test('Webhooks: the memory replay store forgets an id after its time, the oldest first', () => { +test('Webhooks: the memory replay store holds an id while it is handled, then keeps it', () => { let time = 0; const store = createMemoryReplayStore({ maxEntries: 2, now: () => time }); - assert.strictEqual(store.add('a', 100), true); - assert.strictEqual(store.add('a', 100), false); - time = 100; - assert.strictEqual(store.add('a', 100), true); - assert.strictEqual(store.add('b', 1000), true); - assert.strictEqual(store.add('c', 1000), true); - // Three ids for two places: a, the oldest, goes. - assert.strictEqual(store.add('a', 100), true); - assert.strictEqual(store.add('c', 1000), false); + assert.strictEqual(store.reserve('a', 10), 'reserved'); + assert.strictEqual(store.reserve('a', 10), 'pending'); + // A lease that runs out frees the id: the handler of the request died. + time = 10; + assert.strictEqual(store.reserve('a', 10), 'reserved'); + store.release('a'); + assert.strictEqual(store.reserve('a', 10), 'reserved'); + store.commit('a', 100); + assert.strictEqual(store.reserve('a', 10), 'done'); + // A done id is not released, and is forgotten after its time. + store.release('a'); + assert.strictEqual(store.reserve('a', 10), 'done'); + time = 110; + assert.strictEqual(store.reserve('a', 10), 'reserved'); + store.commit('a', 1000); assert.throws(() => createMemoryReplayStore({ maxEntries: 0 }), /maxEntries/); assert.throws(() => createMemoryReplayStore({ now: 1 }), /now must be a function/); - assert.strictEqual(createMemoryReplayStore().add('x', 1), true); + assert.strictEqual(createMemoryReplayStore().reserve('x', 1), 'reserved'); +}); + +test('Webhooks: the memory replay store drops the oldest done ids, never a pending one', () => { + let time = 0; + const store = createMemoryReplayStore({ maxEntries: 2, now: () => time }); + assert.strictEqual(store.reserve('held', 1000), 'reserved'); + store.reserve('b', 1000); + store.commit('b', 1000); + store.reserve('c', 1000); + store.commit('c', 1000); + // Three ids for two places: b, the oldest done one, goes; held stays. + assert.strictEqual(store.reserve('held', 1000), 'pending'); + assert.strictEqual(store.reserve('c', 1000), 'done'); + assert.strictEqual(store.reserve('b', 1000), 'reserved'); + time = 1000; + // Expired ids go first. + assert.strictEqual(store.reserve('held', 1000), 'reserved'); +}); + +const { createHmac } = require('node:crypto'); +const { + WebhookVerificationError, + createAzureWebhookVerifier, + createBrevoWebhookVerifier, + createLettermintWebhookVerifier, + createMailerooWebhookVerifier, + createMailerSendWebhookVerifier, + createMailgunWebhookVerifier, + createMailtrapWebhookVerifier, + createPostmarkWebhookVerifier, + createResendWebhookVerifier, + createScalewayWebhookVerifier, + createSendGridWebhookVerifier, + createSesWebhookVerifier, + createSmtp2goWebhookVerifier, +} = require('../../src/webhooks/index.js'); +const { SECRET, signSvix } = require('../helpers/webhooks/svix.js'); +const sns = require('../helpers/webhooks/sns.js'); + +const NOW = 1_800_000_000_000; +const refusal = (reason, status) => (error) => { + assert.ok(error instanceof WebhookVerificationError, String(error)); + assert.strictEqual(error.reason, reason, error.message); + assert.strictEqual(error.status, status); + return true; +}; +const resendEvent = JSON.stringify({ + type: 'email.delivered', + data: { email_id: 'e-1', to: ['bob@example.com', 'carol@example.com'] }, +}); +const mailtrapBatch = (...ids) => + JSON.stringify({ events: ids.map((id) => ({ event: 'delivery', event_id: id })) }); +const signMailtrap = (body) => ({ + body, + headers: { 'mailtrap-signature': createHmac('sha256', 'mt').update(body).digest('hex') }, +}); + +test('Webhooks: an event whose handler failed is taken again, once released', async () => { + // Before: the id was kept as taken inside verify(), so a provider's + // redelivery after a failed handler was a replay, answered 2xx, and lost. + let time = NOW; + const resend = createResendWebhookVerifier({ + secret: SECRET, + now: () => time, + replayStore: createMemoryReplayStore({ now: () => time }), + }); + const request = signSvix(resendEvent, 'msg_fail'); + const failed = await resend.verify(request); + await failed.release(); + const again = await resend.verify(request); + assert.strictEqual(again.events.length, 2); + // While it is handled, a redelivery waits: 503 asks the provider to retry. + await assert.rejects(resend.verify(request), refusal('pending', 503)); + // A handler that died frees the id when its lease runs out. + time += 30 * 1000; + const retried = await resend.verify(request); + await retried.commit(); + await assert.rejects(resend.verify(request), refusal('replay', 200)); +}); + +test('Webhooks: a batch commits the events handled, the others are taken again', async () => { + const mailtrap = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: createMemoryReplayStore(), + }); + const request = signMailtrap(mailtrapBatch('ev-1', 'ev-2', 'ev-3')); + const first = await mailtrap.verify(request); + assert.strictEqual(first.events.length, 3); + await first.commit([first.events[0], first.events[2]]); + // An event committed already, or of another delivery, changes nothing. + await first.commit([first.events[0], { id: 'foreign' }]); + await first.release(); + const second = await mailtrap.verify(request); + assert.deepStrictEqual( + second.events.map((event) => event.id), + ['ev-2'], + ); + await second.commit(); + await assert.rejects(mailtrap.verify(request), refusal('replay', 200)); +}); + +test('Webhooks: an id several events share is kept once all of them are committed', async () => { + const served = sns.certificateFetch(); + const ses = createSesWebhookVerifier({ + topicArns: [sns.TOPIC], + fetch: served.fetch, + now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: createMemoryReplayStore(), + }); + const message = sns.notification({ + eventType: 'Delivery', + mail: { messageId: 'm-1', destination: ['bob@example.com', 'carol@example.com'] }, + delivery: { recipients: ['bob@example.com', 'carol@example.com'] }, + }); + const request = { body: JSON.stringify(message) }; + const first = await ses.verify(request); + assert.strictEqual(first.events.length, 2); + await first.commit([first.events[0]]); + await first.release(); + const second = await ses.verify(request); + assert.strictEqual(second.events.length, 2); + await second.commit(second.events); + await assert.rejects(ses.verify(request), refusal('replay', 200)); +}); + +test('Webhooks: a store that fails releases the ids reserved before it', async () => { + const calls = []; + const store = { + reserve: async (id) => { + calls.push(['reserve', id]); + if (id === 'mailtrap:ev-2') throw new Error('store down'); + return 'reserved'; + }, + commit: async (id) => { + calls.push(['commit', id]); + }, + release: async (id) => { + calls.push(['release', id]); + }, + }; + const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', replayStore: store }); + await assert.rejects(mailtrap.verify(signMailtrap(mailtrapBatch('ev-1', 'ev-2'))), /store down/); + assert.deepStrictEqual(calls, [ + ['reserve', 'mailtrap:ev-1'], + ['reserve', 'mailtrap:ev-2'], + ['release', 'mailtrap:ev-1'], + ]); + const odd = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: { ...store, reserve: () => true }, + }); + await assert.rejects(odd.verify(signMailtrap(mailtrapBatch('ev-9'))), /answered true/); +}); + +test('Webhooks: a batch with nothing new waits for its other delivery, or is a replay', async () => { + const mailtrap = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: createMemoryReplayStore(), + }); + const request = signMailtrap(mailtrapBatch('ev-1', 'ev-2')); + const first = await mailtrap.verify(request); + await assert.rejects(mailtrap.verify(request), refusal('pending', 503)); + await first.commit(); + await assert.rejects(mailtrap.verify(request), refusal('replay', 200)); + // A body with no record is no replay: nothing was reserved, nothing refused. + assert.deepStrictEqual((await mailtrap.verify(signMailtrap('{"events":[]}'))).events, []); +}); + +test('Webhooks: every verifier that keeps ids needs a replay store, null to keep none', () => { + // Before: an in-process store was made silently for each verifier, which a + // function that makes its verifier per request, or several processes, do + // not share. + const { publicKey } = require('node:crypto').generateKeyPairSync('ec', { + namedCurve: 'prime256v1', + }); + const makers = [ + () => createResendWebhookVerifier({ secret: SECRET }), + () => + createSendGridWebhookVerifier({ + publicKey: publicKey.export({ format: 'pem', type: 'spki' }), + }), + () => createMailgunWebhookVerifier({ signingKey: 'k' }), + () => createMailerSendWebhookVerifier({ secret: 's' }), + () => createMailtrapWebhookVerifier({ secret: 's' }), + () => createMailerooWebhookVerifier({ secret: 's' }), + () => createLettermintWebhookVerifier({ secret: 's' }), + () => createAzureWebhookVerifier({ token: 't' }), + () => createSesWebhookVerifier({ topicArns: [sns.TOPIC] }), + () => + createScalewayWebhookVerifier({ + topicArns: [sns.TOPIC], + trustChain: sns.scaleway.trustChain, + }), + ]; + for (const make of makers) assert.throws(make, /replayStore is required/); + assert.throws( + () => createMailtrapWebhookVerifier({ secret: 's', replayStore: { reserve() {} } }), + /replayStore must have a commit method/, + ); + assert.throws( + () => createMailtrapWebhookVerifier({ secret: 's', replayStore: null, leaseMs: 0 }), + /leaseMs/, + ); + // The providers that sign no id keep nothing, and need no store. + assert.ok(createPostmarkWebhookVerifier({ username: 'u', password: 'p' })); + assert.ok(createBrevoWebhookVerifier({ token: 't' })); + assert.ok(createSmtp2goWebhookVerifier({ username: 'u', password: 'p' })); +}); + +test('Webhooks: a refusal says what to answer, a certificate not read asks for a retry', async () => { + assert.strictEqual(new WebhookVerificationError('signature', 'x').status, 401); + assert.strictEqual(new WebhookVerificationError('payload', 'x').status, 400); + assert.strictEqual(new WebhookVerificationError('topic', 'x', { status: 403 }).status, 403); + const offline = createSesWebhookVerifier({ + topicArns: [sns.TOPIC], + now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: null, + fetch: async () => { + throw new Error('offline'); + }, + }); + const message = sns.notification({ eventType: 'Send', mail: { messageId: 'm' } }); + await assert.rejects( + offline.verify({ body: JSON.stringify(message) }), + refusal('certificate', 503), + ); + // A delivery of a provider that keeps nothing commits nothing. + const postmark = createPostmarkWebhookVerifier({ username: 'u', password: 'p' }); + const delivery = await postmark.verify({ + body: JSON.stringify({ RecordType: 'Delivery', MessageID: 'm', Recipient: 'bob@example.com' }), + headers: { authorization: `Basic ${Buffer.from('u:p').toString('base64')}` }, + }); + await delivery.commit(); + await delivery.release(); + assert.strictEqual(delivery.events.length, 1); }); diff --git a/tests/unit/webhooks-resend.test.js b/tests/unit/webhooks-resend.test.js index 1833c08..f4cf53f 100644 --- a/tests/unit/webhooks-resend.test.js +++ b/tests/unit/webhooks-resend.test.js @@ -3,6 +3,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { + createMemoryReplayStore, createResendWebhookVerifier, WebhookVerificationError, } = require('../../src/webhooks/index.js'); @@ -10,7 +11,12 @@ const { SECRET, signSvix } = require('../helpers/webhooks/svix.js'); const NOW = 1_800_000_000_000; const verifier = (options = {}) => - createResendWebhookVerifier({ secret: SECRET, now: () => NOW, ...options }); + createResendWebhookVerifier({ + secret: SECRET, + now: () => NOW, + replayStore: createMemoryReplayStore({ now: () => NOW }), + ...options, + }); const event = (type, data = {}) => JSON.stringify({ type, @@ -33,7 +39,11 @@ const rejects = async (promise, reason) => { }; test('Webhooks Resend: the vector of the Svix documentation verifies', async () => { - const resend = createResendWebhookVerifier({ secret: SECRET, now: () => 1_614_265_330_000 }); + const resend = createResendWebhookVerifier({ + secret: SECRET, + now: () => 1_614_265_330_000, + replayStore: null, + }); const { events } = await resend.verify({ body: '{"test": 2432232314}', headers: { @@ -127,7 +137,7 @@ test('Webhooks Resend: a forged, old or repeated request is refused', async () = ); await rejects(resend.verify(signSvix(body, 'msg_old', '1799999000')), 'timestamp'); await rejects(resend.verify(signSvix(body, 'msg_bad', 'soon')), 'timestamp'); - await resend.verify(signed); + await (await resend.verify(signed)).commit(); await rejects(resend.verify(signed), 'replay'); await rejects(resend.verify(signSvix('[1]', 'msg_array')), 'payload'); // Standard Webhooks names its headers webhook-*. @@ -145,5 +155,5 @@ test('Webhooks Resend: a secret that is not one is refused when the verifier is assert.throws(() => createResendWebhookVerifier({}), /signing secret/); assert.throws(() => createResendWebhookVerifier({ secret: 'whsec_not base64!' }), /base64/); assert.throws(() => createResendWebhookVerifier({ secret: 'whsec_' }), /base64/); - assert.ok(createResendWebhookVerifier({ secret: SECRET.slice(6) })); + assert.ok(createResendWebhookVerifier({ secret: SECRET.slice(6), replayStore: null })); }); diff --git a/tests/unit/webhooks-sendgrid.test.js b/tests/unit/webhooks-sendgrid.test.js index 2bd0540..6d55e42 100644 --- a/tests/unit/webhooks-sendgrid.test.js +++ b/tests/unit/webhooks-sendgrid.test.js @@ -4,6 +4,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { generateKeyPairSync, sign } = require('node:crypto'); const { + createMemoryReplayStore, createSendGridWebhookVerifier, WebhookVerificationError, } = require('../../src/webhooks/index.js'); @@ -24,7 +25,12 @@ const signed = (events, timestamp = '1800000000') => { }; }; const verifier = (options = {}) => - createSendGridWebhookVerifier({ publicKey: PUBLIC_KEY, now: () => NOW, ...options }); + createSendGridWebhookVerifier({ + publicKey: PUBLIC_KEY, + now: () => NOW, + replayStore: createMemoryReplayStore({ now: () => NOW }), + ...options, + }); const record = (event, fields = {}) => ({ email: 'bob@example.com', timestamp: 1_799_999_990, @@ -118,7 +124,9 @@ test('Webhooks SendGrid: a forged, old or repeated batch is refused', async () = await rejects(sendgrid.verify({ body: good.body, headers: {} }), 'signature'); await rejects(sendgrid.verify(signed([record('delivered')], '1799999000')), 'timestamp'); await rejects(sendgrid.verify(signed([record('delivered')], 'later')), 'timestamp'); - assert.strictEqual((await sendgrid.verify(good)).events.length, 2); + const first = await sendgrid.verify(good); + assert.strictEqual(first.events.length, 2); + await first.commit(); await rejects(sendgrid.verify(good), 'replay'); // A batch sent again with a new event keeps the new one alone. const more = await sendgrid.verify(signed([record('delivered'), record('click')])); @@ -129,9 +137,11 @@ test('Webhooks SendGrid: a forged, old or repeated batch is refused', async () = await rejects(sendgrid.verify(signed({ event: 'delivered' })), 'payload'); const failing = verifier({ replayStore: { - add: () => { + reserve: () => { throw new WebhookVerificationError('payload', 'store down'); }, + commit: () => {}, + release: () => {}, }, }); await rejects(failing.verify(signed([record('delivered')])), 'payload'); @@ -139,7 +149,11 @@ test('Webhooks SendGrid: a forged, old or repeated batch is refused', async () = test('Webhooks SendGrid: the key may be PEM, and one that is not a key is refused', async () => { const pem = publicKey.export({ format: 'pem', type: 'spki' }); - const fromPem = createSendGridWebhookVerifier({ publicKey: pem, now: () => NOW }); + const fromPem = createSendGridWebhookVerifier({ + publicKey: pem, + now: () => NOW, + replayStore: null, + }); assert.strictEqual((await fromPem.verify(signed([record('delivered')]))).events.length, 1); assert.throws(() => createSendGridWebhookVerifier({}), /publicKey is the verification key/); assert.throws( diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js index 5016566..b0c0eaa 100644 --- a/tests/unit/webhooks-ses.test.js +++ b/tests/unit/webhooks-ses.test.js @@ -3,6 +3,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { + createMemoryReplayStore, createSesWebhookVerifier, WebhookVerificationError, } = require('../../src/webhooks/index.js'); @@ -21,6 +22,7 @@ const verifier = (options = {}) => { topicArns: [TOPIC], fetch: served.fetch, now: () => NOW, + replayStore: createMemoryReplayStore({ now: () => NOW }), ...options, }); return { ...made, requests: served.requests }; @@ -214,16 +216,19 @@ test('Webhooks SES: a subscription is verified and handed back, not confirmed', Timestamp: '2027-01-01T12:00:00.000Z', }); const result = await verify(request(message)); - assert.deepStrictEqual(result, { - events: [], - confirmation: { - type: 'SubscriptionConfirmation', - topicArn: TOPIC, - subscribeUrl: message.SubscribeURL, - token: 'token-1', - message: 'You have chosen to subscribe', + assert.deepStrictEqual( + { events: result.events, confirmation: result.confirmation }, + { + events: [], + confirmation: { + type: 'SubscriptionConfirmation', + topicArn: TOPIC, + subscribeUrl: message.SubscribeURL, + token: 'token-1', + message: 'You have chosen to subscribe', + }, }, - }); + ); // Only the certificate was fetched: the subscription is the caller's to confirm. assert.strictEqual(requests.length, 1); const elsewhere = signSns({ @@ -261,7 +266,9 @@ test('Webhooks SES: a forged, foreign, old or repeated message is refused', asyn ), 'timestamp', ); - assert.strictEqual((await verify(request(good))).events.length, 2); + const taken = await verify(request(good)); + assert.strictEqual(taken.events.length, 2); + await taken.commit(); await rejects(verify(request(good)), 'replay'); const sha1 = signSns( { ...notification({ eventType: 'Send', mail: mail() }, { MessageId: 'v1' }) }, @@ -290,6 +297,7 @@ test('Webhooks SES: a certificate that cannot be read or is not valid now is ref const failing = createSesWebhookVerifier({ topicArns: [TOPIC], now: () => NOW, + replayStore: null, fetch: async () => { throw new Error('offline'); }, @@ -298,12 +306,14 @@ test('Webhooks SES: a certificate that cannot be read or is not valid now is ref const missing = createSesWebhookVerifier({ topicArns: [TOPIC], now: () => NOW, + replayStore: null, fetch: async () => new Response('gone', { status: 404 }), }); await rejects(missing.verify(request(good)), 'certificate'); const garbage = createSesWebhookVerifier({ topicArns: [TOPIC], now: () => NOW, + replayStore: null, fetch: async () => new Response('not a certificate'), }); await rejects(garbage.verify(request(good)), 'certificate'); @@ -314,7 +324,11 @@ test('Webhooks SES: a certificate that cannot be read or is not valid now is ref test('Webhooks SES: the options are checked, and a replay store may be given or none', async () => { assert.throws(() => createSesWebhookVerifier({}), /topicArns must list/); assert.throws(() => createSesWebhookVerifier({ topicArns: [''] }), /topic ARN is a string/); - assert.throws(() => createSesWebhookVerifier({ topicArns: [TOPIC], fetch: 'x' }), /fetch/); + assert.throws(() => createSesWebhookVerifier({ topicArns: [TOPIC] }), /replayStore is required/); + assert.throws( + () => createSesWebhookVerifier({ topicArns: [TOPIC], replayStore: null, fetch: 'x' }), + /fetch/, + ); assert.throws( () => createSesWebhookVerifier({ topicArns: [TOPIC], toleranceMs: 0 }), /toleranceMs/, @@ -325,11 +339,22 @@ test('Webhooks SES: the options are checked, and a replay store may be given or ); const seen = []; const { verify } = verifier({ - replayStore: { add: async (id, ttlMs) => seen.push([id, ttlMs]) > 0 }, + replayStore: { + reserve: async (id, leaseMs) => seen.push(['reserve', id, leaseMs]) && 'reserved', + commit: async (id, ttlMs) => { + seen.push(['commit', id, ttlMs]); + }, + release: async (id) => { + seen.push(['release', id]); + }, + }, }); const good = notification({ eventType: 'Send', mail: mail() }); - await verify(request(good)); - assert.deepStrictEqual(seen, [['ses:sns-1', 2 * 60 * 60 * 1000]]); + await (await verify(request(good))).commit(); + assert.deepStrictEqual(seen, [ + ['reserve', 'ses:sns-1', 30 * 1000], + ['commit', 'ses:sns-1', 2 * 60 * 60 * 1000], + ]); const open = verifier({ replayStore: null }); await open.verify(request(good)); await open.verify(request(good)); @@ -342,7 +367,11 @@ test('Webhooks SES: the options are checked, and a replay store may be given or return new Response(require('../helpers/webhooks/sns.js').certificate); }; try { - await createSesWebhookVerifier({ topicArns: [TOPIC], now: () => NOW }).verify({ + await createSesWebhookVerifier({ + topicArns: [TOPIC], + now: () => NOW, + replayStore: null, + }).verify({ body: Buffer.from(JSON.stringify(good)), headers: new Headers(), }); From 5eb15ec4fa2b5efd9cde5758aa22cd80be6ca09f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:07:32 +0300 Subject: [PATCH 053/143] fix(attachments): bound inlineDataUrls, refuse more ranges and a downgrade The media type of the data URI pattern of inlineDataUrls was `[^;]+`, so each ` --- CHANGELOG.md | 8 +- SECURITY.md | 17 +++- docs/messages/attachments.md | 19 ++-- src/attachments/data-uri.js | 27 ++++-- src/attachments/index.d.ts | 19 ++-- src/attachments/url.js | 55 ++++++++--- tests/unit/attachments.test.js | 167 +++++++++++++++++++++++++++++++++ 7 files changed, 272 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98f32b7..69e67a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,9 +63,11 @@ additive but the defaults listed below. `metadata` (each page says how). - **`./attachments`:** `fromPath`, `fromUrl`, `fromDataUri`, `fromStream` and `inlineDataUrls`, for what nodemailer read through `path`, `href` and data URIs. Nothing is read implicitly: - `fromUrl` goes through an injected `fetch`, checks each redirect with `allowUrl` (by default no - `localhost`, loopback, private or link-local address), and bounds time and bytes; `fromPath` - can be held to a `root` by real path. + `fromUrl` goes through an injected `fetch`, checks each redirect with `allowUrl` (by default the + host as written: no `localhost`, and no IP literal of a loopback, private, link-local, + multicast, reserved or documentation range, NAT64 and 6to4 included; a host name is not + resolved), refuses a redirect from https to http, and bounds time and bytes; `fromPath` can be + held to a `root` by real path. - **`./render`:** `renderContent(renderer, input)` for the content of `createMessage`, `createReactEmailRenderer({ render })` with the `render` of `@react-email/render` passed in, and `htmlToText`, linear in the length of its input. diff --git a/SECURITY.md b/SECURITY.md index b7b7128..1ac306b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -45,10 +45,19 @@ designed around: host); and an answer is read up to 4 MiB. - **Attachments do not read the machine by default.** No file or URL is read for an attachment but through `fromPath` and `fromUrl` of `./attachments`, which the application calls. - `fromUrl` checks every URL, redirects included, with `allowUrl`, by default refusing - `localhost` and loopback, private and link-local addresses; it sends its headers to the first - origin only and bounds time and bytes. `fromPath` with a `root` compares real paths, so a - symbolic link cannot lead out of it. + `fromUrl` checks every URL, redirects included, with `allowUrl`; it refuses a redirect from + https to http, sends its headers to the first origin only and bounds time and bytes. `fromPath` + with a `root` compares real paths, so a symbolic link cannot lead out of it. +- **The default `allowUrl` of `fromUrl` reads the host as written, and resolves nothing.** + `isPublicUrl` refuses `localhost`, `*.localhost` and an IP literal in a loopback, private, + link-local (the cloud metadata address among them), carrier-grade NAT, multicast, reserved, + documentation or benchmarking range, and in the IPv6 forms that carry an IPv4 address + (IPv4-compatible `::/96`, NAT64 `64:ff9b::/96` and `64:ff9b:1::/48`, 6to4 `2002::/16`). A host + *name* is not looked up, so a name that resolves to a private address passes, and so does one + that answers with a public address to the check and a private one to the fetch (DNS rebinding). + An application that fetches URLs its users give must pass an `allowUrl` of its own, or a + `fetch` that checks the address it connects to; the default is a guard against a mistake, not + against an attacker who controls a name. - **The sendmail command gets arguments, not a shell line.** The recipients follow `--`, none may start with `-`, and Bcc recipients go on the command line, never into the message. - **Credentials and personal data stay out of logs.** A transport shows `[REDACTED]` in place of diff --git a/docs/messages/attachments.md b/docs/messages/attachments.md index 4320b95..8204ff6 100644 --- a/docs/messages/attachments.md +++ b/docs/messages/attachments.md @@ -68,12 +68,19 @@ const message = createMessage({ - **`fromUrl(url, options)`** fetches http and https URLs once, through the `fetch` you pass or the global one, and holds the bytes. It follows at most `maxRedirects` (5), checks every URL with `allowUrl`, and sends `headers` to the origin of the - first URL only. It gives up after `timeout` (30 s) or `maxBytes` (25 MiB). By - default `allowUrl` is `isPublicUrl`, which refuses `localhost` and loopback, - private and link-local addresses (the cloud metadata endpoint among them). A host - name is fetched whatever it resolves to, so a sender that fetches URLs its users - give passes an `allowUrl` of its own or a `fetch` whose dispatcher checks the - address it connects to. + first URL only. It gives up after `timeout` (30 s) or `maxBytes` (25 MiB), and + refuses a redirect from https to http, whatever `allowUrl` says. By default + `allowUrl` is `isPublicUrl`, which looks at the host of each URL as written. It + refuses `localhost`, `*.localhost` and an IP literal in a loopback, private, + link-local (the cloud metadata address among them), carrier-grade NAT, multicast, + reserved, documentation or benchmarking range, and in the IPv6 forms that carry an + IPv4 address (IPv4-compatible `::/96`, NAT64 `64:ff9b::/96` and `64:ff9b:1::/48`, + 6to4 `2002::/16`). A host name is not resolved, so a name that resolves to a + private address passes, and so does one that answers with a public address to the + check and a private one to the fetch (DNS rebinding). An application that fetches + URLs its users give must pass an `allowUrl` of its own, or a `fetch` that checks + the address it connects to; the default guards against a mistake, not against + someone who controls a name. - **`fromDataUri(uri, options)`** decodes a base64 or percent-encoded `data:` URI (RFC 2397) up to `maxBytes`, and refuses one that does not parse. - **`fromStream(source, options)`** reads a Node.js `Readable`, a web diff --git a/src/attachments/data-uri.js b/src/attachments/data-uri.js index 5481a3b..8a3fff3 100644 --- a/src/attachments/data-uri.js +++ b/src/attachments/data-uri.js @@ -22,22 +22,28 @@ const HEX = /^[0-9A-Fa-f]{2}$/; const invalid = (uri) => new TypeError(`Invalid data URI: ${JSON.stringify(String(uri).slice(0, 64))}`); +const tooLong = (maxBytes) => new RangeError(`The data URI holds more than ${maxBytes} bytes.`); + const decodeBase64 = (uri, data, maxBytes) => { const text = data.replace(ASCII_WHITESPACE, ''); if (!BASE64.test(text) || text.length % 4 === 1) throw invalid(uri); - if (Math.floor((text.length * 3) / 4) > maxBytes) { - throw new RangeError(`The data URI holds more than ${maxBytes} bytes.`); - } + if (Math.floor((text.length * 3) / 4) > maxBytes) throw tooLong(maxBytes); return new Uint8Array(Buffer.from(text, 'base64')); }; // RFC 2397 data is a URL-encoded string of octets: `%XX` is a byte, and any -// other character stands for its UTF-8 bytes. +// other character stands for its UTF-8 bytes. The size is bounded as the data +// is read, not once it has all been decoded into an array: a byte takes at +// most three characters (`%XX`), so more than three times `maxBytes` cannot +// fit, and every byte so far, and every character waiting to be encoded, is at +// least one byte of the result. const decodePercent = (uri, data, maxBytes) => { + if (data.length > maxBytes * 3) throw tooLong(maxBytes); const bytes = []; let text = ''; const flush = () => { if (text === '') return; + if (bytes.length + Buffer.byteLength(text, 'utf8') > maxBytes) throw tooLong(maxBytes); const encoded = Buffer.from(text, 'utf8'); for (let i = 0; i < encoded.length; i++) bytes.push(encoded[i]); text = ''; @@ -45,18 +51,17 @@ const decodePercent = (uri, data, maxBytes) => { for (let i = 0; i < data.length; i++) { if (data.charCodeAt(i) !== 0x25) { text += data[i]; + if (bytes.length + text.length > maxBytes) throw tooLong(maxBytes); continue; } const hex = data.slice(i + 1, i + 3); if (!HEX.test(hex)) throw invalid(uri); flush(); bytes.push(Number.parseInt(hex, 16)); + if (bytes.length > maxBytes) throw tooLong(maxBytes); i += 2; } flush(); - if (bytes.length > maxBytes) { - throw new RangeError(`The data URI holds more than ${maxBytes} bytes.`); - } return Uint8Array.from(bytes); }; @@ -90,8 +95,12 @@ const fromDataUri = (uri, options = {}) => { }; // nodemailer's pattern, bounded so that no input makes it backtrack long: an -// whose src is a data URI with a parameter (`;base64`). -const DATA_IMAGE = /(]{0,1024} src\s{0,20}=[\s"']{0,20})(data:([^;]+);[^"'>\s]+)/gi; +// whose src is a data URI with a parameter (`;base64`). Its media type +// was `[^;]+`, so each `]{0,1024} src\s{0,20}=[\s"']{0,20})(data:([^;"'\s>]{1,127});[^"'>\s]+)/gi; /** * Turns the images of HTML given as data URIs into inline attachments, each diff --git a/src/attachments/index.d.ts b/src/attachments/index.d.ts index 2d70a4c..49607f6 100644 --- a/src/attachments/index.d.ts +++ b/src/attachments/index.d.ts @@ -61,10 +61,12 @@ export interface FromUrlOptions extends AttachmentOptions { readonly maxRedirects?: number; /** * Whether to fetch a URL, the first and each one a redirect leads to. By - * default ({@link isPublicUrl}) `localhost` and loopback, private and - * link-local addresses are refused; a name is fetched whatever it resolves - * to, so a sender that fetches URLs from users checks them here, or passes - * a `fetch` whose dispatcher checks the address it connects to. + * default ({@link isPublicUrl}) `localhost` and IP literals in the loopback, + * private, link-local and other ranges it lists are refused; a host name is + * not resolved, so one that resolves to a private address passes, and a + * sender that fetches URLs from users checks them here, or passes a `fetch` + * that checks the address it connects to. A redirect from https to http is + * refused whatever this says. */ readonly allowUrl?: (url: URL) => boolean | Promise; } @@ -153,7 +155,12 @@ export declare function inlineDataUrls( ): { readonly html: string; readonly attachments: readonly Attachment[] }; /** - * The check of a URL {@link fromUrl} makes when no `allowUrl` is given: no - * `localhost`, and no loopback, private or link-local address. + * The check of a URL {@link fromUrl} makes when no `allowUrl` is given. It + * reads the host of the URL as written: no `localhost`, and no IP literal in a + * loopback, private, link-local, multicast, reserved or documentation range, + * or in an IPv6 range that carries an IPv4 address (IPv4-compatible, NAT64, + * 6to4). A host name is not resolved, so one that resolves to such an address + * passes, and so does one that changes its address between the check and the + * fetch (DNS rebinding). */ export declare function isPublicUrl(url: URL): boolean; diff --git a/src/attachments/url.js b/src/attachments/url.js index 1fc5075..98c7449 100644 --- a/src/attachments/url.js +++ b/src/attachments/url.js @@ -5,7 +5,8 @@ // through the `fetch` passed in, over http and https only; every redirect is // checked again and sends no header to another origin; time and bytes are // bounded; and a loopback, private or link-local address is refused unless -// `allowUrl` says otherwise. nodemailer's own client followed five redirects +// `allowUrl` says otherwise; and a redirect from https to http is refused, +// whatever `allowUrl` says. nodemailer's own client followed five redirects // with none of these checks, whatever the address. const net = require('node:net'); @@ -15,8 +16,14 @@ const { DEFAULT_TYPE, extensionOfType, typeOfFilename } = require('./mime-types. // The addresses no attachment of a sender's message should come from: this // network, the private ranges, CGNAT, loopback, link-local (where cloud -// metadata answers) and benchmarking; IPv6 unspecified, loopback, unique -// local and link-local. BlockList matches IPv4-mapped IPv6 forms too. +// metadata answers), the ranges of the IETF protocol assignments, benchmarking +// and documentation, multicast and the reserved block (the broadcast address +// among them); and in IPv6 the unspecified and loopback addresses with the +// whole of ::/96 (IPv4-compatible, deprecated), unique local, link-local, +// site-local (deprecated) and multicast. An address of NAT64 or 6to4 embeds an +// IPv4 address that a gateway on the way may reach, a private one included, so +// their prefixes are refused whole. BlockList matches IPv4-mapped IPv6 forms +// against the IPv4 rules. const PRIVATE = new net.BlockList(); PRIVATE.addSubnet('0.0.0.0', 8, 'ipv4'); PRIVATE.addSubnet('10.0.0.0', 8, 'ipv4'); @@ -24,23 +31,39 @@ PRIVATE.addSubnet('100.64.0.0', 10, 'ipv4'); PRIVATE.addSubnet('127.0.0.0', 8, 'ipv4'); PRIVATE.addSubnet('169.254.0.0', 16, 'ipv4'); PRIVATE.addSubnet('172.16.0.0', 12, 'ipv4'); +PRIVATE.addSubnet('192.0.0.0', 24, 'ipv4'); +PRIVATE.addSubnet('192.0.2.0', 24, 'ipv4'); PRIVATE.addSubnet('192.168.0.0', 16, 'ipv4'); PRIVATE.addSubnet('198.18.0.0', 15, 'ipv4'); -PRIVATE.addAddress('::', 'ipv6'); -PRIVATE.addAddress('::1', 'ipv6'); +PRIVATE.addSubnet('198.51.100.0', 24, 'ipv4'); +PRIVATE.addSubnet('203.0.113.0', 24, 'ipv4'); +PRIVATE.addSubnet('224.0.0.0', 4, 'ipv4'); +PRIVATE.addSubnet('240.0.0.0', 4, 'ipv4'); +PRIVATE.addSubnet('::', 96, 'ipv6'); +PRIVATE.addSubnet('64:ff9b::', 96, 'ipv6'); +PRIVATE.addSubnet('64:ff9b:1::', 48, 'ipv6'); +PRIVATE.addSubnet('2002::', 16, 'ipv6'); PRIVATE.addSubnet('fc00::', 7, 'ipv6'); PRIVATE.addSubnet('fe80::', 10, 'ipv6'); +PRIVATE.addSubnet('fec0::', 10, 'ipv6'); +PRIVATE.addSubnet('ff00::', 8, 'ipv6'); const REDIRECTS = new Set([301, 302, 303, 307, 308]); const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_MAX_REDIRECTS = 5; /** - * The check of a URL when `allowUrl` is not given: no `localhost`, and no - * loopback, private or link-local address. The URL parser has already - * written an address such as `0x7f.1` or `2130706433` as `127.0.0.1`. What a - * name resolves to is not known here: a sender that fetches URLs from users - * passes an `allowUrl` of its own, or a `fetch` whose dispatcher checks the + * The check of a URL when `allowUrl` is not given. It reads the host of the + * URL as written: no `localhost`, and no IP literal in a loopback, private, + * link-local, multicast, reserved or documentation range, or in an IPv6 range + * that carries an IPv4 address (IPv4-compatible, NAT64, 6to4). The URL parser + * has already written an address such as `0x7f.1` or `2130706433` as + * `127.0.0.1`. + * + * A host name is not resolved, so one that resolves to such an address passes, + * and so does one that resolves to a public address when it is checked and to + * a private one when it is fetched (DNS rebinding). A sender that fetches URLs + * its users give passes an `allowUrl` of its own, or a `fetch` that checks the * address it connects to. * * @param url The URL to fetch. @@ -123,7 +146,8 @@ const filenameOf = (url, contentType) => { * server gives. * @returns The attachment, its content in memory. * @throws {TypeError} If the URL is not http or https, or a field is invalid. - * @throws {RangeError} If a URL is not allowed, or the body is too long. + * @throws {RangeError} If a URL is not allowed (a redirect from https to http + * never is), or the body is too long. * @throws {Error} If the server answers with an error or too many redirects. */ const fromUrl = async (input, options = {}) => { @@ -149,7 +173,14 @@ const fromUrl = async (input, options = {}) => { const location = response.headers.get('location'); if (location === null) throw new Error(`HTTP ${response.status} without a Location.`); if (redirects >= maxRedirects) throw new Error(`More than ${maxRedirects} redirects.`); - url = parseHttpUrl(new URL(location, url)); + const next = parseHttpUrl(new URL(location, url)); + // The request that led here was protected by TLS; the next would not be. + if (url.protocol === 'https:' && next.protocol === 'http:') { + throw new RangeError( + `The attachment URL is not allowed: a redirect must not downgrade https to ${next.origin}`, + ); + } + url = next; continue; } if (!response.ok) { diff --git a/tests/unit/attachments.test.js b/tests/unit/attachments.test.js index f58a8c6..b2a922f 100644 --- a/tests/unit/attachments.test.js +++ b/tests/unit/attachments.test.js @@ -258,6 +258,100 @@ test('Attachments: fromUrl refuses local addresses by default, and takes an inje await assert.rejects(fromUrl('https://example.com', { maxRedirects: 1.5 }), /maxRedirects/); }); +test('Attachments: isPublicUrl refuses the ranges that reach a local address or are not unicast', () => { + // Each of these reached the machine, or a network next to it, and was allowed. + const refused = [ + // ::/96, the IPv4-compatible form (it holds :: and ::1 too). + 'http://[::127.0.0.1]/a', + 'http://[::10.0.0.1]/a', + 'http://[::a9fe:a9fe]/a', + // NAT64, 64:ff9b::/96 and 64:ff9b:1::/48: the whole prefix, whatever it embeds. + 'http://[64:ff9b::7f00:1]/a', + 'http://[64:ff9b::808:808]/a', + 'http://[64:ff9b:1::1]/a', + 'http://[64:ff9b:1:ffff::1]/a', + // 6to4, 2002::/16. + 'http://[2002:7f00:1::]/a', + 'http://[2002:a9fe:a9fe::1]/a', + 'http://[2002:5db8:d822::1]/a', + // Site-local, fec0::/10, and multicast, ff00::/8. + 'http://[fec0::1]/a', + 'http://[feff::1]/a', + 'http://[ff02::1]/a', + 'http://[ff0e::1]/a', + // IETF protocol assignments, multicast, reserved, broadcast. + 'http://192.0.0.1/a', + 'http://192.0.0.255/a', + 'http://224.0.0.1/a', + 'http://239.255.255.255/a', + 'http://240.0.0.1/a', + 'http://255.255.255.254/a', + 'http://255.255.255.255/a', + // The documentation ranges. + 'http://192.0.2.1/a', + 'http://198.51.100.7/a', + 'http://203.0.113.9/a', + ]; + for (const url of refused) assert.strictEqual(isPublicUrl(new URL(url)), false, url); + // The edges of the new ranges stay public. + const allowed = [ + 'http://192.0.1.1/a', + 'http://192.0.3.1/a', + 'http://223.255.255.255/a', + 'http://198.51.99.1/a', + 'http://198.51.101.1/a', + 'http://203.0.112.1/a', + 'http://203.0.114.1/a', + 'http://[::1:0:0]/a', + 'http://[64:ff9a::1]/a', + 'http://[64:ff9c::1]/a', + 'http://[2003::1]/a', + 'http://[2606:4700:4700::1111]/a', + ]; + for (const url of allowed) assert.strictEqual(isPublicUrl(new URL(url)), true, url); +}); + +test('Attachments: fromUrl refuses a redirect from https to http, and no other change of scheme', async () => { + const calls = []; + const redirecting = (to) => async (url) => { + calls.push(url.href); + if (calls.length === 1) return new Response(null, { status: 302, headers: { location: to } }); + return new Response('body', { headers: { 'content-type': 'text/plain' } }); + }; + // The downgrade is refused before the second request, so nothing is sent in clear. + calls.length = 0; + await assert.rejects( + fromUrl('https://cdn.example.com/a.txt', { + fetch: redirecting('http://cdn.example.com/b.txt'), + headers: { authorization: 'Bearer secret' }, + }), + /not allowed.*downgrade/s, + ); + assert.deepStrictEqual(calls, ['https://cdn.example.com/a.txt']); + // It is refused even when allowUrl allows everything, and when the host is another one. + calls.length = 0; + await assert.rejects( + fromUrl('https://cdn.example.com/a.txt', { + fetch: redirecting('http://other.example.org/b.txt'), + allowUrl: anyUrl, + }), + /downgrade/, + ); + assert.strictEqual(calls.length, 1); + // A relative Location keeps the scheme; an upgrade and http to http are allowed. + for (const [from, to] of [ + ['https://cdn.example.com/a.txt', '/b.txt'], + ['https://cdn.example.com/a.txt', 'https://other.example.org/b.txt'], + ['http://cdn.example.com/a.txt', 'https://cdn.example.com/b.txt'], + ['http://cdn.example.com/a.txt', 'http://cdn.example.com/b.txt'], + ]) { + calls.length = 0; + const attachment = await fromUrl(from, { fetch: redirecting(to) }); + assert.strictEqual(await textOf(attachment), 'body', `${from} to ${to}`); + assert.strictEqual(calls.length, 2, `${from} to ${to}`); + } +}); + test('Attachments: fromDataUri decodes base64 and percent-encoded bytes', async () => { const png = fromDataUri('data:image/png;base64,iVBO\nRw0K'); assert.strictEqual(png.contentType, 'image/png'); @@ -311,6 +405,41 @@ test('Attachments: fromDataUri refuses what does not parse or is too long', () = assert.throws(() => fromDataUri('data:,x', { inline: 'yes' }), /inline must be a boolean/); }); +test('Attachments: fromDataUri refuses a percent-encoded part that cannot fit before decoding it', () => { + const tooLong = (uri, maxBytes) => + assert.throws(() => fromDataUri(uri, { maxBytes }), RangeError); + // A byte takes at most three characters (%XX), so more than three times maxBytes cannot fit. + // The invalid escapes are never read: before, the first one was a TypeError. + tooLong(`data:,${'%G0'.repeat(1001)}`, 1000); + tooLong(`data:,${'%41'.repeat(1000)}a`, 1000); + // Fewer characters can still be too many bytes: a character is up to three. + tooLong(`data:,${'é'.repeat(501)}`, 1000); + tooLong(`data:,${'€'.repeat(334)}`, 1000); + // The count is kept as the input is read, so the decoded part never outgrows maxBytes + // before the check: it does not wait for an invalid escape at the end. + tooLong(`data:,${'a'.repeat(1001)}%G0`, 1000); + tooLong(`data:,%41${'a'.repeat(1000)}%G0`, 1000); + // What fits is decoded, to the last byte. + assert.strictEqual( + fromDataUri(`data:,${'%41'.repeat(1000)}`, { maxBytes: 1000 }).content.length, + 1000, + ); + assert.strictEqual( + fromDataUri(`data:,${'é'.repeat(500)}`, { maxBytes: 1000 }).content.length, + 1000, + ); + assert.strictEqual( + fromDataUri(`data:,${'a'.repeat(999)}%41`, { maxBytes: 1000 }).content.length, + 1000, + ); + assert.strictEqual( + fromDataUri(`data:,${'😀'.repeat(250)}`, { maxBytes: 1000 }).content.length, + 1000, + ); + // A part that is invalid and short enough is still invalid. + assert.throws(() => fromDataUri('data:,a%G0', { maxBytes: 1000 }), /Invalid data URI/); +}); + test('Attachments: fromStream reads a Node stream, a web stream or an async iterable once', async () => { const node = await fromStream(Readable.from([Buffer.from('ab'), Buffer.from('cd')]), { filename: 'n.csv', @@ -376,6 +505,44 @@ test('Attachments: inlineDataUrls makes inline attachments of the images of HTML assert.throws(() => inlineDataUrls('', { domain: 'a b' }), /Content-ID domain/); }); +test('Attachments: inlineDataUrls reads a long input in linear time', () => { + // The media type was `[^;]+`, which every ` unit.repeat(Math.ceil(bytes / unit.length)); + const time = (html) => { + const start = process.hrtime.bigint(); + const result = inlineDataUrls(html); + const elapsed = Number(process.hrtime.bigint() - start) / 1e6; + assert.strictEqual(result.html, html); + assert.deepStrictEqual(result.attachments, []); + return elapsed; + }; + time(sized('`); + assert.strictEqual(found.attachments.length, 1); + // A type past 127 characters, or with a space or a quote, is not one: the image is left as it is. + for (const type of [`${longest}a`, 'image/pn g', 'image/"png']) { + const html = ``; + assert.deepStrictEqual(inlineDataUrls(html), { html, attachments: [] }, type); + } +}); + test('Attachments: a composed message carries them, read back by postal-mime', async (t) => { const dir = tempDir(t); const file = path.join(dir, 'notes.txt'); From 61848bee2508c71e40decfbf0c9e6b2a3debc41d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:07:43 +0300 Subject: [PATCH 054/143] fix(render): htmlToText linear in its input The text was a string grown with `+=` and read at its end for each piece (`endsWith`, an index, the slice of a link's label). V8 keeps such a string as a rope and flattens it on every read, so 1 MB of `a

` took 10 s where the changelog says linear. The output is kept in pieces with its length, only the last piece is read, and trailing spaces are trimmed from the last pieces alone: the same megabyte takes about 40 ms. Co-Authored-By: Claude Opus 5.5 --- src/render/html-to-text.js | 70 ++++++++++++++++++++++++++++++-------- tests/unit/render.test.js | 36 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/src/render/html-to-text.js b/src/render/html-to-text.js index 011b4ac..7412e7b 100644 --- a/src/render/html-to-text.js +++ b/src/render/html-to-text.js @@ -115,7 +115,12 @@ const isUsableHref = (href) => { * @internal */ class TextWriter { - out = ''; + // The output in pieces, none of them empty, and the length of all. A + // string grown with `+=` is a rope that V8 flattens on any read of its + // end, so reading it for each piece was quadratic; here only the last + // piece is read, and a piece is trimmed or dropped at most once. + #chunks = []; + #length = 0; #breaks = 0; #prefix = ''; #pre = 0; @@ -129,22 +134,22 @@ class TextWriter { lineBreak() { this.#breaks = Math.min(this.#breaks + 1, 2); - if (this.out === '') this.#breaks = 0; + if (this.#length === 0) this.#breaks = 0; } text(raw) { let text = this.#pre > 0 ? raw.replaceAll('\r\n', '\n') : raw.replace(WHITESPACE, ' '); text = decodeEntities(text); - if (this.#pre === 0 && (this.out === '' || this.#breaks > 0 || this.out.endsWith('\n'))) { + if (this.#pre === 0 && (this.#length === 0 || this.#breaks > 0 || this.#endsInNewline())) { text = text.trimStart(); } if (text === '') return; - if (this.#breaks > 0 && this.out !== '') { + if (this.#breaks > 0 && this.#length > 0) { this.#trimEnd(); - this.out += this.#breaks === 2 ? '\n\n' : '\n'; + this.#append(this.#breaks === 2 ? '\n\n' : '\n'); } this.#breaks = 0; - this.out += this.#prefix + text; + this.#append(this.#prefix + text); this.#prefix = ''; } @@ -159,7 +164,7 @@ class TextWriter { } if (name === 'a') { const href = attributes?.get('href'); - this.#links.push({ href, start: this.out.length }); + this.#links.push({ href, start: this.#length }); } } @@ -178,24 +183,59 @@ class TextWriter { const address = href.toLowerCase().startsWith('mailto:') ? href.slice(7) : href; // A label much longer than the address differs from it; one this long // is not read, so that nested links stay linear. - if (this.out.length - link.start > href.length + 8) { - this.out += ` (${href})`; + if (this.#length - link.start > href.length + 8) { + this.#append(` (${href})`); return; } - const label = this.out.slice(link.start).trim(); + const label = this.#sliceFrom(link.start).trim(); if (label === '') this.text(address); - else if (label !== href && label !== address) this.out += ` (${href})`; + else if (label !== href && label !== address) this.#append(` (${href})`); } + #append(piece) { + this.#chunks.push(piece); + this.#length += piece.length; + } + + // Only called once something is written, so there is a last piece. + #endsInNewline() { + const last = this.#chunks[this.#chunks.length - 1]; + return last.charCodeAt(last.length - 1) === 10; + } + + // What the output holds from index `start` on, read from its end and no + // further than that. + #sliceFrom(start) { + const need = this.#length - start; + if (need <= 0) return ''; + let first = this.#chunks.length; + let covered = 0; + while (covered < need) covered += this.#chunks[--first].length; + let text = ''; + for (let i = first; i < this.#chunks.length; i++) text += this.#chunks[i]; + return text.slice(covered - need); + } + + // Drops the spaces and tabs the output ends with, which are in the last + // pieces alone. #trimEnd() { - let end = this.out.length; - while (end > 0 && (this.out[end - 1] === ' ' || this.out[end - 1] === '\t')) end--; - if (end < this.out.length) this.out = this.out.slice(0, end); + while (this.#chunks.length > 0) { + const last = this.#chunks[this.#chunks.length - 1]; + let end = last.length; + while (end > 0 && (last.charCodeAt(end - 1) === 32 || last.charCodeAt(end - 1) === 9)) end--; + if (end === last.length) return; + this.#length -= last.length - end; + if (end > 0) { + this.#chunks[this.#chunks.length - 1] = last.slice(0, end); + return; + } + this.#chunks.pop(); + } } finish() { this.#trimEnd(); - return this.out; + return this.#chunks.join(''); } } diff --git a/tests/unit/render.test.js b/tests/unit/render.test.js index 23e97cf..65b72dc 100644 --- a/tests/unit/render.test.js +++ b/tests/unit/render.test.js @@ -122,6 +122,42 @@ test('Render: htmlToText is linear in its input', () => { assert.throws(() => htmlToText(undefined), /htmlToText takes a string/); }); +test('Render: the spaces and tabs before a break are dropped, in however many pieces', () => { + assert.strictEqual(htmlToText('a

b'), 'a\n\nb'); + assert.strictEqual(htmlToText('x \t
y'), 'x\ny'); + assert.strictEqual(htmlToText('

a
b'), 'a\n\nb'); + assert.strictEqual(htmlToText('
 \t
'), ''); + // The break after the link's opening trims the output below where its + // label starts. + assert.strictEqual(htmlToText('x

y

'), 'x\n\ny (h)'); + assert.strictEqual(htmlToText(' z'), 'hz'); +}); + +test('Render: htmlToText takes time in proportion to its input, not its square', () => { + // The output was a string grown with `+=` and read at its end for each + // piece of text (`endsWith`, an index, the `slice` of a link's label); V8 + // flattens such a rope on every read, so a megabyte of `a

` took + // seconds. Ratios, not limits, so that a slow machine does not fail it. + const bestOf = (input) => { + let best = Infinity; + for (let run = 0; run < 3; run++) { + const started = process.hrtime.bigint(); + htmlToText(input); + best = Math.min(best, Number(process.hrtime.bigint() - started) / 1e6); + } + return best; + }; + const megabyte = 1024 * 1024; + for (const piece of ['a

', '<', 'x']) { + const one = bestOf(piece.repeat(Math.ceil(megabyte / piece.length))); + if (piece === 'a

') assert.ok(one < 1000, `1 MB of ${piece} took ${one} ms`); + const two = bestOf(piece.repeat(Math.ceil((2 * megabyte) / piece.length))); + // Linear is about 2; the floor keeps a few milliseconds of noise from + // deciding it. + assert.ok(two <= 3 * Math.max(one, 25), `${piece}: 1 MB ${one} ms, 2 MB ${two} ms`); + } +}); + test('Render: renderContent derives the text, or keeps the one given', async () => { const upper = { render: (source) => ({ html: `

${source.toUpperCase()}

` }) }; assert.deepStrictEqual(await renderContent(upper, 'hello'), { From 960e543abff0c3ebf5ea1f2d9c47e472f75b0a9b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:14:39 +0300 Subject: [PATCH 055/143] fix(webhooks): bound SNS certificate reads, check the Azure validation URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A message of a listed topic, which is no secret, made the verifier read the SigningCertURL it named on an SNS host before its signature was checked, whatever its path and query, each a new entry of the cache: a forged message with `?x=N` made a read with a 10 s timeout each time and pushed the real certificate out. An SES certificate URL must be the file of SNS, /SimpleNotificationService-….pem, with no query or fragment; at most four certificates are read at once, the next refused with 503; the timeout is certificateTimeoutMs; and signatureVersions can refuse SHA1. The Event Grid validation was handed back unchecked: its validationUrl, which an application validating by hand fetches, could name any address. It must be HTTPS on *.eventgrid.azure.net, the validationCode a string, and a token the request carries the subscription's. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 14 ++-- src/webhooks/azure.js | 46 +++++++++---- src/webhooks/index.d.ts | 37 ++++++++--- src/webhooks/scaleway.js | 6 +- src/webhooks/ses.js | 7 +- src/webhooks/sns.js | 100 +++++++++++++++++++++-------- tests/types/webhooks.test-d.ts | 9 +++ tests/unit/webhooks-others.test.js | 36 +++++++++++ tests/unit/webhooks-ses.test.js | 81 +++++++++++++++++++++++ 9 files changed, 283 insertions(+), 53 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 3ac7d85..8b056fd 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -143,9 +143,13 @@ const replayStore = { SES publishes its events to an SNS topic, which posts them to the webhook. `createSesWebhookVerifier` takes a message only from a topic in `topicArns`, signed (SignatureVersion 1 or 2) with the certificate of an -`https://sns..amazonaws.com/….pem` URL, which it reads once through -`fetch` and keeps; within an hour, as SNS retries a delivery with the time it -first published it; and once, by its `MessageId`. +`https://sns..amazonaws.com/SimpleNotificationService-….pem` URL, with +no query, which it reads once through `fetch` and keeps; within an hour, as +SNS retries a delivery with the time it first published it; and once, by its +`MessageId`. A topic set to sign with version 2 takes `signatureVersions: +['2']`, so that a message signed with SHA1 is refused. A certificate is read +within `certificateTimeoutMs` (10 seconds), and at most four at once: a +message that needs a fifth is refused with 503, for SNS to send it again. A subscription message is verified and handed back as `confirmation`, with `subscribeUrl` to fetch when the application wants the subscription; nothing is @@ -329,7 +333,9 @@ nothing, so give the subscription a secret delivery property: `Authorization` with `Bearer `, or a header of your own with `header`. Every notification must carry it; each event is taken once by its id. The validation of a new subscription carries no event and is handed back for the -application to answer. Delivery reports are `delivered`, `bounced`, +application to answer, its token checked when it carries one; its +`validationUrl`, for a validation by hand, must be on `*.eventgrid.azure.net` +over HTTPS, or the request is refused. Delivery reports are `delivered`, `bounced`, `rejected` (Suppressed) or `deferred` (Quarantined); engagement reports `opened` or `clicked`. The message id is that of the send operation, the receipt's. diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index 5ea17a9..ac15de5 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -21,6 +21,9 @@ const { // Event Grid retries a delivery for a day: an event id is kept a day and an // hour. const VALIDATION = 'Microsoft.EventGrid.SubscriptionValidationEvent'; +// Where Event Grid has a subscription validated by hand: an application that +// fetches the URL of a validation must not be sent anywhere else by it. +const EVENT_GRID_HOST = /^[a-z0-9-]+\.eventgrid\.azure\.net$/i; const DELIVERY = 'Microsoft.Communication.EmailDeliveryReportReceived'; const ENGAGEMENT = 'Microsoft.Communication.EmailEngagementTrackingReportReceived'; const DELIVERY_TYPES = new Map([ @@ -82,26 +85,45 @@ const eventOf = (record) => { * release }`. * @throws {TypeError} If an option is invalid. */ +const isEventGridUrl = (value) => { + let url; + try { + url = new URL(value); + } catch { + return false; + } + return url.protocol === 'https:' && EVENT_GRID_HOST.test(url.hostname) && url.username === ''; +}; + +const validationOf = (payload) => { + const first = Array.isArray(payload) ? payload[0] : undefined; + const data = isRecord(first) ? first.data : undefined; + if ( + first?.eventType !== VALIDATION || + !isRecord(data) || + stringOf(data.validationCode) === undefined + ) { + throw new WebhookVerificationError('payload', 'The body is not a validation event.'); + } + if (data.validationUrl !== undefined && !isEventGridUrl(data.validationUrl)) { + throw new WebhookVerificationError('payload', 'The validation URL is not of Event Grid.'); + } + return { validationCode: data.validationCode, validationUrl: data.validationUrl }; +}; + const createAzureWebhookVerifier = (options = {}) => { const checkToken = tokenCheckOf(options, 'Event Grid'); + const tokenHeader = options.header ?? 'authorization'; const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { async verify(request) { const bytes = bodyBytesOf(request, common); const kind = headerOf(request, 'aeg-event-type'); - // A validation carries no event, and answering it gives nothing away. + // A validation carries no event, and answering it gives nothing away; + // the token is checked when it comes with it. if (kind === 'SubscriptionValidation') { - const payload = parseJsonOf(bytes); - const first = Array.isArray(payload) ? payload[0] : undefined; - if (!isRecord(first) || first.eventType !== VALIDATION || !isRecord(first.data)) { - throw new WebhookVerificationError('payload', 'The body is not a validation event.'); - } - return deliveryOf([], { - validation: { - validationCode: first.data.validationCode, - validationUrl: first.data.validationUrl, - }, - }); + if (headerOf(request, tokenHeader) !== undefined) checkToken(request); + return deliveryOf([], { validation: validationOf(parseJsonOf(bytes)) }); } checkToken(request); const payload = parseJsonOf(bytes); diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 09230f1..af30446 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -164,13 +164,33 @@ export interface WebhookVerifier { } /** - * The options of {@link createSesWebhookVerifier}. + * The options of the verifiers of SNS messages (SES, Scaleway). */ -export interface SesWebhookVerifierOptions extends WebhookVerifierOptions { +export interface SnsWebhookVerifierOptions extends WebhookVerifierOptions { /** The SNS topics taken; a message of another is refused. */ readonly topicArns: readonly string[]; /** The fetch the signing certificates are read with. */ readonly fetch?: typeof globalThis.fetch; + /** + * The signature versions taken: `'1'` (RSA-SHA1), the default of an SNS + * topic, and `'2'` (RSA-SHA256). A topic set to sign with version 2 takes + * `['2']`. + * @default ['1', '2'] + */ + readonly signatureVersions?: readonly ('1' | '2')[]; + /** + * How long a signing certificate is read for, in milliseconds. At most + * four are read at once; a message that needs a fifth is refused with 503, + * for the provider to try again. + * @default 10000 + */ + readonly certificateTimeoutMs?: number; +} + +/** + * The options of {@link createSesWebhookVerifier}. + */ +export interface SesWebhookVerifierOptions extends SnsWebhookVerifierOptions { /** * How far from now an SNS message may have been published, as SNS * retries it with its first time. @@ -202,8 +222,9 @@ export interface SesWebhookResult extends WebhookResult<'ses'> { /** * Makes the verifier of the SNS messages of Amazon SES. A message is taken * when its topic is listed, its signature (version 1 or 2) matches the - * certificate of its `sns..amazonaws.com` URL, it was published within - * `toleranceMs`, and its id was not taken before. A bounce says whether it + * certificate of its `https://sns..amazonaws.com/SimpleNotificationService-….pem` + * URL, it was published within `toleranceMs`, and its id was not taken + * before. A bounce says whether it * is hard (Permanent) or soft (Transient); the `category` tags are the tags * of the message, the other tags of the application its metadata. * @@ -405,7 +426,7 @@ export declare function createSparkPostWebhookVerifier( /** * The options of {@link createScalewayWebhookVerifier}. */ -export interface ScalewayWebhookVerifierOptions extends WebhookVerifierOptions { +export interface ScalewayWebhookVerifierOptions extends SnsWebhookVerifierOptions { /** The topics of Topics and Events taken; a message of another is refused. */ readonly topicArns: readonly string[]; /** @@ -413,8 +434,6 @@ export interface ScalewayWebhookVerifierOptions extends WebhookVerifierOptions { * message is signed with. */ readonly trustChain: string; - /** The fetch the signing certificates are read with. */ - readonly fetch?: typeof globalThis.fetch; /** * How far from now a message may have been published. * @default 3600000 @@ -511,7 +530,9 @@ export interface AzureWebhookVerifierOptions { /** * The validation of a new Event Grid subscription, which the application - * answers with `{ validationResponse: validationCode }`. + * answers with `{ validationResponse: validationCode }`, or confirms by + * fetching `validationUrl`, which is on `*.eventgrid.azure.net` over HTTPS + * (another is refused). The token is checked when the validation carries it. */ export interface EventGridValidation { readonly validationCode: string; diff --git a/src/webhooks/scaleway.js b/src/webhooks/scaleway.js index 8f9a98e..3b32898 100644 --- a/src/webhooks/scaleway.js +++ b/src/webhooks/scaleway.js @@ -9,6 +9,7 @@ const { certificateError, createSnsVerifier, isHttpsUrlOf } = require('./sns.js' const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; const SCALEWAY_HOST = /(^|\.)(scw\.cloud|scaleway\.com)$/; +const isScalewayUrl = (value) => isHttpsUrlOf(value, SCALEWAY_HOST); const PEM_CERTIFICATE = /-----BEGIN CERTIFICATE-----[^-]+-----END CERTIFICATE-----/g; const TYPES = new Map([ ['email_queued', 'accepted'], @@ -77,7 +78,10 @@ const createScalewayWebhookVerifier = (options = {}) => { const verify = createSnsVerifier(options, { provider: 'scaleway', defaultToleranceMs: DEFAULT_TOLERANCE_MS, - isCertificateUrl: (value) => isHttpsUrlOf(value, SCALEWAY_HOST), + // Scaleway documents no form of its certificate URLs: the trust chain, + // not the host, is what a certificate is taken by. + isCertificateUrl: isScalewayUrl, + isSubscribeUrl: isScalewayUrl, trust: (certificate) => { for (const authority of authorities) { if (certificate.checkIssued(authority) && certificate.verify(authority.publicKey)) return; diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index 892e71e..a5bba37 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -17,11 +17,15 @@ const { createSnsVerifier, isHttpsUrlOf } = require('./sns.js'); // by default. const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; const SNS_HOST = /^sns\.[a-z0-9-]+\.amazonaws\.com(\.cn)?$/; +// Where SNS keeps its signing certificates: a file of its own, with no query +// a forged message could vary to make each a new one to read. +const SNS_CERTIFICATE_PATH = /^\/SimpleNotificationService-[A-Za-z0-9]+\.pem$/; // The tags SES writes itself, and the one `./ses` writes for the priority. const OWN_TAG = /^(ses|aws):/; // The certificates and subscriptions of SNS are on its hosts, over HTTPS. const isSnsUrl = (value) => isHttpsUrlOf(value, SNS_HOST); +const isSnsCertificateUrl = (value) => isHttpsUrlOf(value, SNS_HOST, SNS_CERTIFICATE_PATH); // The tags and metadata of `mail.tags`: `category` holds the message's tags, // as `./ses` sends them, and the others its metadata. @@ -203,7 +207,8 @@ const createSesWebhookVerifier = (options = {}) => ({ verify: createSnsVerifier(options, { provider: 'ses', defaultToleranceMs: DEFAULT_TOLERANCE_MS, - isCertificateUrl: isSnsUrl, + isCertificateUrl: isSnsCertificateUrl, + isSubscribeUrl: isSnsUrl, eventsOf: (message, record) => record === undefined ? [ diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index c2be0f4..3b5fd79 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -6,7 +6,7 @@ const { X509Certificate, verify: verifySignature } = require('node:crypto'); const { readText } = require('../internal/body.js'); -const { checkOptionalFunction } = require('../internal/options.js'); +const { checkOptionalFunction, checkPositiveNumber } = require('../internal/options.js'); const { WebhookVerificationError, bodyBytesOf, @@ -19,6 +19,11 @@ const { const MAX_CERTIFICATE_BYTES = 64 * 1024; const MAX_CERTIFICATES = 32; +// A message of a listed topic makes the verifier read its certificate before +// the signature can be checked, so a few reads at a time are made, and the +// next is refused for the provider to try again, rather than a crowd of them. +const MAX_LOADING = 4; +const DEFAULT_CERTIFICATE_TIMEOUT_MS = 10_000; const ALGORITHMS = new Map([ ['1', 'RSA-SHA1'], ['2', 'RSA-SHA256'], @@ -41,11 +46,12 @@ const SIGNED_FIELDS = new Map([ /** * Whether a value is an HTTPS URL on a host the pattern takes, with no port - * or credentials. + * or credentials, and with a path the other pattern takes and no query or + * fragment when one is given. * * @internal */ -const isHttpsUrlOf = (value, host) => { +const isHttpsUrlOf = (value, host, path) => { let url; try { url = new URL(value); @@ -57,7 +63,8 @@ const isHttpsUrlOf = (value, host) => { host.test(url.hostname) && url.port === '' && url.username === '' && - url.password === '' + url.password === '' && + (path === undefined || (path.test(url.pathname) && url.search === '' && url.hash === '')) ); }; @@ -81,11 +88,11 @@ const certificateError = (message, cause, status) => // delivery, so the answer asks for one. const UNREAD = 503; -const loadCertificate = async (fetchOption, url) => { +const loadCertificate = async (fetchOption, url, timeoutMs) => { const fetch = fetchOption ?? globalThis.fetch; let response; try { - response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(10_000) }); + response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(timeoutMs) }); } catch (error) { throw certificateError('The signing certificate could not be read.', error, UNREAD); } @@ -104,27 +111,38 @@ const loadCertificate = async (fetchOption, url) => { }; // The certificates by URL: each read once while it is cached, the oldest -// dropped past MAX_CERTIFICATES, one that failed to load not kept. -const createCertificateCache = (fetchOption, trust) => { +// dropped past MAX_CERTIFICATES, one that failed to load not kept, and no +// more than MAX_LOADING read at once. +const createCertificateCache = (fetchOption, trust, timeoutMs) => { const certificates = new Map(); + let loading = 0; return (url) => { let pending = certificates.get(url); - if (pending === undefined) { - pending = loadCertificate(fetchOption, url).then((certificate) => { + if (pending !== undefined) return pending; + if (loading >= MAX_LOADING) { + return Promise.reject( + certificateError('Too many signing certificates are being read.', undefined, UNREAD), + ); + } + loading++; + pending = loadCertificate(fetchOption, url, timeoutMs) + .then((certificate) => { trust(certificate); return certificate; + }) + .finally(() => { + loading--; }); - pending.catch(() => certificates.delete(url)); - certificates.set(url, pending); - if (certificates.size > MAX_CERTIFICATES) { - certificates.delete(certificates.keys().next().value); - } + pending.catch(() => certificates.delete(url)); + certificates.set(url, pending); + if (certificates.size > MAX_CERTIFICATES) { + certificates.delete(certificates.keys().next().value); } return pending; }; }; -const checkSignature = async (message, certificateOf, isCertificateUrl, common) => { +const checkSignature = async (message, certificateOf, service, algorithms, common) => { const fields = SIGNED_FIELDS.get(message.Type); if (fields === undefined) { throw new WebhookVerificationError( @@ -132,11 +150,15 @@ const checkSignature = async (message, certificateOf, isCertificateUrl, common) `The SNS message type ${message.Type} is unknown.`, ); } - const algorithm = ALGORITHMS.get(message.SignatureVersion); + const algorithm = algorithms.get(message.SignatureVersion); if (algorithm === undefined || typeof message.Signature !== 'string') { - throw new WebhookVerificationError('signature', 'The SNS signature version is unknown.'); + throw new WebhookVerificationError( + 'signature', + 'The SNS signature version is unknown, or not taken.', + ); } - if (typeof message.SigningCertURL !== 'string' || !isCertificateUrl(message.SigningCertURL)) { + const url = message.SigningCertURL; + if (typeof url !== 'string' || !service.isCertificateUrl(url)) { throw certificateError('The signing certificate is not on a host of the service.'); } const text = stringToSign(message, fields); @@ -151,6 +173,24 @@ const checkSignature = async (message, certificateOf, isCertificateUrl, common) } }; +// The algorithms of the signature versions taken: both by default, as a +// topic signs with version 1, SHA1, unless it is set to sign with 2. +const algorithmsOf = (versions) => { + if (versions === undefined) return ALGORITHMS; + if (!Array.isArray(versions) || versions.length === 0) { + throw new TypeError("signatureVersions lists the versions taken, '1' or '2'."); + } + const algorithms = new Map(); + for (const version of versions) { + const algorithm = ALGORITHMS.get(version); + if (algorithm === undefined) { + throw new TypeError(`signatureVersions takes '1' or '2': ${String(version)}`); + } + algorithms.set(version, algorithm); + } + return algorithms; +}; + const topicsOf = (topicArns) => { if (!Array.isArray(topicArns) || topicArns.length === 0) { throw new TypeError('topicArns must list the topics the verifier takes.'); @@ -168,20 +208,26 @@ const topicsOf = (topicArns) => { * delivery of the events of a notification, or of the `confirmation` of a * subscription message, under the id of the SNS message. * - * @param options `topicArns`, `fetch`, and the options every verifier takes. - * @param service `provider`, `defaultToleranceMs`, `isCertificateUrl` (the - * hosts of certificates and subscriptions), `trust(certificate)`, which - * throws for a certificate the service did not issue, and `eventsOf(message, - * record)`, the events of a notification, `record` its parsed message when it - * is a JSON object. + * @param options `topicArns`, `fetch`, `signatureVersions`, + * `certificateTimeoutMs`, and the options every verifier takes. + * @param service `provider`, `defaultToleranceMs`, `isCertificateUrl` and + * `isSubscribeUrl` (where its certificates and subscriptions are), + * `trust(certificate)`, which throws for a certificate the service did not + * issue, and `eventsOf(message, record)`, the events of a notification, + * `record` its parsed message when it is a JSON object. * @internal */ const createSnsVerifier = (options, service) => { const topics = topicsOf(options.topicArns); const common = resolveCommonOptions(options, service.defaultToleranceMs); + const algorithms = algorithmsOf(options.signatureVersions); const certificateOf = createCertificateCache( checkOptionalFunction(options.fetch, 'fetch'), service.trust ?? (() => {}), + checkPositiveNumber( + options.certificateTimeoutMs ?? DEFAULT_CERTIFICATE_TIMEOUT_MS, + 'certificateTimeoutMs', + ), ); return async (request) => { const message = parseJsonOf(bodyBytesOf(request, common)); @@ -191,11 +237,11 @@ const createSnsVerifier = (options, service) => { if (!topics.has(message.TopicArn)) { throw new WebhookVerificationError('topic', `The topic ${message.TopicArn} is not taken.`); } - await checkSignature(message, certificateOf, service.isCertificateUrl, common); + await checkSignature(message, certificateOf, service, algorithms, common); checkTime(Date.parse(message.Timestamp), common); const key = `${service.provider}:${message.MessageId}`; if (message.Type !== 'Notification') { - if (!service.isCertificateUrl(message.SubscribeURL)) { + if (!service.isSubscribeUrl(message.SubscribeURL)) { throw new WebhookVerificationError( 'payload', 'The subscription URL is not of the service.', diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index 5336773..ba128f6 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -107,3 +107,12 @@ const azure = createAzureWebhookVerifier({ expectType((await azure.verify({ body: '' })).validation?.validationCode); expectError(createScalewayWebhookVerifier({ topicArns: ['arn'] })); expectError(createAzureWebhookVerifier({})); +createSesWebhookVerifier({ + topicArns: ['arn'], + replayStore: null, + signatureVersions: ['2'], + certificateTimeoutMs: 5000, +}); +expectError( + createSesWebhookVerifier({ topicArns: ['arn'], replayStore: null, signatureVersions: ['3'] }), +); diff --git a/tests/unit/webhooks-others.test.js b/tests/unit/webhooks-others.test.js index 90be1dd..d2e0fdf 100644 --- a/tests/unit/webhooks-others.test.js +++ b/tests/unit/webhooks-others.test.js @@ -516,3 +516,39 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande assert.deepStrictEqual(ok.events, []); assert.throws(() => createAzureWebhookVerifier({}), /token is required/); }); + +test('Webhooks Azure: a validation sends nowhere but Event Grid, and checks a token it carries', async () => { + // Before: the validation was parsed with no check and its validationUrl + // handed back as given, so an application that fetched it could be sent to + // any address. + const verifier = createAzureWebhookVerifier({ token: 'eg-token', replayStore: null }); + const validation = (data) => ({ + body: JSON.stringify([{ eventType: 'Microsoft.EventGrid.SubscriptionValidationEvent', data }]), + headers: { 'aeg-event-type': 'SubscriptionValidation' }, + }); + const url = 'https://rp-eastus2.eventgrid.azure.net:553/eventsubscriptions/s/validate?id=1'; + const good = await verifier.verify(validation({ validationCode: 'c', validationUrl: url })); + assert.strictEqual(good.validation.validationUrl, url); + const codeOnly = await verifier.verify(validation({ validationCode: 'c' })); + assert.strictEqual(codeOnly.validation.validationUrl, undefined); + for (const validationUrl of [ + 'http://rp-eastus2.eventgrid.azure.net/validate', + 'https://169.254.169.254/latest/meta-data', + 'https://eventgrid.azure.net.example.com/validate', + 'https://user@rp.eventgrid.azure.net/validate', + 'not a url', + ]) { + await rejects(verifier.verify(validation({ validationCode: 'c', validationUrl })), 'payload'); + } + await rejects(verifier.verify(validation({ validationUrl: url })), 'payload'); + const signed = validation({ validationCode: 'c' }); + await rejects( + verifier.verify({ ...signed, headers: { ...signed.headers, authorization: 'Bearer other' } }), + 'credentials', + ); + const withToken = await verifier.verify({ + ...signed, + headers: { ...signed.headers, authorization: 'Bearer eg-token' }, + }); + assert.strictEqual(withToken.validation.validationCode, 'c'); +}); diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js index b0c0eaa..e1c1d53 100644 --- a/tests/unit/webhooks-ses.test.js +++ b/tests/unit/webhooks-ses.test.js @@ -380,3 +380,84 @@ test('Webhooks SES: the options are checked, and a replay store may be given or } assert.deepStrictEqual(fetched, [CERT_URL]); }); + +test('Webhooks SES: a certificate URL off the path of SNS, or with a query, is not read', async () => { + // Before: any path and query of an SNS host was read before the signature, + // so a message of a listed topic with `?x=N` made a new read each time and + // pushed the real certificate out of the cache. + const { verify, requests } = verifier(); + for (const url of [ + `${CERT_URL}?x=1`, + `${CERT_URL}#x`, + 'https://sns.eu-west-1.amazonaws.com/other.pem', + 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-abc.pem/x', + ]) { + const message = notification({ eventType: 'Send', mail: mail() }, { SigningCertURL: url }); + await rejects(verify(request(message)), 'certificate'); + } + assert.strictEqual(requests.length, 0); +}); + +test('Webhooks SES: a few certificates are read at a time, the next one is asked again', async () => { + const waiting = []; + const fetch = () => + new Promise((resolve) => { + waiting.push(() => resolve(new Response(require('../helpers/webhooks/sns.js').certificate))); + }); + const ses = createSesWebhookVerifier({ + topicArns: [TOPIC], + fetch, + now: () => NOW, + replayStore: null, + }); + const message = (index) => + notification( + { eventType: 'Send', mail: mail() }, + { + MessageId: `m-${index}`, + SigningCertURL: `https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-c${index}.pem`, + }, + ); + const reads = []; + for (let index = 0; index < 4; index++) reads.push(ses.verify(request(message(index)))); + await assert.rejects(ses.verify(request(message(4))), (error) => { + assert.strictEqual(error.reason, 'certificate'); + assert.strictEqual(error.status, 503); + return true; + }); + assert.strictEqual(waiting.length, 4); + for (const release of waiting) release(); + for (const read of reads) assert.strictEqual((await read).events.length, 2); + // Once read, the slots are free again. + const later = ses.verify(request(message(5))); + waiting[4](); + assert.strictEqual((await later).events.length, 2); +}); + +test('Webhooks SES: the signature versions taken, and the time a certificate is read for', async () => { + const sha1 = signSns(notification({ eventType: 'Send', mail: mail() }, { MessageId: 'v1' }), '1'); + const strict = verifier({ signatureVersions: ['2'], replayStore: null }); + await rejects(strict.verify(request(sha1)), 'signature'); + assert.strictEqual( + (await verifier({ signatureVersions: ['1'], replayStore: null }).verify(request(sha1))).events + .length, + 2, + ); + assert.throws(() => verifier({ signatureVersions: [] }), /signatureVersions/); + assert.throws(() => verifier({ signatureVersions: ['3'] }), /signatureVersions takes/); + assert.throws(() => verifier({ certificateTimeoutMs: 0 }), /certificateTimeoutMs/); + const slow = createSesWebhookVerifier({ + topicArns: [TOPIC], + now: () => NOW, + replayStore: null, + certificateTimeoutMs: 5, + fetch: (url, init) => + new Promise((resolve, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }), + }); + await assert.rejects( + slow.verify(request(notification({ eventType: 'Send', mail: mail() }))), + (error) => error.reason === 'certificate' && error.status === 503, + ); +}); From d7272e63adf03b7523a69af166ad365a10f27247 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:18:26 +0300 Subject: [PATCH 056/143] fix(attachments): resolve names in the default check, read the file checked The default check of fromUrl read the host as written, so a name of a private address, such as 127.0.0.1.nip.io or metadata.google.internal, reached it. Every address the name resolves to is now checked as an IP literal is, through a `lookup` that defaults to dns.promises.lookup; DNS rebinding is still left to a fetch that checks the address it connects to, as SECURITY.md says. An allowUrl of the application decides alone. fromPath checked the path when the attachment was made and opened it again at each reading, maybe hours later from a queue, so a directory replaced by a link in between led out of root; and no size bounded it. A reading opens the file and refuses one whose device and inode are not those checked, and maxBytes (25 MiB) holds the file when made and when read. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +-- SECURITY.md | 15 +++--- docs/messages/attachments.md | 23 ++++----- scripts/size-budgets.json | 2 +- src/attachments/index.d.ts | 31 ++++++++--- src/attachments/path.js | 47 ++++++++++++----- src/attachments/url.js | 53 ++++++++++++++----- tests/types/attachments.test-d.ts | 5 ++ tests/unit/attachments.test.js | 85 ++++++++++++++++++++++++++++++- 9 files changed, 211 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e67a3..60b6244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,9 +65,9 @@ additive but the defaults listed below. for what nodemailer read through `path`, `href` and data URIs. Nothing is read implicitly: `fromUrl` goes through an injected `fetch`, checks each redirect with `allowUrl` (by default the host as written: no `localhost`, and no IP literal of a loopback, private, link-local, - multicast, reserved or documentation range, NAT64 and 6to4 included; a host name is not - resolved), refuses a redirect from https to http, and bounds time and bytes; `fromPath` can be - held to a `root` by real path. + multicast, reserved or documentation range, NAT64 and 6to4 included, as written or as a name + resolves), refuses a redirect from https to http, and bounds time and bytes; `fromPath` can be + held to a `root` by real path and to `maxBytes`, and reads the file it checked. - **`./render`:** `renderContent(renderer, input)` for the content of `createMessage`, `createReactEmailRenderer({ render })` with the `render` of `@react-email/render` passed in, and `htmlToText`, linear in the length of its input. diff --git a/SECURITY.md b/SECURITY.md index 1ac306b..77cfa7a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,16 +48,15 @@ designed around: `fromUrl` checks every URL, redirects included, with `allowUrl`; it refuses a redirect from https to http, sends its headers to the first origin only and bounds time and bytes. `fromPath` with a `root` compares real paths, so a symbolic link cannot lead out of it. -- **The default `allowUrl` of `fromUrl` reads the host as written, and resolves nothing.** - `isPublicUrl` refuses `localhost`, `*.localhost` and an IP literal in a loopback, private, +- **The default check of `fromUrl` refuses local addresses, written or resolved, but not DNS + rebinding.** It refuses `localhost`, `*.localhost` and an address in a loopback, private, link-local (the cloud metadata address among them), carrier-grade NAT, multicast, reserved, documentation or benchmarking range, and in the IPv6 forms that carry an IPv4 address - (IPv4-compatible `::/96`, NAT64 `64:ff9b::/96` and `64:ff9b:1::/48`, 6to4 `2002::/16`). A host - *name* is not looked up, so a name that resolves to a private address passes, and so does one - that answers with a public address to the check and a private one to the fetch (DNS rebinding). - An application that fetches URLs its users give must pass an `allowUrl` of its own, or a - `fetch` that checks the address it connects to; the default is a guard against a mistake, not - against an attacker who controls a name. + (IPv4-compatible `::/96`, NAT64 `64:ff9b::/96` and `64:ff9b:1::/48`, 6to4 `2002::/16`): an IP + literal as written (`isPublicUrl`), and every address a host name resolves to. A name that + answers the fetch with another address than it gave the check (DNS rebinding) passes: an + application that fetches URLs its users give passes a `fetch` that checks the address it + connects to. An `allowUrl` of the application replaces the check, resolution included. - **The sendmail command gets arguments, not a shell line.** The recipients follow `--`, none may start with `-`, and Bcc recipients go on the command line, never into the message. - **Credentials and personal data stay out of logs.** A transport shows `[REDACTED]` in place of diff --git a/docs/messages/attachments.md b/docs/messages/attachments.md index 8204ff6..c41f15d 100644 --- a/docs/messages/attachments.md +++ b/docs/messages/attachments.md @@ -69,18 +69,17 @@ const message = createMessage({ you pass or the global one, and holds the bytes. It follows at most `maxRedirects` (5), checks every URL with `allowUrl`, and sends `headers` to the origin of the first URL only. It gives up after `timeout` (30 s) or `maxBytes` (25 MiB), and - refuses a redirect from https to http, whatever `allowUrl` says. By default - `allowUrl` is `isPublicUrl`, which looks at the host of each URL as written. It - refuses `localhost`, `*.localhost` and an IP literal in a loopback, private, - link-local (the cloud metadata address among them), carrier-grade NAT, multicast, - reserved, documentation or benchmarking range, and in the IPv6 forms that carry an - IPv4 address (IPv4-compatible `::/96`, NAT64 `64:ff9b::/96` and `64:ff9b:1::/48`, - 6to4 `2002::/16`). A host name is not resolved, so a name that resolves to a - private address passes, and so does one that answers with a public address to the - check and a private one to the fetch (DNS rebinding). An application that fetches - URLs its users give must pass an `allowUrl` of its own, or a `fetch` that checks - the address it connects to; the default guards against a mistake, not against - someone who controls a name. + refuses a redirect from https to http, whatever `allowUrl` says. Without an + `allowUrl`, each URL is checked as `isPublicUrl` checks its host as written, and + then by every address its host name resolves to (through `lookup`, + `dns.promises.lookup` by default): `localhost`, `*.localhost` and an address in + a loopback, private, link-local (the cloud metadata address among them), + carrier-grade NAT, multicast, reserved, documentation or benchmarking range, or + in the IPv6 forms that carry an IPv4 address (IPv4-compatible `::/96`, NAT64 + `64:ff9b::/96` and `64:ff9b:1::/48`, 6to4 `2002::/16`), are refused. A name + that answers the fetch with another address than it gave the check (DNS + rebinding) passes, so an application that fetches URLs its users give passes a + `fetch` that checks the address it connects to. - **`fromDataUri(uri, options)`** decodes a base64 or percent-encoded `data:` URI (RFC 2397) up to `maxBytes`, and refuses one that does not parse. - **`fromStream(source, options)`** reads a Node.js `Readable`, a web diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 19ad2bb..122a956 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -1,7 +1,7 @@ { ".": 10, "./mime": 7.5, - "./attachments": 4, + "./attachments": 4.5, "./render": 3, "./webhooks": 13, "./smtp": 23, diff --git a/src/attachments/index.d.ts b/src/attachments/index.d.ts index 49607f6..0c69705 100644 --- a/src/attachments/index.d.ts +++ b/src/attachments/index.d.ts @@ -33,6 +33,12 @@ export interface FromPathOptions extends AttachmentOptions { * the path comes from a user. */ readonly root?: string; + /** + * The longest file taken, in bytes, when it is made and each time it is + * read. + * @default 26214400 + */ + readonly maxBytes?: number; } /** Options of {@link fromUrl}. */ @@ -61,14 +67,24 @@ export interface FromUrlOptions extends AttachmentOptions { readonly maxRedirects?: number; /** * Whether to fetch a URL, the first and each one a redirect leads to. By - * default ({@link isPublicUrl}) `localhost` and IP literals in the loopback, - * private, link-local and other ranges it lists are refused; a host name is - * not resolved, so one that resolves to a private address passes, and a - * sender that fetches URLs from users checks them here, or passes a `fetch` - * that checks the address it connects to. A redirect from https to http is - * refused whatever this says. + * default, {@link isPublicUrl} of the host as written, then every address + * its name resolves to through `lookup`: `localhost` and the loopback, + * private, link-local and other ranges it lists are refused. A name that + * answers the fetch with another address than it gave the check (DNS + * rebinding) passes, so a sender that fetches URLs from users passes a + * `fetch` that checks the address it connects to. A redirect from https to + * http is refused whatever this says. */ readonly allowUrl?: (url: URL) => boolean | Promise; + /** + * Resolves a host name for the default check, as `dns.promises.lookup` + * with `{ all: true, verbatim: true }` does. Not called when `allowUrl` is + * given. + */ + readonly lookup?: ( + hostname: string, + options: { readonly all: true; readonly verbatim: true }, + ) => Promise; } /** Options of {@link fromDataUri}. */ @@ -109,6 +125,9 @@ export interface InlineDataUrlsOptions { /** * Makes an attachment of a file, read again each time the message is read. + * Each reading opens the file that was checked: one that a link or a renamed + * directory has put in its place since, or that has grown past `maxBytes`, + * fails the reading. * * @example * ```js diff --git a/src/attachments/path.js b/src/attachments/path.js index 27f8caa..9d18fac 100644 --- a/src/attachments/path.js +++ b/src/attachments/path.js @@ -6,14 +6,14 @@ // https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman // Changed in the port: the path and the root are resolved with realpath, so a // symbolic link cannot lead out of the root, which sently compared as written; -// the file is read again for each reading rather than held in memory; and no -// `path` of an attachment is read but through this helper, where nodemailer -// read any it was given. +// the file is read again for each reading rather than held in memory, and it +// must still be the file that was checked, within `maxBytes`; and no `path` of +// an attachment is read but through this helper, where nodemailer read any it +// was given. -const fs = require('node:fs'); const fsp = require('node:fs/promises'); const path = require('node:path'); -const { createAttachment } = require('./attachment.js'); +const { checkMaxBytes, createAttachment } = require('./attachment.js'); const { typeOfFilename } = require('./mime-types.js'); // Whether a resolved path is the directory or under it; a name such as @@ -24,28 +24,50 @@ const isWithin = (file, directory) => { return relative !== '..' && !relative.startsWith(`..${path.sep}`); }; +const tooLong = (filePath, maxBytes) => + new RangeError(`${JSON.stringify(filePath)} holds more than ${maxBytes} bytes.`); + // Opens the file for each reading, so the content can be read again (for -// DKIM, a retry, another transport of a pool) without holding it. -const readFile = (file) => +// DKIM, a retry, another transport of a pool) without holding it. A reading +// may come long after the check, when the message leaves a queue: what is +// opened must be the file that was checked, not one a link or a renamed +// directory put in its place, and it must not have outgrown `maxBytes`. +const readFile = (file, filePath, checked, maxBytes) => async function* (signal) { - yield* fs.createReadStream(file, { signal }); + const handle = await fsp.open(file, 'r'); + try { + const stats = await handle.stat(); + if (stats.dev !== checked.dev || stats.ino !== checked.ino) { + throw new RangeError(`${JSON.stringify(filePath)} is no longer the file it was.`); + } + let size = 0; + for await (const chunk of handle.createReadStream({ signal, autoClose: false })) { + size += chunk.length; + if (size > maxBytes) throw tooLong(filePath, maxBytes); + yield chunk; + } + } finally { + await handle.close(); + } }; /** * Makes an attachment of a file, read when the message is sent. * * @param filePath The path of the file. - * @param options The `root` the file must be in, and the fields of the - * attachment: by default its file name and the type of its extension. + * @param options The `root` the file must be in, `maxBytes` (25 MiB), and the + * fields of the attachment: by default its file name and the type of its + * extension. * @returns The attachment. * @throws {Error} If the file does not exist or cannot be resolved. - * @throws {RangeError} If it is outside `root`. + * @throws {RangeError} If it is outside `root`, or holds more than `maxBytes`. * @throws {TypeError} If it is not a regular file, or a field is invalid. */ const fromPath = async (filePath, options = {}) => { if (typeof filePath !== 'string' || filePath === '') { throw new TypeError('fromPath() needs the path of a file.'); } + const maxBytes = checkMaxBytes(options.maxBytes); const file = await fsp.realpath(filePath); if (options.root != null) { const root = await fsp.realpath(options.root); @@ -57,8 +79,9 @@ const fromPath = async (filePath, options = {}) => { } const stats = await fsp.stat(file); if (!stats.isFile()) throw new TypeError(`${JSON.stringify(filePath)} is not a file.`); + if (stats.size > maxBytes) throw tooLong(filePath, maxBytes); const filename = path.basename(filePath); - return createAttachment(options, readFile(file), { + return createAttachment(options, readFile(file, filePath, stats, maxBytes), { filename, contentType: typeOfFilename(options.filename ?? filename), }); diff --git a/src/attachments/url.js b/src/attachments/url.js index 98c7449..fa4f26f 100644 --- a/src/attachments/url.js +++ b/src/attachments/url.js @@ -9,6 +9,7 @@ // whatever `allowUrl` says. nodemailer's own client followed five redirects // with none of these checks, whatever the address. +const dns = require('node:dns'); const net = require('node:net'); const { readBytes } = require('../internal/body.js'); const { checkMaxBytes, createAttachment, mediaTypeOf } = require('./attachment.js'); @@ -52,31 +53,55 @@ const REDIRECTS = new Set([301, 302, 303, 307, 308]); const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_MAX_REDIRECTS = 5; +const hostOf = (url) => { + let host = url.hostname.toLowerCase(); + if (host.startsWith('[')) host = host.slice(1, -1); + return host.endsWith('.') ? host.slice(0, -1) : host; +}; + +const isPrivateAddress = (address, family) => + PRIVATE.check(address, family === 6 || family === 'IPv6' ? 'ipv6' : 'ipv4'); + /** - * The check of a URL when `allowUrl` is not given. It reads the host of the - * URL as written: no `localhost`, and no IP literal in a loopback, private, + * The check of the host of a URL as written, the first of those `fromUrl` + * makes when no `allowUrl` is given: no `localhost`, and no IP literal in a loopback, private, * link-local, multicast, reserved or documentation range, or in an IPv6 range * that carries an IPv4 address (IPv4-compatible, NAT64, 6to4). The URL parser * has already written an address such as `0x7f.1` or `2130706433` as * `127.0.0.1`. * - * A host name is not resolved, so one that resolves to such an address passes, - * and so does one that resolves to a public address when it is checked and to - * a private one when it is fetched (DNS rebinding). A sender that fetches URLs - * its users give passes an `allowUrl` of its own, or a `fetch` that checks the - * address it connects to. + * It resolves no name: `fromUrl`, when no `allowUrl` is given, also checks + * every address a host name resolves to, but a name that answers the fetch + * otherwise than it answered the check (DNS rebinding) passes. A sender that + * fetches URLs its users give passes a `fetch` that checks the address it + * connects to. * * @param url The URL to fetch. * @returns Whether to fetch it. */ const isPublicUrl = (url) => { - let host = url.hostname.toLowerCase(); - if (host.startsWith('[')) host = host.slice(1, -1); - if (host.endsWith('.')) host = host.slice(0, -1); + const host = hostOf(url); if (host === 'localhost' || host.endsWith('.localhost')) return false; const family = net.isIP(host); - if (family === 0) return true; - return !PRIVATE.check(host, family === 4 ? 'ipv4' : 'ipv6'); + return family === 0 || !isPrivateAddress(host, family); +}; + +// The check `fromUrl` makes when no `allowUrl` is given: `isPublicUrl`, then +// every address a host name resolves to, so that a name of a private address +// (`127.0.0.1.nip.io`, `metadata.google.internal`) is refused as its address +// is. A name that answers the fetch otherwise than it answered here (DNS +// rebinding) is caught only by a `fetch` that checks the address it connects +// to. +const resolvesPublic = async (url, lookup) => { + if (!isPublicUrl(url)) return false; + const host = hostOf(url); + if (net.isIP(host) !== 0) return true; + const addresses = await lookup(host, { all: true, verbatim: true }); + if (!Array.isArray(addresses) || addresses.length === 0) return false; + for (const entry of addresses) { + if (isPrivateAddress(entry.address, entry.family)) return false; + } + return true; }; const parseHttpUrl = (value) => { @@ -157,7 +182,9 @@ const fromUrl = async (input, options = {}) => { if (!Number.isSafeInteger(maxRedirects) || maxRedirects < 0) { throw new TypeError(`maxRedirects must be a whole number: ${maxRedirects}`); } - const allowUrl = options.allowUrl ?? isPublicUrl; + const lookup = options.lookup ?? dns.promises.lookup; + if (typeof lookup !== 'function') throw new TypeError('lookup must be a function.'); + const allowUrl = options.allowUrl ?? ((url) => resolvesPublic(url, lookup)); const signal = signalOf(checkTimeout(options.timeout), options.signal); const origin = parseHttpUrl(input).origin; let url = parseHttpUrl(input); diff --git a/tests/types/attachments.test-d.ts b/tests/types/attachments.test-d.ts index 580d64b..2afd90f 100644 --- a/tests/types/attachments.test-d.ts +++ b/tests/types/attachments.test-d.ts @@ -40,3 +40,8 @@ createMessage({ expectError(fromPath(42)); expectError(fromUrl('https://example.com', { contentType: 'pdf' })); expectError(fromDataUri('data:,x', { inline: 'yes' })); +await fromUrl('https://example.com/a', { + lookup: async () => [{ address: '93.184.215.14', family: 4 }], +}); +await fromPath('./a.pdf', { root: '.', maxBytes: 1024 }); +expectError(fromUrl('https://example.com/a', { lookup: 'dns' })); diff --git a/tests/unit/attachments.test.js b/tests/unit/attachments.test.js index b2a922f..84c2ccf 100644 --- a/tests/unit/attachments.test.js +++ b/tests/unit/attachments.test.js @@ -24,6 +24,9 @@ const bytesOf = (text) => new TextEncoder().encode(text); const textOf = async (attachment) => new TextDecoder().decode(await readAttachmentContent(attachment.content)); +// A resolver of every name to a public address, so that no test asks DNS. +const publicLookup = async () => [{ address: '93.184.215.14', family: 4 }]; + const tempDir = (t) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mailer-attachments-')); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -241,6 +244,7 @@ test('Attachments: fromUrl refuses local addresses by default, and takes an inje await assert.rejects(fromUrl('not a url'), /Invalid attachment URL/); const calls = []; const attachment = await fromUrl('https://cdn.example.com/a.css', { + lookup: publicLookup, fetch: async (url, init) => { calls.push({ url: url.href, redirect: init.redirect }); return new Response('body{}', { headers: { 'content-type': 'text/css' } }); @@ -250,6 +254,7 @@ test('Attachments: fromUrl refuses local addresses by default, and takes an inje assert.strictEqual(attachment.contentType, 'text/css'); // A segment that does not decode keeps its percent-encoding. const raw = await fromUrl('https://cdn.example.com/a%E0.txt', { + lookup: publicLookup, fetch: async () => new Response('x'), }); assert.strictEqual(raw.filename, 'a%E0.txt'); @@ -322,6 +327,7 @@ test('Attachments: fromUrl refuses a redirect from https to http, and no other c calls.length = 0; await assert.rejects( fromUrl('https://cdn.example.com/a.txt', { + lookup: publicLookup, fetch: redirecting('http://cdn.example.com/b.txt'), headers: { authorization: 'Bearer secret' }, }), @@ -346,7 +352,7 @@ test('Attachments: fromUrl refuses a redirect from https to http, and no other c ['http://cdn.example.com/a.txt', 'http://cdn.example.com/b.txt'], ]) { calls.length = 0; - const attachment = await fromUrl(from, { fetch: redirecting(to) }); + const attachment = await fromUrl(from, { lookup: publicLookup, fetch: redirecting(to) }); assert.strictEqual(await textOf(attachment), 'body', `${from} to ${to}`); assert.strictEqual(calls.length, 2, `${from} to ${to}`); } @@ -575,3 +581,80 @@ test('Attachments: a composed message carries them, read back by postal-mime', a assert.deepStrictEqual(found['s.txt'], ['text/plain', 'streamed', false]); assert.strictEqual(found['image-1.png'][2], true); }); + +test('Attachments: fromPath holds a file to maxBytes, when made and when read', async (t) => { + // Before: no bound, and the whole file was buffered for a provider. + const file = path.join(tempDir(t), 'big.bin'); + fs.writeFileSync(file, Buffer.alloc(16)); + await assert.rejects(fromPath(file, { maxBytes: 8 }), /holds more than 8 bytes/); + await assert.rejects(fromPath(file, { maxBytes: 0 }), /maxBytes must be/); + const attachment = await fromPath(file, { maxBytes: 16 }); + fs.appendFileSync(file, Buffer.alloc(1)); + await assert.rejects(textOf(attachment), /holds more than 16 bytes/); +}); + +test('Attachments: fromPath reads the file that was checked, not one put in its place', async (t) => { + // Before: the path was checked once and opened again at each reading, so a + // directory renamed and replaced by a link after the check led out of root. + const base = tempDir(t); + const root = path.join(base, 'root'); + const outside = path.join(base, 'outside'); + fs.mkdirSync(path.join(root, 'uploads'), { recursive: true }); + fs.mkdirSync(outside); + fs.writeFileSync(path.join(root, 'uploads', 'a.txt'), 'mine'); + fs.writeFileSync(path.join(outside, 'a.txt'), 'secret'); + const attachment = await fromPath(path.join(root, 'uploads', 'a.txt'), { root }); + assert.strictEqual(await textOf(attachment), 'mine'); + fs.renameSync(path.join(root, 'uploads'), path.join(root, 'moved')); + try { + fs.symlinkSync(outside, path.join(root, 'uploads'), 'dir'); + } catch (error) { + // Windows asks for a privilege to make a link. + if (error.code === 'EPERM') return; + throw error; + } + await assert.rejects(textOf(attachment), /no longer the file it was/); +}); + +test('Attachments: fromUrl refuses a name that resolves to a private address', async () => { + // Before: only the host as written was checked, so 127.0.0.1.nip.io or + // metadata.google.internal reached the address their name stands for. + const fetched = []; + const fetch = async (url) => { + fetched.push(url.href); + return new Response('x'); + }; + const resolving = + (...addresses) => + async (host, options) => { + assert.deepStrictEqual(options, { all: true, verbatim: true }); + return addresses; + }; + for (const addresses of [ + [{ address: '127.0.0.1', family: 4 }], + [{ address: '169.254.169.254', family: 4 }], + [{ address: '::1', family: 6 }], + [ + { address: '93.184.215.14', family: 4 }, + { address: '10.0.0.7', family: 4 }, + ], + [], + ]) { + await assert.rejects( + fromUrl('https://127.0.0.1.nip.io/a', { fetch, lookup: resolving(...addresses) }), + /not allowed/, + JSON.stringify(addresses), + ); + } + assert.deepStrictEqual(fetched, []); + await fromUrl('https://cdn.example.com/a', { + fetch, + lookup: resolving({ address: '2606:4700::1', family: 6 }), + }); + // An address is not resolved; an allowUrl of the application decides alone. + const unused = async () => assert.fail('resolved'); + await fromUrl('http://93.184.215.14/a', { fetch, lookup: unused }); + await fromUrl('https://intranet.local/a', { fetch, lookup: unused, allowUrl: () => true }); + assert.strictEqual(fetched.length, 3); + await assert.rejects(fromUrl('https://a.example/', { lookup: 'x' }), /lookup must be a function/); +}); From f4cc19135f6e2cfbe95eb7d97472c67399517cf6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:23:10 +0300 Subject: [PATCH 057/143] fix(smtp): proxy credentials not in clear, a deadline for the whole CONNECT createHttpProxyConnector sent the Basic credentials of the proxy URL over plain HTTP to any host, where the providers' keys are refused over it. An http:// proxy takes credentials only on this machine, or with allowPlaintextCredentials for a network where that is accepted. Its timeout was a socket timeout, reset by each byte, so a proxy writing one byte at a time held the connection past it. It is a deadline for the whole exchange, from the connection to the reply. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- docs/transports/smtp.md | 11 ++++++--- src/smtp/http-proxy.js | 43 ++++++++++++++++++++++++--------- src/smtp/index.d.ts | 11 +++++++-- tests/unit/smtp-connect.test.js | 32 ++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60b6244..41e7e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,8 @@ additive but the defaults listed below. `htmlToText`, linear in the length of its input. - **`./smtp` connects through what you give it:** `connect` opens each connection (a proxy, a tunnel, a socket of another module) with implicit TLS or STARTTLS over it; - `createHttpProxyConnector` makes one for an HTTP or HTTPS proxy; `localAddress` picks the address + `createHttpProxyConnector` makes one for an HTTP or HTTPS proxy, whose credentials go in clear + to an `http://` proxy only on this machine or with `allowPlaintextCredentials`; `localAddress` picks the address to connect from. - **`./sendmail`:** the sendmail command of the system (Postfix, Exim, msmtp), one process per message, composed by `./mime` with DKIM; `sendRaw` and `verify`. Only an exit with 0 succeeds, a diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index a56860d..9b00030 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -204,14 +204,17 @@ const transport = new SmtpTransport({ port: 465, auth: { user: 'user@example.com', pass: 'secret' }, connect: createHttpProxyConnector({ - url: 'http://proxy-user:proxy-pass@proxy.internal:3128', - // For an https:// proxy: tls: { ca: [proxyCa] } + url: 'https://proxy-user:proxy-pass@proxy.internal:3129', + tls: { ca: [proxyCa] }, }), }); ``` -The credentials of the proxy URL go as Basic authentication, as UTF-8; `headers` -adds fields to each CONNECT request, and `timeout` bounds its answer (30 s). A +The credentials of the proxy URL go as Basic authentication, as UTF-8, which is +the password itself: they go to an `http://` proxy only on this machine, and to +another only with `allowPlaintextCredentials: true`, for a network where that is +accepted. `headers` adds fields to each CONNECT request, and `timeout` bounds the +whole exchange, from the connection to the answer (30 s). A SOCKS proxy is a module of your own in `connect`, which receives `{ host, port, localAddress, signal }` and resolves to a connected `net.Socket`, paused or with nothing read from it yet: diff --git a/src/smtp/http-proxy.js b/src/smtp/http-proxy.js index d7cb605..229dbbc 100644 --- a/src/smtp/http-proxy.js +++ b/src/smtp/http-proxy.js @@ -6,12 +6,16 @@ // request line and Host field, which nodemailer wrote bare; an HTTPS proxy // takes `ca` and the other TLS options and is sent its name for SNI, where // nodemailer passed `rejectUnauthorized` alone; the credentials of the proxy URL -// are percent-decoded and encoded as UTF-8; and the socket is paused with what -// came after the reply put back, so a greeting sent at once is not lost. +// are percent-decoded and encoded as UTF-8, and go to an `http:` proxy only on +// this machine unless the application allows them in clear; the socket is +// paused with what came after the reply put back, so a greeting sent at once +// is not lost; and `timeout` bounds the whole exchange, where a socket timeout +// bounded only each wait for a byte. const { Buffer } = require('node:buffer'); const net = require('node:net'); const tls = require('node:tls'); +const { isLoopbackHost } = require('../internal/loopback.js'); // The most bytes of the reply to CONNECT before its blank line, so a proxy // that never ends its header fields cannot fill the memory of the sender. @@ -95,9 +99,12 @@ const proxyError = (status, target) => { * * @param options `url` of the proxy, with credentials for Basic authentication * if it needs them; `headers` to add to each CONNECT; `tls` options for an - * HTTPS proxy; `timeout` of the CONNECT exchange (30 s, 0 for none). + * HTTPS proxy; `timeout` of the CONNECT exchange (30 s, 0 for none); and + * `allowPlaintextCredentials`, to send credentials to an `http:` proxy that is + * not on this machine. * @returns The connector. - * @throws {TypeError} If the URL is not http or https, or a header is invalid. + * @throws {TypeError} If the URL is not http or https, a header is invalid, or + * credentials would go in clear. */ const createHttpProxyConnector = (options) => { if (options == null || typeof options !== 'object') { @@ -110,10 +117,25 @@ const createHttpProxyConnector = (options) => { throwTypeError(`Invalid proxy timeout: ${timeout}`); } const isHttps = url.protocol === 'https:'; + const proxyHost = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; + // Basic credentials are the password itself: over plain HTTP, anyone on the + // way to the proxy reads them, as the providers' keys are refused over it. + const hasCredentials = url.username !== '' || url.password !== ''; + if ( + hasCredentials && + !isHttps && + !isLoopbackHost(proxyHost) && + options.allowPlaintextCredentials !== true + ) { + throwTypeError( + 'The credentials of an http:// proxy would go in clear: use an https:// proxy, or set ' + + 'allowPlaintextCredentials for a network where that is accepted.', + ); + } const defaultPort = isHttps ? 443 : 80; const proxy = { isHttps, - host: url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname, + host: proxyHost, port: url.port === '' ? defaultPort : Number(url.port), tls: options.tls, }; @@ -147,14 +169,14 @@ const tunnel = (socket, isHttps, request, target, timeout, signal) => let received = 0; let tail = ''; + let timer; const cleanup = () => { socket.off(connected, onConnect); socket.off('data', onData); socket.off('error', fail); socket.off('close', onClose); - socket.off('timeout', onTimeout); signal?.removeEventListener('abort', onAbort); - socket.setTimeout(0); + clearTimeout(timer); }; const fail = (error) => { cleanup(); @@ -196,10 +218,9 @@ const tunnel = (socket, isHttps, request, target, timeout, signal) => socket.on('data', onData); socket.once('error', fail); socket.once('close', onClose); - if (timeout > 0 && timeout !== Number.POSITIVE_INFINITY) { - socket.setTimeout(timeout); - socket.once('timeout', onTimeout); - } + // The whole exchange, not each wait for a byte: a proxy that answers a + // byte at a time would hold a socket timeout off forever. + if (timeout > 0 && timeout !== Number.POSITIVE_INFINITY) timer = setTimeout(onTimeout, timeout); signal?.addEventListener('abort', onAbort, { once: true }); }); diff --git a/src/smtp/index.d.ts b/src/smtp/index.d.ts index 33e156f..0fdeebe 100644 --- a/src/smtp/index.d.ts +++ b/src/smtp/index.d.ts @@ -681,11 +681,18 @@ export interface HttpProxyConnectorOptions { /** TLS options of an `https://` proxy, such as its `ca`. */ readonly tls?: SmtpTlsOptions; /** - * The time the proxy may take to answer CONNECT, in milliseconds; 0 or - * `Infinity` for none. + * The time the proxy may take to connect and answer CONNECT, in all, in + * milliseconds; 0 or `Infinity` for none. * @default 30000 */ readonly timeout?: number; + /** + * Sends the credentials of an `http://` proxy that is not on this machine, + * in clear, which is refused otherwise: for a network where that is + * accepted. An `https://` proxy needs no such thing. + * @default false + */ + readonly allowPlaintextCredentials?: boolean; } /** * Makes a `connect` for {@link SmtpConfig} that opens each connection through diff --git a/tests/unit/smtp-connect.test.js b/tests/unit/smtp-connect.test.js index 3d77c6d..50a46b7 100644 --- a/tests/unit/smtp-connect.test.js +++ b/tests/unit/smtp-connect.test.js @@ -314,3 +314,35 @@ test('SMTP proxy: the options and the destination are checked', async () => { await assert.rejects(connector({ host: 'a\r\nb', port: 25 }), /destination host/); await assert.rejects(connector({ host: 'smtp.example.com', port: 0 }), /destination port/); }); + +test('SMTP proxy: credentials go to an http proxy only on this machine, unless allowed', () => { + // Before: Basic credentials of the proxy URL went in clear to any host. + assert.throws( + () => createHttpProxyConnector({ url: 'http://user:pass@proxy.example.com:3128' }), + /would go in clear/, + ); + assert.throws(() => createHttpProxyConnector({ url: 'http://:pass@10.0.0.2:3128' }), /in clear/); + assert.ok(createHttpProxyConnector({ url: 'http://user:pass@127.0.0.1:3128' })); + assert.ok(createHttpProxyConnector({ url: 'http://user:pass@localhost:3128' })); + assert.ok(createHttpProxyConnector({ url: 'https://user:pass@proxy.example.com' })); + assert.ok(createHttpProxyConnector({ url: 'http://proxy.example.com:3128' })); + assert.ok( + createHttpProxyConnector({ + url: 'http://user:pass@proxy.example.com:3128', + allowPlaintextCredentials: true, + }), + ); +}); + +test('SMTP proxy: the timeout bounds the whole CONNECT, not each wait for a byte', async (t) => { + // Before: a socket timeout, which a proxy writing a byte at a time never let fire. + const trickle = (request, client) => { + const timer = setInterval(() => client.write('H'), 20); + client.on('close', () => clearInterval(timer)); + }; + const proxy = await proxyServer(t, { reply: trickle }); + const connector = createHttpProxyConnector({ url: proxy.url, timeout: 150 }); + const started = Date.now(); + await assert.rejects(connector({ host: 'smtp.example.com', port: 25 }), /timed out on CONNECT/); + assert.ok(Date.now() - started < 1000); +}); From 05f62b37d61eab0eca54cff41653414ad964a886 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:26:36 +0300 Subject: [PATCH 058/143] fix(ratelimit): hold maxKeys buckets, stop the waits when disposed of maxKeys only started a sweep of the buckets full again, so with a long interval and a key per recipient every key used in it was kept, and each sweep moved the next to twice as many: no bound. The buckets are kept in the order of their use, and past maxKeys the least used one with no waiter is dropped. Disposal left the waiters queued with their timer, to send through the disposed transport when their tokens came; it now clears the timers and rejects them with an AbortError, and refuses later sends. A signal that aborted between two messages of a batch fires no event, so the next one waited and took its tokens; it is refused at once. Co-Authored-By: Claude Opus 5.5 --- docs/transports/ratelimit.md | 9 ++-- src/ratelimit/index.d.ts | 9 +++- src/ratelimit/ratelimit-transport.js | 75 +++++++++++++++++++--------- tests/unit/ratelimit.test.js | 54 ++++++++++++++++++++ 4 files changed, 118 insertions(+), 29 deletions(-) diff --git a/docs/transports/ratelimit.md b/docs/transports/ratelimit.md index 7353732..449c42e 100644 --- a/docs/transports/ratelimit.md +++ b/docs/transports/ratelimit.md @@ -35,9 +35,10 @@ takes `cost(message)` of them, 1 without `cost`. ## Per key `key(message)` chooses the bucket, such as a tenant's, so one tenant's burst -does not hold back the others. Buckets that are full again and have no waiter -are dropped once there are `maxKeys` of them, as one made anew would be the -same. +does not hold back the others. No more than `maxKeys` buckets are kept: past +it, the least used one with no waiter is dropped, and its key starts full when +it comes back, the price of a bound on memory. Disposing of the transport stops +the waits, each rejecting with an `AbortError`. ```js const perTenant = new RateLimitTransport(transport, { @@ -57,7 +58,7 @@ const perTenant = new RateLimitTransport(transport, { | `maxWaitMs` | `30000` | The longest wait, from the arrival. | | `key` | one bucket | The bucket of a message. | | `cost` | `1` | The tokens a message takes. | -| `maxKeys` | `10000` | The buckets kept before full idle ones are dropped. | +| `maxKeys` | `10000` | The most buckets kept, the least used dropped first. | | `now` | `Date.now` | The clock. | `sendMany()` goes through the `sendMany()` of the wrapped transport, each diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts index 808a009..56a5067 100644 --- a/src/ratelimit/index.d.ts +++ b/src/ratelimit/index.d.ts @@ -52,7 +52,8 @@ export interface RateLimitConfig { */ readonly cost?: (message: Message | RawMessage) => number; /** - * The buckets kept before those full and idle are dropped. + * The most buckets kept: past it, the least used bucket with no waiter is + * dropped, and its key starts full when it comes back. * @default 10000 */ readonly maxKeys?: number; @@ -126,6 +127,10 @@ export declare class RateLimitTransport ): AsyncIterable>; /** Verifies the wrapped transport, without taking tokens. */ verify(options?: TransportOptions): Promise; - /** Disposes of the wrapped transport when it supports disposal. */ + /** + * Stops the waits, each send waiting for tokens rejecting with an + * `AbortError`, refuses the sends after it, and disposes of the wrapped + * transport when it supports disposal. + */ [Symbol.asyncDispose](): Promise; } diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index dc03319..c9c6fde 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -7,7 +7,8 @@ // the arrival, so a message is refused at once when the queue ahead of it // would hold it longer, where unemail started counting only at its turn; a // cost over the capacity is refused, where unemail cut it to the capacity and -// sent more than the limit; and the buckets of the keys are bounded. +// sent more than the limit; the buckets of the keys are held to `maxKeys`, the +// least used dropped first; and disposal stops the waits. const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); @@ -30,9 +31,10 @@ class RateLimitTransport { id; config; wrappedTransport; + // In the order of their last use, the least recent first. #buckets = new Map(); - #sweepAt; #refillPerMs; + #disposed = false; /** * @param transport The transport to throttle. @@ -45,7 +47,6 @@ class RateLimitTransport { this.id = transport.id; this.config = createRateLimitConfig(config); this.#refillPerMs = this.config.limit / this.config.intervalMs; - this.#sweepAt = this.config.maxKeys; } /** @@ -91,14 +92,30 @@ class RateLimitTransport { if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); } - /** Disposes of the wrapped transport when it supports disposal. */ + /** + * Stops the waits, each send that waits rejecting with an `AbortError`, + * and disposes of the wrapped transport when it supports disposal. + */ async [Symbol.asyncDispose]() { + this.#disposed = true; + const reason = disposedError(); + for (const bucket of this.#buckets.values()) { + clearTimeout(bucket.timer); + while (bucket.waiters.size > 0) { + const waiter = bucket.waiters.shift(); + if (waiter.cancelled) continue; + waiter.signal?.removeEventListener('abort', waiter.onAbort); + waiter.reject(reason); + } + } + this.#buckets.clear(); await disposeOf(this.wrappedTransport); } // Takes the tokens of a message, waiting for them: `undefined` once taken, // or the receipt of a refusal. #admit(message, signal) { + if (this.#disposed) return Promise.reject(disposedError()); const { cost: costOf, key: keyOf } = this.config; const cost = costOf === undefined ? 1 : costOf(message); if (typeof cost !== 'number' || !(cost >= 0) || cost === Infinity) { @@ -147,7 +164,13 @@ class RateLimitTransport { #wait(bucket, cost, signal) { return new Promise((resolve, reject) => { - const waiter = { cost, resolve, cancelled: false, signal, onAbort: undefined }; + // A signal that aborted before, between two messages of a batch, fires + // no event: the message would wait and take its tokens. + if (signal?.aborted) { + reject(abortReason(signal)); + return; + } + const waiter = { cost, resolve, reject, cancelled: false, signal, onAbort: undefined }; if (signal !== undefined) { waiter.onAbort = () => { waiter.cancelled = true; @@ -194,34 +217,40 @@ class RateLimitTransport { bucket.last = now; } + // The bucket of a key, moved to the end of the order of use. #bucketOf(key) { let bucket = this.#buckets.get(key); - if (bucket === undefined) { - if (this.#buckets.size >= this.#sweepAt) this.#sweep(); - bucket = { - tokens: this.config.burst, - last: this.config.now(), - waiters: new Fifo(), - queuedCost: 0, - timer: undefined, - }; + if (bucket !== undefined) { + this.#buckets.delete(key); this.#buckets.set(key, bucket); + return bucket; } + if (this.#buckets.size >= this.config.maxKeys) this.#evict(); + bucket = { + tokens: this.config.burst, + last: this.config.now(), + waiters: new Fifo(), + queuedCost: 0, + timer: undefined, + }; + this.#buckets.set(key, bucket); return bucket; } - // Drops the buckets that are full again and have no waiter: one made anew - // would be the same. When too few are, the next sweep waits for twice as - // many, so a crowd of busy keys does not sweep on every new one. - #sweep() { + // Makes room for a key: drops the least used buckets that have no waiter, + // until there is room. One that is not full starts full when its key comes + // back, the price of holding no more than `maxKeys` keys; a bucket with + // waiters is never dropped, so they alone can hold more. + #evict() { for (const [key, bucket] of this.#buckets) { - this.#refill(bucket); - if (bucket.waiters.size === 0 && bucket.tokens >= this.config.burst) { - this.#buckets.delete(key); - } + if (this.#buckets.size < this.config.maxKeys) return; + if (bucket.waiters.size === 0) this.#buckets.delete(key); } - this.#sweepAt = Math.max(this.config.maxKeys, this.#buckets.size * 2); } } +// What a send waiting for tokens rejects with when the transport is disposed of. +const disposedError = () => + new DOMException('The rate limit transport was disposed of.', 'AbortError'); + module.exports = { RateLimitTransport }; diff --git a/tests/unit/ratelimit.test.js b/tests/unit/ratelimit.test.js index 3d57e05..8ea937c 100644 --- a/tests/unit/ratelimit.test.js +++ b/tests/unit/ratelimit.test.js @@ -210,3 +210,57 @@ test('RateLimit: sendRaw, verify, dispose and the options', async (t) => { assert.strictEqual(defaults.config.mode, 'wait'); assert.strictEqual(defaults.id, 'fake'); }); + +test('RateLimit: no more than maxKeys keys are held, the least used dropped first', async (t) => { + // Before: maxKeys only started a sweep of the full buckets, so keys whose + // buckets had not refilled were all kept, and the next sweep waited for + // twice as many. + const clock = timeline(t); + const limiter = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 3_600_000, + mode: 'reject', + maxKeys: 2, + key: (m) => m.recipients[0].address, + now: clock.now, + }); + const to = (address) => limiter.send(message('x', address)); + assert.strictEqual((await to('a@example.com')).successful, true); + assert.strictEqual((await to('b@example.com')).successful, true); + // a is limited for an hour; used again, it is the most recent. + assert.strictEqual((await to('a@example.com')).successful, false); + // c needs a place: b, the least used, goes, and starts full when it is back. + assert.strictEqual((await to('c@example.com')).successful, true); + assert.strictEqual((await to('a@example.com')).successful, false); + assert.strictEqual((await to('b@example.com')).successful, true); +}); + +test('RateLimit: dispose stops the waits, and a later send is refused', async (t) => { + // Before: the waiters stayed queued with their timer, and sent through the + // disposed transport when their tokens came. + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 1, intervalMs: 1000, now: clock.now }); + await limiter.send(message('first')); + const waiting = limiter.send(message('second')); + await limiter[Symbol.asyncDispose](); + await assert.rejects(waiting, { name: 'AbortError' }); + await assert.rejects(limiter.send(message('third')), { name: 'AbortError' }); + await clock.advance(5000); + assert.strictEqual(inner.calls.length, 1); +}); + +test('RateLimit: a signal that aborted between two messages of a batch stops the next', async (t) => { + // Before: an aborted signal fires no event, so the next message waited and + // took its tokens. + const clock = timeline(t); + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 1, intervalMs: 1000, now: clock.now }); + const controller = new AbortController(); + const receipts = limiter.sendMany([message('a'), message('b')], { signal: controller.signal }); + const iterator = receipts[Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).value.successful, true); + controller.abort(new Error('stop')); + await assert.rejects(iterator.next(), /stop/); + assert.strictEqual(inner.calls.length, 1); +}); From fe66528a3f17f41d073446d32c697b087d9a00f5 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:30:27 +0300 Subject: [PATCH 059/143] fix(tee): bound each copy, wait for them when disposed of; limit warnings A mirror's timeout reached it only through its signal, so one that ignored the signal never settled and held a slot of maxPending for good. Its send races a timer as well. Disposal disposed of the mirrors while their copies were in flight; it waits for them first, each bounded by its timeout. The primary and each mirror composed their own Message-ID; the copies share the primary's. Without onMirror, a failed copy was a process warning with the first error text of its receipt as it was, the recipient of an SMTP rejection in it, once for every message. The text of a mirror's failure and of what a hook, a listener or a store threw has its addresses redacted, and no more than ten warnings of a code go out a minute, the first after it counting those left out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++- docs/transports/tee.md | 13 ++++-- scripts/size-budgets.json | 6 +-- src/internal/callback.js | 50 +++++++++++++++++++++-- src/tee/index.d.ts | 13 +++--- src/tee/tee-transport.js | 80 +++++++++++++++++++++++++++---------- tests/unit/privacy.test.js | 26 ++++++++++++ tests/unit/tee.test.js | 82 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 236 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e7e62..b6b8fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,8 +102,9 @@ additive but the defaults listed below. `reserve`, `complete` and `release`. - **`./hooks`:** `beforeSend` may replace or refuse a message, `afterSend`, `onRetry` and `onError` hear each send, with a context made for each. -- **`./tee`:** mirrors each message to other transports, each with a timeout of its own and not - awaited by default, the primary's receipt the one returned. +- **`./tee`:** mirrors each message to other transports, with one Message-ID, each with a + timeout of its own and not awaited by default, the primary's receipt the one returned; + disposal waits for the copies in flight. - **Delivery events and `./webhooks`:** the `DeliveryEvent` model at the root, one event per message and recipient, and a verifier per provider for SES, Resend, SendGrid, Mailgun, Postmark, MailerSend, Mailtrap, Brevo, Mailjet, SparkPost, Lettermint, Maileroo, SMTP2GO, Azure diff --git a/docs/transports/tee.md b/docs/transports/tee.md index dc2acac..dd9643b 100644 --- a/docs/transports/tee.md +++ b/docs/transports/tee.md @@ -19,17 +19,22 @@ const transport = new TeeTransport(ses, { ## Mirrors never hold the primary -Each mirror's send has a timeout of its own (`timeout`, 10 seconds), and by -default it is not awaited: the primary's receipt comes back as soon as the +Each mirror's send has a timeout of its own (`timeout`, 10 seconds), which ends +it whether or not the mirror heeds its signal, and by default it is not +awaited: the primary's receipt comes back as soon as the primary answers, and the copies go on alone, not stopped by the caller's signal. At most `maxPending` copies are in flight; one over that is skipped and reported. On a runtime that freezes after the response, such as a serverless function, set `awaitMirrors`: the copies are then awaited, bounded by the -timeout, and follow the caller's signal. +timeout, and follow the caller's signal. Disposing of the tee waits for the +copies in flight before it disposes of the mirrors. The primary and the mirrors +send one Message-ID, so a copy reads as the message it is. `onMirror` hears the outcome of every copy: its `receipt`, the `error` its send threw, or why it was `skipped`. Without it, a failed or skipped copy becomes a -process warning (`MAILER_TEE_MIRROR_FAILED`). +process warning (`MAILER_TEE_MIRROR_FAILED`), with the addresses of its text +redacted, and no more than ten a minute: the first after a minute says how many +were left out. ## Options diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 122a956..a4314ae 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -30,9 +30,9 @@ "./mock": 2, "./otel": 5, "./logger": 2.5, - "./breaker": 2, + "./breaker": 2.5, "./ratelimit": 2.5, "./idempotency": 3, - "./hooks": 2, - "./tee": 2 + "./hooks": 2.5, + "./tee": 2.5 } diff --git a/src/internal/callback.js b/src/internal/callback.js index 2b8bdd4..0bcb5b1 100644 --- a/src/internal/callback.js +++ b/src/internal/callback.js @@ -2,6 +2,50 @@ // hook, a store) that throws: the send and its receipt go on, and the failure // becomes a process warning rather than vanishing, as a logger's does. +const { redactAddresses } = require('./redact-addresses.js'); + +// A listener or a mirror that fails for every message would fill the output +// of the process: no more than BURST warnings of a code go out a minute, and +// the first after it says how many were left out. +const BURST = 10; +const WINDOW_MS = 60 * 1000; +const periods = new Map(); + +/** + * Emits a process warning, at most `BURST` of a code a minute. + * + * @param code The code of the warning. + * @param text What it says. + * @internal + */ +const warnLimited = (code, text) => { + const now = performance.now(); + let period = periods.get(code); + let said = text; + if (period === undefined || now - period.start >= WINDOW_MS) { + if (period !== undefined && period.skipped > 0) { + said += ` (${period.skipped} more warnings of ${code} were left out in the last minute)`; + } + period = { start: now, count: 0, skipped: 0 }; + periods.set(code, period); + } + if (period.count >= BURST) { + period.skipped++; + return; + } + period.count++; + process.emitWarning(said, { code }); +}; + +/** + * The text of what was thrown, with the addresses in it redacted: an SMTP + * rejection or a store's error names a recipient, which no warning should. + * + * @internal + */ +const describeThrown = (error) => + redactAddresses(error instanceof Error ? error.message : String(error)); + /** * Emits a process warning for a function of the application that threw. * @@ -11,9 +55,7 @@ * @internal */ const warnThrown = (what, error, code) => { - process.emitWarning(`${what} threw: ${error instanceof Error ? error.message : String(error)}`, { - code, - }); + warnLimited(code, `${what} threw: ${describeThrown(error)}`); }; -module.exports = { warnThrown }; +module.exports = { describeThrown, warnLimited, warnThrown }; diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts index 5c793c0..d8b0d5a 100644 --- a/src/tee/index.d.ts +++ b/src/tee/index.d.ts @@ -35,7 +35,8 @@ export interface TeeConfig { /** The transports each message is mirrored to. */ readonly mirrors: readonly Transport[]; /** - * The time a mirror's send may take, in milliseconds. + * The time a mirror's send may take, in milliseconds: its signal aborts + * then, and a send that ignores it fails all the same. * @default 10000 */ readonly timeout?: number; @@ -54,8 +55,9 @@ export interface TeeConfig { readonly maxPending?: number; /** * Hears every outcome of a mirror. Without it, a failure or a skipped copy - * becomes a process warning (`MAILER_TEE_MIRROR_FAILED`); one that throws - * becomes one too (`MAILER_TEE_LISTENER_FAILED`). + * becomes a process warning (`MAILER_TEE_MIRROR_FAILED`), with the + * addresses of its text redacted and no more than ten a minute; one that + * throws becomes one too (`MAILER_TEE_LISTENER_FAILED`). */ readonly onMirror?: (outcome: MirrorOutcome) => void; } @@ -117,8 +119,9 @@ export declare class TeeTransport /** Verifies the primary; the mirrors are not checked. */ verify(options?: TransportOptions): Promise; /** - * Disposes of the primary and the mirrors, each once, whatever fails; the - * failures reject together as an `AggregateError`. + * Waits for the copies in flight, each bounded by `timeout`, then disposes + * of the primary and the mirrors, each once, whatever fails; the failures + * reject together as an `AggregateError`. */ [Symbol.asyncDispose](): Promise; } diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 9f12c69..c41b45d 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -5,11 +5,15 @@ // Changed in the port: a mirror has a timeout of its own and, by default, its // send is not awaited, where unemail awaited every leg with no bound, so a // slow or hung mirror held the primary's receipt; with `awaitMirrors` they -// are awaited, bounded by the timeout; and the detached sends in flight are -// bounded too. +// are awaited, bounded by the timeout; the detached sends in flight are +// bounded too, and disposal waits for them; the primary and the mirrors send +// one Message-ID; and a failure without `onMirror` is a warning with no +// address in it. +const { pinMessageId } = require('../core/message-id.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); -const { warnThrown } = require('../internal/callback.js'); +const { describeThrown, warnLimited, warnThrown } = require('../internal/callback.js'); +const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseRaw } = require('../internal/capabilities.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createTeeConfig } = require('./config.js'); @@ -22,10 +26,23 @@ const disposeOf = async (transport) => { const describe = (outcome) => { if (outcome.skipped !== undefined) return outcome.skipped; - if (outcome.error !== undefined) { - return outcome.error instanceof Error ? outcome.error.message : String(outcome.error); - } - return outcome.receipt.errorMessages[0]; + if (outcome.error !== undefined) return describeThrown(outcome.error); + return redactAddresses(outcome.receipt.errorMessages[0] ?? 'no error message'); +}; + +// A send of a mirror that does not settle by `timeout`, as one that ignores +// its signal would not, fails then all the same. +const withDeadline = (sending, timeout, mirror) => { + let timer; + const deadline = new Promise((resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`The mirror ${mirror} did not answer within ${timeout} ms.`)), + timeout, + ); + // A detached send does not keep the process alive; disposal waits for it. + timer.unref?.(); + }); + return Promise.race([sending, deadline]).finally(() => clearTimeout(timer)); }; /** @@ -37,6 +54,7 @@ class TeeTransport { config; primary; #pending = 0; + #detached = new Set(); /** * @param primary The transport whose receipt is returned. @@ -58,8 +76,10 @@ class TeeTransport { */ async send(message, options) { options?.signal?.throwIfAborted(); - const mirrored = this.#mirror(message, options?.signal, 'send'); - const receipt = await this.primary.send(message, options); + // The copies are of one message, so they carry one Message-ID. + const pinned = pinMessageId(message); + const mirrored = this.#mirror(pinned, options?.signal, 'send'); + const receipt = await this.primary.send(pinned, options); await mirrored; return receipt; } @@ -84,7 +104,10 @@ class TeeTransport { */ sendMany(messages, options) { return gatedSendMany(this.primary, messages, options, { - admit: (message) => ({ message, mirrored: this.#mirror(message, options?.signal, 'send') }), + admit: (message) => { + const pinned = pinMessageId(message); + return { message: pinned, mirrored: this.#mirror(pinned, options?.signal, 'send') }; + }, settle: async (receipt, admitted) => { await admitted.mirrored; return receipt; @@ -99,10 +122,12 @@ class TeeTransport { } /** - * Disposes of the primary and the mirrors, each once, all of them whatever + * Waits for the mirrored sends in flight, each bounded by `timeout`, then + * disposes of the primary and the mirrors, each once, all of them whatever * fails; the failures reject together. */ async [Symbol.asyncDispose]() { + await Promise.allSettled(this.#detached); const seen = new Set(); const errors = []; const disposeOnce = async (transport) => { @@ -148,15 +173,27 @@ class TeeTransport { const mirrorSignal = detached || signal === undefined ? timeout : AbortSignal.any([signal, timeout]); if (detached) this.#pending++; - try { - const receipt = await (operation === 'sendRaw' + const sending = withDeadline( + operation === 'sendRaw' ? mirror.sendRaw(message, { signal: mirrorSignal }) - : mirror.send(message, { signal: mirrorSignal })); - this.#report({ mirror: mirror.id, message, receipt }); - } catch (error) { - this.#report({ mirror: mirror.id, message, error }); + : mirror.send(message, { signal: mirrorSignal }), + this.config.timeout, + mirror.id, + ); + const settled = sending.then( + (receipt) => this.#report({ mirror: mirror.id, message, receipt }), + (error) => this.#report({ mirror: mirror.id, message, error }), + ); + if (!detached) { + await settled; + return; + } + this.#detached.add(settled); + try { + await settled; } finally { - if (detached) this.#pending--; + this.#detached.delete(settled); + this.#pending--; } } @@ -166,9 +203,10 @@ class TeeTransport { const listener = this.config.onMirror; if (listener === undefined) { if (outcome.receipt?.successful !== true) { - process.emitWarning(`The mirror ${outcome.mirror} failed: ${describe(outcome)}`, { - code: 'MAILER_TEE_MIRROR_FAILED', - }); + warnLimited( + 'MAILER_TEE_MIRROR_FAILED', + `The mirror ${outcome.mirror} failed: ${describe(outcome)}`, + ); } return; } diff --git a/tests/unit/privacy.test.js b/tests/unit/privacy.test.js index 78726e6..9c30861 100644 --- a/tests/unit/privacy.test.js +++ b/tests/unit/privacy.test.js @@ -27,3 +27,29 @@ test('Privacy: redaction runs in linear time on a long text without an address', assert.strictEqual(redactAddresses('"@['.repeat(100_000)).length, 300_000); assert.ok(performance.now() - started < 1000); }); + +test('Privacy: the warning of a hook that threw names no address', async (t) => { + // Before: the message of what a hook, a listener or a store threw went into + // the process warning as it was. + const { createMessage } = require('../../index.js'); + const { HooksTransport } = require('../../src/hooks/index.js'); + const { fakeTransport } = require('../helpers/decorators/fake.js'); + const warnings = []; + t.mock.method(process, 'emitWarning', (text) => warnings.push(text)); + const hooks = new HooksTransport(fakeTransport(), { + afterSend: () => { + throw new Error('cannot record the delivery to carol@example.com'); + }, + }); + await hooks.send( + createMessage({ + from: 'a@example.com', + to: 'carol@example.com', + subject: 's', + content: { text: 'x' }, + }), + ); + assert.strictEqual(warnings.length, 1); + assert.ok(!warnings[0].includes('carol@example.com'), warnings[0]); + assert.match(warnings[0], /cannot record the delivery to/); +}); diff --git a/tests/unit/tee.test.js b/tests/unit/tee.test.js index 6d473d3..7265e0f 100644 --- a/tests/unit/tee.test.js +++ b/tests/unit/tee.test.js @@ -202,3 +202,85 @@ test('Tee: the options are checked', () => { assert.strictEqual(tee.config.awaitMirrors, false); assert.strictEqual(tee.config.maxPending, 100); }); + +test('Tee: a mirror that ignores its signal fails at its timeout all the same', async () => { + // Before: the timeout reached a mirror only through its signal, so one that + // ignored it held a slot of maxPending for ever. + const deaf = fakeTransport(() => new Promise(() => {})); + const outcomes = []; + const tee = new TeeTransport(fakeTransport(), { + mirrors: [deaf], + timeout: 20, + maxPending: 1, + onMirror: (outcome) => outcomes.push(outcome), + }); + await tee.send(message('one')); + await waitFor(() => outcomes.length === 1); + assert.match(outcomes[0].error.message, /did not answer within 20 ms/); + // The slot is free again: the next copy is sent, not skipped. + await tee.send(message('two')); + await waitFor(() => outcomes.length === 2); + assert.strictEqual(outcomes[1].skipped, undefined); +}); + +test('Tee: dispose waits for the copies in flight before disposing of the mirrors', async () => { + // Before: the mirrors were disposed of while their copies were being sent. + const order = []; + const slow = fakeTransport( + () => + new Promise((resolve) => + setTimeout(() => { + order.push('copied'); + resolve(sent('m')); + }, 30), + ), + { + overrides: { + async [Symbol.asyncDispose]() { + order.push('disposed'); + }, + }, + }, + ); + const tee = new TeeTransport(fakeTransport(), { mirrors: [slow], onMirror: () => {} }); + await tee.send(message()); + await tee[Symbol.asyncDispose](); + assert.deepStrictEqual(order, ['copied', 'disposed']); +}); + +test('Tee: the primary and the mirrors send one Message-ID', async () => { + // Before: each composed its own, so a copy could not be told for the message. + const primary = fakeTransport(); + const mirror = fakeTransport(); + const tee = new TeeTransport(primary, { mirrors: [mirror], awaitMirrors: true }); + await tee.send(message()); + await collect(tee.sendMany([message('b')])); + for (let i = 0; i < 2; i++) { + const id = primary.calls[i].message.messageId; + assert.match(id, /^[^<>]+@example\.com$/); + assert.strictEqual(mirror.calls[i].message.messageId, id); + } +}); + +test('Tee: a failed copy is a warning with no address, and a flood of them is cut', async (t) => { + // Before: the first error text of the receipt went out as it was, the + // recipient of an SMTP rejection in it, once for every message. + // An hour on, past the minute of the warnings of the tests before. + let time = performance.now() + 60 * 60 * 1000; + t.mock.method(performance, 'now', () => time); + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); + const rejecting = fakeTransport(() => ({ + ...failed('rejected'), + errorMessages: ['550 5.1.1 : Recipient address rejected'], + })); + const tee = new TeeTransport(fakeTransport(), { mirrors: [rejecting], awaitMirrors: true }); + for (let i = 0; i < 12; i++) await tee.send(message()); + assert.strictEqual(warnings.length, 10); + assert.ok(!warnings[0][0].includes('bob@example.com'), warnings[0][0]); + assert.match(warnings[0][0], /The mirror fake failed: 550 5\.1\.1/); + time += 60 * 1000; + await tee.send(message()); + assert.strictEqual(warnings.length, 11); + assert.match(warnings[10][0], /2 more warnings of MAILER_TEE_MIRROR_FAILED were left out/); +}); From 54777656c5aa96133504301e7b7d04eeb595eb66 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:31:25 +0300 Subject: [PATCH 060/143] fix(idempotency): a lease for a reservation, fenced by a token One ttlMs, a day by default, held a pending reservation as long as a done record, so a crash after reserve, or a consumer that left sendMany early, kept the key pending for a day, and every later send of the message was answered in_flight, of unknown delivery, which neither ./retry nor ./pool sends again. A reservation lasts leaseMs (10 minutes); the receipt is kept ttlMs. reserve gives a free key a token, and complete and release act only while that token holds it, so a send whose lease passed does not overwrite or free the reservation of the next. The receipts in_flight, aborted and store_failed had their category chosen by the words of their text, which holds the key: a key `order-timeout-1` made them timeouts, which a circuit breaker counts against the provider, as it counted store_failed, service-unavailable. They are `unknown`, store_failed still retryable. The memory store dropped the oldest keys whatever their state, a pending one included, which let a duplicate go; it drops expired keys, then the oldest done ones, never a pending one. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- docs/transports/idempotency.md | 146 ++++++++++++--- scripts/size-budgets.json | 2 +- src/idempotency/config.js | 8 +- src/idempotency/idempotency-transport.js | 57 ++++-- src/idempotency/index.d.ts | 72 ++++++-- src/idempotency/memory-store.js | 84 ++++++--- tests/types/idempotency.test-d.ts | 37 +++- tests/unit/idempotency.test.js | 215 ++++++++++++++++++++--- 9 files changed, 518 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b8fd6..dedf8fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,8 +98,10 @@ additive but the defaults listed below. refused. - **`./idempotency`:** a key reserved in a store before the send, so a repeat is answered with the first receipt, a send in flight as of unknown delivery, and the same key for another - message refused. `createMemoryIdempotencyStore` is the in-process store; a shared one implements - `reserve`, `complete` and `release`. + message refused. A reservation lasts `leaseMs` (10 minutes), so a key a crash left reserved is + free again, and its token fences it: only its owner completes or releases it; the receipt is + kept `ttlMs` (a day). `createMemoryIdempotencyStore` is the in-process store; a shared one + implements `reserve`, `complete` and `release` with the token. - **`./hooks`:** `beforeSend` may replace or refuse a message, `afterSend`, `onRetry` and `onError` hear each send, with a context made for each. - **`./tee`:** mirrors each message to other transports, with one Message-ID, each with a diff --git a/docs/transports/idempotency.md b/docs/transports/idempotency.md index 8b7081d..4d16a95 100644 --- a/docs/transports/idempotency.md +++ b/docs/transports/idempotency.md @@ -2,8 +2,8 @@ `IdempotencyTransport` sends a message with an idempotency key once. A later send under the same key answers with the receipt of the first, so a job that -runs twice, or a retry after a crash, does not send the message twice. It comes -from the idempotency middleware of unemail. +runs twice, or a retry after a timeout, does not send the message twice. It +comes from the idempotency middleware of unemail. ## Basic usage @@ -37,37 +37,137 @@ under one key, only one goes: | another message | refused as `validation` (code `idempotency.key_reused`) | A send that failed frees its key for another attempt; one of unknown delivery -(`deliveryUnknown`, or one the caller aborted after it began) keeps it, as the -message may have gone. A message is told from another by a fingerprint of its -addresses, subject, body, header fields, the names and types of its -attachments (not their bytes), its threading, schedule, tracking and metadata. -A message without a key goes as it is. +(`deliveryUnknown`, or one the caller aborted after it began, code +`idempotency.aborted`) keeps it, as the message may have gone. A message is +told from another by a fingerprint of its addresses, subject, body, header +fields, the names and types of its attachments (not their bytes), its +threading, schedule, tracking and metadata. A message without a key goes as it +is. + +A key is reserved for `leaseMs` (ten minutes), and the receipt of its send is +kept for `ttlMs` (a day). The lease bounds how long a process that crashed, or +a batch left before its end, holds a key: after it, a send under the key goes. +So the lease must be longer than a send takes with all its attempts, or a +second send may go while the first is still out; and a crash after the +provider took the message, before its receipt was kept, lets the message go +again once the lease has passed. The decorator sends at least once with +deduplication, not exactly once. When the store fails to reserve a key, that message alone fails, retryable (`idempotency.store_failed`); when it fails to keep the receipt of a send that went, the receipt still comes back and the failure becomes a process warning -(`MAILER_IDEMPOTENCY_STORE_FAILED`). +(`MAILER_IDEMPOTENCY_STORE_FAILED`). The receipts the decorator makes, except +`key_reused`, are of the category `unknown`, so [`./breaker`](/transports/breaker) +does not count them against the provider, whatever the key says. ## Stores The default store is in this process: it deduplicates across the transports given it, not across processes, and forgets on a restart. For several -processes, pass a store on a shared database whose `reserve` is atomic: +processes, pass a store on a shared database. It has three methods, each one +atomic step: + +| Method | Does | +| --- | --- | +| `reserve(key, fingerprint, leaseMs)` | When no record holds the key within its time, holds it `pending` for `leaseMs` under a new token that cannot be guessed and resolves to `{ token }`; otherwise resolves to `{ record }`, the `state`, `fingerprint` and, once `done`, `receipt` of what holds it. | +| `complete(key, token, { fingerprint, receipt }, ttlMs)` | Keeps the key `done` with the receipt for `ttlMs`, when `token` holds it or nothing does. | +| `release(key, token)` | Frees the key, when `token` holds it `pending`. | + +The token fences a reservation. Once the lease of a `pending` key has passed, +another `reserve` may take it under a new token; the first process, slow +rather than gone, then calls `complete` or `release` with its old token, and +the store must leave the new owner's key alone. A `reserve` that answers +anything but one of `{ token }` and `{ record }` fails the message as +`idempotency.store_failed`, as sending it could send it twice. + +On Redis 7 with node-redis 4, `SET NX GET` reserves in one command, and a +script checks the token before it writes: ```js +const { randomUUID } = require('node:crypto'); + +// Writes the done record unless a record of another token holds the key. +const COMPLETE = ` +local current = redis.call('GET', KEYS[1]) +if current and cjson.decode(current).token ~= ARGV[1] then return 0 end +redis.call('SET', KEYS[1], ARGV[2], 'PX', ARGV[3]) +return 1`; +// Deletes the key while ARGV[1] holds it pending. +const RELEASE = ` +local current = redis.call('GET', KEYS[1]) +if not current then return 0 end +local record = cjson.decode(current) +if record.token ~= ARGV[1] or record.state ~= 'pending' then return 0 end +return redis.call('DEL', KEYS[1])`; + const redisStore = { - // SET NX reserves the key; the record is JSON. - async reserve(key, fingerprint, ttlMs) { - const pending = JSON.stringify({ state: 'pending', fingerprint }); - const ok = await redis.set(key, pending, { NX: true, PX: ttlMs }); - if (ok) return undefined; - return JSON.parse(await redis.get(key)); + async reserve(key, fingerprint, leaseMs) { + const token = randomUUID(); + const pending = JSON.stringify({ state: 'pending', fingerprint, token }); + const current = await redis.set(key, pending, { NX: true, GET: true, PX: leaseMs }); + return current === null ? { token } : { record: JSON.parse(current) }; + }, + async complete(key, token, { fingerprint, receipt }, ttlMs) { + const done = JSON.stringify({ state: 'done', fingerprint, receipt, token }); + await redis.eval(COMPLETE, { keys: [key], arguments: [token, done, String(ttlMs)] }); + }, + async release(key, token) { + await redis.eval(RELEASE, { keys: [key], arguments: [token] }); + }, +}; +``` + +On MongoDB, the unique `_id` makes the reservation atomic: an upsert that +matches only an expired record collides with a live one. + +```js +const { randomUUID } = require('node:crypto'); + +const keys = db.collection('idempotency'); +// Removes expired records in the background; the filters do not wait for it. +await keys.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }); + +const DUPLICATE_KEY = 11000; + +const mongoStore = { + async reserve(key, fingerprint, leaseMs) { + for (;;) { + const token = randomUUID(); + const now = new Date(); + const expiresAt = new Date(now.getTime() + leaseMs); + try { + await keys.updateOne( + { _id: key, expiresAt: { $lte: now } }, + { $set: { state: 'pending', fingerprint, token, expiresAt }, $unset: { receipt: '' } }, + { upsert: true }, + ); + return { token }; + } catch (error) { + if (error.code !== DUPLICATE_KEY) throw error; + } + const held = await keys.findOne({ _id: key, expiresAt: { $gt: new Date() } }); + if (held !== null) { + const record = { state: held.state, fingerprint: held.fingerprint, receipt: held.receipt }; + return { record }; + } + // Freed or expired since the upsert: reserve it again. + } }, - async complete(key, { fingerprint, receipt }, ttlMs) { - await redis.set(key, JSON.stringify({ state: 'done', fingerprint, receipt }), { PX: ttlMs }); + async complete(key, token, { fingerprint, receipt }, ttlMs) { + try { + const expiresAt = new Date(Date.now() + ttlMs); + await keys.updateOne( + { _id: key, $or: [{ token }, { expiresAt: { $lte: new Date() } }] }, + { $set: { state: 'done', fingerprint, receipt, token, expiresAt } }, + { upsert: true }, + ); + } catch (error) { + // Another token holds the key. + if (error.code !== DUPLICATE_KEY) throw error; + } }, - async release(key) { - await redis.del(key); + async release(key, token) { + await keys.deleteOne({ _id: key, token, state: 'pending' }); }, }; ``` @@ -77,12 +177,16 @@ const redisStore = { | Option | Default | Description | | --- | --- | --- | | `store` | in-process | `reserve`, `complete` and `release` of the keys. | -| `ttlMs` | `86400000` | How long a key is kept. | +| `leaseMs` | `600000` | How long a key stays reserved for a send. | +| `ttlMs` | `86400000` | How long the receipt of a send is kept under its key. | | `key` | `message.idempotencyKey` | The key of a message. | | `namespace` | the wrapped transport's id | The prefix of the keys in the store. | `createMemoryIdempotencyStore({ maxEntries, now })` makes the in-process store, -which keeps at most `maxEntries` keys (100000), dropping the oldest first. +whose tokens come from `crypto.randomUUID()`. It keeps at most `maxEntries` +keys (100000): to make room, it drops the expired keys first, then the oldest +`done` ones, still within their time, so a message sent again under one of +them goes again. A `pending` key is never dropped before its lease ends. Put this decorator outside `./retry` and `./pool`, so that a message is reserved once for all their attempts. `sendMany()` goes through the `sendMany()` of the diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index a4314ae..5d4afb1 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -32,7 +32,7 @@ "./logger": 2.5, "./breaker": 2.5, "./ratelimit": 2.5, - "./idempotency": 3, + "./idempotency": 3.5, "./hooks": 2.5, "./tee": 2.5 } diff --git a/src/idempotency/config.js b/src/idempotency/config.js index dd975c0..df8b175 100644 --- a/src/idempotency/config.js +++ b/src/idempotency/config.js @@ -6,6 +6,11 @@ const { checkOptionalFunction, checkPositiveNumber } = require('../internal/opti const { createMemoryIdempotencyStore } = require('./memory-store.js'); const STORE_METHODS = ['reserve', 'complete', 'release']; +// Longer than a send with all its attempts takes, so that a second send does +// not find the key free while the first is still out; short enough that a +// key a crash left reserved is taken again within minutes. +const DEFAULT_LEASE_MS = 10 * 60 * 1000; +const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000; /** * Resolves the options of the idempotency decorator. @@ -29,7 +34,8 @@ const createIdempotencyConfig = (config = {}, transportId) => { } return { store, - ttlMs: checkPositiveNumber(config.ttlMs ?? 24 * 60 * 60 * 1000, 'ttlMs'), + leaseMs: checkPositiveNumber(config.leaseMs ?? DEFAULT_LEASE_MS, 'leaseMs'), + ttlMs: checkPositiveNumber(config.ttlMs ?? DEFAULT_TTL_MS, 'ttlMs'), key: checkOptionalFunction(config.key, 'key'), namespace, }; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index b34c050..f303b94 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -7,8 +7,10 @@ // before and wrote it after, and two concurrent sends both went; a key used // again for another message is refused, where unemail answered with the // first's receipt; a delivery of unknown outcome keeps the key, where unemail -// remembered successes only and a retry sent again; and a store that fails -// fails the message it was asked about alone. +// remembered successes only and a retry sent again; a reservation lasts a +// lease shorter than the time a record is kept, and only the token it was +// given completes or releases it; and a store that fails fails the message it +// was asked about alone. const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); @@ -20,6 +22,9 @@ const { fingerprintOf } = require('./fingerprint.js'); const STORE_FAILED = 'MAILER_IDEMPOTENCY_STORE_FAILED'; +const isToken = (value) => typeof value === 'string' && value !== ''; +const isRecord = (value) => typeof value === 'object' && value !== null; + const isAbortOf = (error, signal) => signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); @@ -41,7 +46,7 @@ class IdempotencyTransport { /** * @param transport The transport to guard. - * @param config `store`, `ttlMs`, `key` and `namespace`. + * @param config `store`, `leaseMs`, `ttlMs`, `key` and `namespace`. * @throws {TypeError} If an option is invalid. */ constructor(transport, config) { @@ -126,20 +131,28 @@ class IdempotencyTransport { return receipt; } - // Reserves the key of a message: `{ key, fingerprint }` to send under it, - // `{}` for a message without one, or `{ receipt }` to answer with. + // Reserves the key of a message: `{ key, fingerprint, token }` to send + // under it, `{}` for a message without one, or `{ receipt }` to answer with. async #reserve(message) { const key = this.config.key === undefined ? message.idempotencyKey : this.config.key(message); if (key === undefined || key === null || key === '') return {}; const storeKey = `${this.config.namespace}:${key}`; const fingerprint = fingerprintOf(message); - let record; + let reservation; try { - record = await this.config.store.reserve(storeKey, fingerprint, this.config.ttlMs); + reservation = await this.config.store.reserve(storeKey, fingerprint, this.config.leaseMs); } catch (error) { return { receipt: this.#storeFailure(error) }; } - if (record === undefined || record === null) return { key: storeKey, fingerprint }; + const token = reservation?.token; + const record = reservation?.record; + // Neither or both is a store that does not keep the contract; sending + // could send twice, so the message fails as when the store is down. + if (isToken(token) === isRecord(record)) { + const error = new TypeError('reserve must resolve to { token } or { record }.'); + return { receipt: this.#storeFailure(error) }; + } + if (isToken(token)) return { key: storeKey, fingerprint, token }; return { receipt: this.#replyOf(record, fingerprint, key) }; } @@ -156,18 +169,28 @@ class IdempotencyTransport { ); } if (record.state === 'done' && record.receipt !== undefined) return record.receipt; + // The category is given, as the words of the text, which hold the key, + // would otherwise choose one (a key `order-timeout-1` made it a timeout, + // which a circuit breaker counts against the provider). return createFailedReceipt( `A message with the idempotency key ${key} is being sent; this one was not sent again.`, - { provider: this.id, deliveryUnknown: true, code: 'idempotency.in_flight' }, + { + provider: this.id, + category: 'unknown', + deliveryUnknown: true, + code: 'idempotency.in_flight', + }, ); } + // Retryable, but not a failure of the provider, which a circuit breaker + // would count. #storeFailure(error) { return createFailedReceipt( `The idempotency store failed: ${error instanceof Error ? error.message : String(error)}`, { provider: this.id, - category: 'service-unavailable', + category: 'unknown', retryable: true, code: 'idempotency.store_failed', }, @@ -182,14 +205,15 @@ class IdempotencyTransport { if (receipt.successful || receipt.deliveryUnknown === true) { await this.config.store.complete( entry.key, + entry.token, { fingerprint: entry.fingerprint, receipt }, this.config.ttlMs, ); } else { - await this.config.store.release(entry.key); + await this.config.store.release(entry.key, entry.token); } } catch (error) { - // The receipt stands; the key stays reserved until it expires. + // The receipt stands; the key stays reserved until its lease ends. warnThrown('The idempotency store', error, STORE_FAILED); } } @@ -202,7 +226,12 @@ class IdempotencyTransport { if (isAbortOf(error, signal)) { const receipt = createFailedReceipt( 'The send was aborted after it began; whether the message went is not known.', - { provider: this.id, deliveryUnknown: true, code: 'idempotency.aborted' }, + { + provider: this.id, + category: 'unknown', + deliveryUnknown: true, + code: 'idempotency.aborted', + }, ); await this.#settle(entry, receipt); return; @@ -213,7 +242,7 @@ class IdempotencyTransport { async #release(entry) { if (entry.key === undefined) return; try { - await this.config.store.release(entry.key); + await this.config.store.release(entry.key, entry.token); } catch (error) { warnThrown('The idempotency store', error, STORE_FAILED); } diff --git a/src/idempotency/index.d.ts b/src/idempotency/index.d.ts index 0548bc8..fe680ec 100644 --- a/src/idempotency/index.d.ts +++ b/src/idempotency/index.d.ts @@ -26,28 +26,55 @@ export interface IdempotencyRecord { } /** - * Where idempotency keys are kept. A store shared by several processes, such - * as one on Redis or MongoDB, must make `reserve` atomic: of two calls for a - * free key, one reserves it and the other gets the record. + * What `reserve` resolves to: `{ token }` when the key was free and is now + * reserved under that token, or `{ record }` with what holds the key. Any + * other answer, both or neither, fails the message as + * `idempotency.store_failed`. + */ +export type IdempotencyReservation = + | { readonly token: string; readonly record?: undefined } + | { readonly record: IdempotencyRecord; readonly token?: undefined }; + +/** + * Where idempotency keys are kept. A reservation is fenced by its token: + * `reserve` gives a free key a new token, and `complete` and `release` act on + * the key only while that token holds it. Once the lease of a `pending` key + * has passed, another `reserve` may take it under a new token, and a late + * `complete` or `release` of the first then leaves it alone. + * + * A store shared by several processes, such as one on Redis or MongoDB, makes + * each method one atomic step: of two `reserve` calls for a free key, one + * reserves it and the other gets the record; `complete` and `release` check + * the token and write in the same step. */ export interface IdempotencyStore { /** - * Reserves a key for `ttlMs`: `undefined` (or `null`) when it was free and - * is now `pending`, or the record that holds it. + * Reserves a key: when no record holds it within its time, holds it + * `pending` for `leaseMs` under a new token that cannot be guessed, and + * resolves to `{ token }`; otherwise resolves to `{ record }`: its + * `state`, its `fingerprint`, and the `receipt` of a `done` one. */ reserve( key: string, fingerprint: string, - ttlMs: number, - ): IdempotencyRecord | undefined | null | Promise; - /** Keeps a key `done` with the receipt of its send, for `ttlMs`. */ + leaseMs: number, + ): IdempotencyReservation | Promise; + /** + * Keeps a key `done` with the receipt of its send, for `ttlMs`, when + * `token` holds it or nothing does; a key another token holds, `pending` or + * `done` within its time, is left alone. + */ complete( key: string, + token: string, record: { readonly fingerprint: string; readonly receipt: Receipt }, ttlMs: number, ): void | Promise; - /** Frees a key whose send failed and may be made again. */ - release(key: string): void | Promise; + /** + * Frees a key whose send failed and may be made again, when `token` holds + * it `pending`; anything else is left alone. + */ + release(key: string, token: string): void | Promise; } /** @@ -60,7 +87,14 @@ export interface IdempotencyConfig { */ readonly store?: IdempotencyStore; /** - * How long a key is kept, in milliseconds. + * How long a key stays reserved for a send, in milliseconds: longer than a + * send takes with all its attempts, as after it another send under the key + * may go. A key a crash left reserved is free again after it. + * @default 600000 + */ + readonly leaseMs?: number; + /** + * How long the receipt of a send is kept under its key, in milliseconds. * @default 86400000 */ readonly ttlMs?: number; @@ -82,6 +116,7 @@ export interface IdempotencyConfig { */ export interface ResolvedIdempotencyConfig { readonly store: IdempotencyStore; + readonly leaseMs: number; readonly ttlMs: number; readonly key?: (message: Message | RawMessage) => string | undefined; readonly namespace: string; @@ -92,7 +127,9 @@ export interface ResolvedIdempotencyConfig { */ export interface MemoryIdempotencyStoreOptions { /** - * The most keys kept, the oldest dropped first. + * The most keys kept. Expired keys are dropped first, then the oldest + * `done` ones, still within their time; a `pending` key is never dropped + * before its lease ends. * @default 100000 */ readonly maxEntries?: number; @@ -102,7 +139,9 @@ export interface MemoryIdempotencyStoreOptions { /** * Makes a store of idempotency keys in this process: shared by the transports - * given it, not by other processes, and lost on a restart. + * given it, not by other processes, and lost on a restart. Its tokens come + * from `crypto.randomUUID()`. Over `maxEntries` it drops `done` keys before + * their time, after which a message sent again under one goes again. * * @throws {TypeError} If an option is invalid. */ @@ -116,8 +155,11 @@ export declare function createMemoryIdempotencyStore( * the first, or, while the first is in flight, with a receipt of unknown * delivery (code `idempotency.in_flight`); the same key for another message * is refused (`idempotency.key_reused`). A send that failed frees its key, one - * of unknown delivery keeps it; a store that fails fails the message it was - * asked about alone (`idempotency.store_failed`, retryable). + * of unknown delivery keeps it; a key stays reserved for `leaseMs` at most, + * its receipt is kept for `ttlMs`. A store that fails fails the message it + * was asked about alone (`idempotency.store_failed`, retryable). The + * receipts it makes, except `key_reused` (`validation`), are of the category + * `unknown`, which a circuit breaker does not count against the provider. * * @example * ```js diff --git a/src/idempotency/memory-store.js b/src/idempotency/memory-store.js index dfe78e0..40cb30e 100644 --- a/src/idempotency/memory-store.js +++ b/src/idempotency/memory-store.js @@ -3,15 +3,25 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind // Changed in the port: a key is reserved before the send, so a second send // under it finds it taken, where unemail looked a key up and wrote it only -// after the send; a record keeps the fingerprint of the message; and the -// clock is injected. +// after the send; a reservation lasts its lease and carries a token, so that +// only its owner completes or releases it; a record keeps the fingerprint of +// the message; the clock is injected; and a pending key is never dropped to +// make room, where unemail dropped the oldest keys whatever their state. +const { randomUUID } = require('node:crypto'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); /** * Makes a store of idempotency keys in this process: shared by the - * transports given it, not by other processes, and lost on a restart. It - * holds at most `maxEntries` keys, dropping the oldest first. + * transports given it, not by other processes, and lost on a restart. A key + * is `pending` for the lease `reserve` gives it, under a token of its own, + * and `done` for the time `complete` gives it. + * + * It holds at most `maxEntries` keys. To make room it drops the keys whose + * time has passed, then the oldest `done` ones, while they are still within + * their time: a message sent again under a key so dropped goes again. A + * `pending` key is never dropped before its lease ends, so while more sends + * than `maxEntries` are in flight the store holds more keys. * * @param options `maxEntries` (100000) and `now`. * @returns The store. @@ -20,43 +30,59 @@ const { checkOptionalFunction, checkPositiveInteger } = require('../internal/opt const createMemoryIdempotencyStore = (options = {}) => { const maxEntries = checkPositiveInteger(options.maxEntries ?? 100000, 'maxEntries'); const now = checkOptionalFunction(options.now, 'now') ?? Date.now; - const entries = new Map(); + // A key is in one of the two. Each Map iterates in the order of insertion + // and an entry is written anew when it changes; as the keys of one state + // share a lease or a time, the first of each is also the first to expire. + const pending = new Map(); + const done = new Map(); + + const isFull = () => pending.size + done.size > maxEntries; - // A Map iterates in the order of insertion, and a record is written anew - // when it changes, so the first ones are the oldest. - const trim = () => { - const at = now(); + const dropExpired = (entries, at) => { for (const [key, entry] of entries) { - if (entries.size <= maxEntries && entry.expiresAt > at) break; + if (!isFull() || entry.expiresAt > at) return; entries.delete(key); } }; - const write = (key, entry) => { - entries.delete(key); + const trim = (at) => { + dropExpired(pending, at); + dropExpired(done, at); + for (const key of done.keys()) { + if (!isFull()) return; + done.delete(key); + } + }; + + const write = (entries, key, entry, at) => { + pending.delete(key); + done.delete(key); entries.set(key, entry); - if (entries.size > maxEntries) trim(); + if (isFull()) trim(at); }; + const entryOf = (key) => pending.get(key) ?? done.get(key); + return { - reserve(key, fingerprint, ttlMs) { - const entry = entries.get(key); - if (entry !== undefined && entry.expiresAt > now()) { - return { state: entry.state, fingerprint: entry.fingerprint, receipt: entry.receipt }; - } - write(key, { state: 'pending', fingerprint, receipt: undefined, expiresAt: now() + ttlMs }); - return undefined; + reserve(key, fingerprint, leaseMs) { + const at = now(); + const entry = entryOf(key); + if (entry !== undefined && entry.expiresAt > at) return { record: entry.record }; + const token = randomUUID(); + const record = { state: 'pending', fingerprint }; + write(pending, key, { record, token, expiresAt: at + leaseMs }, at); + return { token }; }, - complete(key, record, ttlMs) { - write(key, { - state: 'done', - fingerprint: record.fingerprint, - receipt: record.receipt, - expiresAt: now() + ttlMs, - }); + complete(key, token, record, ttlMs) { + const at = now(); + const entry = entryOf(key); + // Taken by another reservation once this one's lease passed: it is theirs. + if (entry !== undefined && entry.token !== token && entry.expiresAt > at) return; + const kept = { state: 'done', fingerprint: record.fingerprint, receipt: record.receipt }; + write(done, key, { record: kept, token, expiresAt: at + ttlMs }, at); }, - release(key) { - entries.delete(key); + release(key, token) { + if (pending.get(key)?.token === token) pending.delete(key); }, }; }; diff --git a/tests/types/idempotency.test-d.ts b/tests/types/idempotency.test-d.ts index 1d27dfb..a8db282 100644 --- a/tests/types/idempotency.test-d.ts +++ b/tests/types/idempotency.test-d.ts @@ -1,4 +1,4 @@ -import { expectAssignable, expectError, expectType } from 'tsd'; +import { expectAssignable, expectError, expectNotAssignable, expectType } from 'tsd'; import { createMessage, type RawTransport, @@ -10,28 +10,49 @@ import { IdempotencyTransport, createMemoryIdempotencyStore, type IdempotencyRecord, + type IdempotencyReservation, type IdempotencyStore, } from '../../src/idempotency/index.js'; declare const inner: Transport<'resend'>; const store: IdempotencyStore = createMemoryIdempotencyStore({ maxEntries: 10, now: Date.now }); const custom: IdempotencyStore = { - reserve: async (_key, fingerprint): Promise => - fingerprint === '' ? undefined : { state: 'pending', fingerprint }, - complete: async () => {}, - release: () => {}, + reserve: async (_key, fingerprint, leaseMs): Promise => + leaseMs > 0 ? { token: 't' } : { record: { state: 'pending', fingerprint } }, + complete: async (_key, token, record, ttlMs) => { + expectType(token); + expectType(record.receipt); + expectType(ttlMs); + }, + release: (_key, token) => { + expectType(token); + }, }; const once = new IdempotencyTransport(inner, { store: custom, + leaseMs: 30_000, ttlMs: 60_000, key: (message) => ('idempotencyKey' in message ? message.idempotencyKey : undefined), namespace: 'tenant-a', }); new IdempotencyTransport(inner, { store }); new IdempotencyTransport(inner); +expectType(once.config.leaseMs); +expectType(once.config.ttlMs); expectType<'resend'>(once.id); expectAssignable>(once); expectAssignable>(once); + +declare const reservation: IdempotencyReservation; +if (reservation.token === undefined) expectType(reservation.record); +else expectType(reservation.token); +expectNotAssignable({}); +expectNotAssignable(undefined); +expectNotAssignable({ + token: 't', + record: { state: 'done', fingerprint: 'f' }, +}); + const message = createMessage({ from: 'a@example.com', to: 'b@example.com', @@ -42,3 +63,9 @@ const message = createMessage({ expectType>>(once.send(message)); expectType>>(once.sendMany([message])); expectError(new IdempotencyTransport(inner, { store: {} })); +// A store of the earlier contract, which answered a free key with `undefined`. +expectError({ + reserve: async () => undefined, + complete: async () => {}, + release: () => {}, +}); diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js index 92f45ae..ac7a868 100644 --- a/tests/unit/idempotency.test.js +++ b/tests/unit/idempotency.test.js @@ -3,6 +3,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { createMessage } = require('../../index.js'); +const { CircuitBreakerTransport } = require('../../src/breaker/index.js'); const { IdempotencyTransport, createMemoryIdempotencyStore, @@ -143,7 +144,7 @@ test('Idempotency: an abort keeps the key as of unknown delivery, another throw ...createMemoryIdempotencyStore(), reserve: async () => { early.abort(new Error('changed my mind')); - return undefined; + return { token: 't' }; }, release: () => {}, }, @@ -169,10 +170,10 @@ test('Idempotency: a store that fails fails its message alone, or becomes a warn if (failing === 'complete') throw new Error('cannot write'); return memory.complete(...args); }, - release: (key) => { + release: (...args) => { // oxlint-disable-next-line no-throw-literal -- a store may throw anything if (failing === 'release') throw 'cannot delete'; - return memory.release(key); + return memory.release(...args); }, }; const answers = [ @@ -198,6 +199,127 @@ test('Idempotency: a store that fails fails its message alone, or becomes a warn ]); }); +test('Idempotency: a reserved key is free again after leaseMs, a receipt kept for ttlMs', async () => { + let time = 0; + const store = createMemoryIdempotencyStore({ now: () => time }); + const hung = deferred(); + const inner = fakeTransport((_m, index) => (index === 0 ? hung.promise : sent(`m${index}`))); + const once = new IdempotencyTransport(inner, { store, leaseMs: 1000, ttlMs: 5000 }); + // The first send does not settle, as after a crash: its key stays reserved + // for its lease, not for the day a receipt is kept. + const first = once.send(message({ idempotencyKey: 'k' })); + time = 999; + const during = await once.send(message({ idempotencyKey: 'k' })); + assert.strictEqual(during.errors[0].code, 'idempotency.in_flight'); + time = 1000; + assert.strictEqual((await once.send(message({ idempotencyKey: 'k' }))).messageId, 'm1'); + time = 5999; + assert.strictEqual((await once.send(message({ idempotencyKey: 'k' }))).messageId, 'm1'); + // The first ends at last, failed: its release does not free the receipt of + // the send that took its key. + hung.resolve(failed('validation', { retryable: false })); + assert.strictEqual((await first).successful, false); + assert.strictEqual((await once.send(message({ idempotencyKey: 'k' }))).messageId, 'm1'); + time = 6000; + assert.strictEqual((await once.send(message({ idempotencyKey: 'k' }))).messageId, 'm2'); + assert.strictEqual(inner.calls.length, 3); +}); + +test('Idempotency: a send that outlived its lease leaves the new owner of its key alone', async () => { + let time = 0; + const store = createMemoryIdempotencyStore({ now: () => time }); + const slow = deferred(); + const owner = deferred(); + const gates = [slow, owner]; + const inner = fakeTransport((_m, index) => gates[index]?.promise ?? sent(`m${index}`)); + const once = new IdempotencyTransport(inner, { store, leaseMs: 1000 }); + const late = once.send(message({ idempotencyKey: 'k' })); + time = 1000; + const taken = once.send(message({ idempotencyKey: 'k' })); + // Slow rather than gone, the first completes under its old token. + slow.resolve(sent('late')); + assert.strictEqual((await late).messageId, 'late'); + const during = await once.send(message({ idempotencyKey: 'k' })); + assert.strictEqual(during.errors[0].code, 'idempotency.in_flight'); + // The owner's failure frees the key for the next send. + owner.resolve(failed('validation', { retryable: false })); + assert.strictEqual((await taken).successful, false); + assert.strictEqual((await once.send(message({ idempotencyKey: 'k' }))).messageId, 'm2'); + assert.strictEqual(inner.calls.length, 3); +}); + +test('Idempotency: its receipts carry their own category, not one the words of a key suggest', async () => { + const gate = deferred(); + const inner = fakeTransport(() => gate.promise); + const once = new IdempotencyTransport(inner); + const breaker = new CircuitBreakerTransport(once, { failureThreshold: 1 }); + const first = once.send(message({ idempotencyKey: 'order-timeout-1' })); + const inFlight = await breaker.send(message({ idempotencyKey: 'order-timeout-1' })); + assert.strictEqual(inFlight.errors[0].code, 'idempotency.in_flight'); + assert.strictEqual(inFlight.errors[0].category, 'unknown'); + assert.strictEqual(inFlight.deliveryUnknown, true); + assert.strictEqual(inFlight.retryable, false); + gate.resolve(sent('m')); + await first; + const controller = new AbortController(); + const started = deferred(); + const stuck = fakeTransport((_m, _index, options) => { + started.resolve(); + return new Promise((_resolve, reject) => { + options.signal.addEventListener('abort', () => reject(options.signal.reason)); + }); + }); + const abortable = new IdempotencyTransport(stuck); + const aborted = abortable.send(message({ idempotencyKey: 'timeout' }), { + signal: controller.signal, + }); + await started.promise; + controller.abort(new Error('left')); + await assert.rejects(aborted, /left/); + const after = await abortable.send(message({ idempotencyKey: 'timeout' })); + assert.strictEqual(after.errors[0].code, 'idempotency.aborted'); + assert.strictEqual(after.errors[0].category, 'unknown'); + assert.strictEqual(after.deliveryUnknown, true); + // A store that fails is retried, but is not the provider failing. + const down = new IdempotencyTransport(inner, { + store: { + reserve: () => { + throw new Error('connection timeout'); + }, + complete() {}, + release() {}, + }, + }); + const guarded = new CircuitBreakerTransport(down, { failureThreshold: 1 }); + const failure = await guarded.send(message({ idempotencyKey: 'k' })); + assert.strictEqual(failure.errors[0].code, 'idempotency.store_failed'); + assert.strictEqual(failure.errors[0].category, 'unknown'); + assert.strictEqual(failure.errors[0].retryable, true); + assert.strictEqual(failure.retryable, true); + assert.strictEqual(breaker.state, 'closed'); + assert.strictEqual(guarded.state, 'closed'); +}); + +test('Idempotency: a reserve that answers neither or both of token and record fails its message', async () => { + const inner = fakeTransport(); + for (const answer of [ + undefined, + null, + {}, + { token: '' }, + { record: null }, + { token: 't', record: { state: 'done', fingerprint: 'f' } }, + ]) { + const store = { reserve: async () => answer, complete() {}, release() {} }; + const receipt = await new IdempotencyTransport(inner, { store }).send( + message({ idempotencyKey: 'k' }), + ); + assert.strictEqual(receipt.errors[0].code, 'idempotency.store_failed', String(answer)); + assert.match(receipt.errorMessages[0], /reserve must resolve to \{ token \} or \{ record \}/); + } + assert.strictEqual(inner.calls.length, 0); +}); + test('Idempotency: sendMany answers repeated keys in place, the batch too', async () => { const inner = fakeTransport(undefined, { readAhead: 1 }); const once = new IdempotencyTransport(inner); @@ -274,29 +396,80 @@ test('Idempotency: a key function, a namespace, sendRaw, verify and dispose', as assert.strictEqual(disposed, 1); assert.throws(() => new IdempotencyTransport(inner, { store: {} }), /store must have a reserve/); assert.throws(() => new IdempotencyTransport(inner, { ttlMs: 0 }), /ttlMs/); + assert.throws(() => new IdempotencyTransport(inner, { leaseMs: Infinity }), /leaseMs/); assert.throws(() => new IdempotencyTransport(inner, { namespace: 1 }), /namespace/); - assert.strictEqual(new IdempotencyTransport(inner).config.namespace, 'fake'); + const defaults = new IdempotencyTransport(inner).config; + assert.strictEqual(defaults.namespace, 'fake'); + assert.strictEqual(defaults.leaseMs, 10 * 60 * 1000); + assert.strictEqual(defaults.ttlMs, 24 * 60 * 60 * 1000); }); -test('Idempotency: the memory store forgets a key after its time, and the oldest first', () => { +test('Idempotency: the memory store reserves under a token, for a lease, then a time', () => { let time = 0; - const store = createMemoryIdempotencyStore({ maxEntries: 2, now: () => time }); - assert.strictEqual(store.reserve('a', 'fa', 100), undefined); + const store = createMemoryIdempotencyStore({ now: () => time }); + const { token } = store.reserve('a', 'fa', 100); + assert.match(token, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); assert.deepStrictEqual(store.reserve('a', 'fa', 100), { - state: 'pending', - fingerprint: 'fa', - receipt: undefined, + record: { state: 'pending', fingerprint: 'fa' }, }); - store.complete('a', { fingerprint: 'fa', receipt: sent('ra') }, 100); - assert.strictEqual(store.reserve('a', 'fa', 100).receipt.messageId, 'ra'); - time = 100; - assert.strictEqual(store.reserve('a', 'fa', 100), undefined); - store.reserve('b', 'fb', 1000); - store.reserve('c', 'fc', 1000); - // Three keys for two places: the oldest, a, goes. - assert.strictEqual(store.reserve('a', 'fa', 100), undefined); - assert.notStrictEqual(store.reserve('c', 'fc', 1000), undefined); - store.release('c'); - assert.strictEqual(store.reserve('c', 'fc', 1000), undefined); + store.complete('a', token, { fingerprint: 'fa', receipt: sent('ra') }, 1000); + // Kept for its time, not for the lease of its reservation. + time = 999; + assert.deepStrictEqual(store.reserve('a', 'fa', 100), { + record: { state: 'done', fingerprint: 'fa', receipt: sent('ra') }, + }); + time = 1000; + const again = store.reserve('a', 'fa', 100); + assert.notStrictEqual(again.token, token); + store.release('a', again.token); + assert.notStrictEqual(store.reserve('a', 'fa', 100).token, undefined); + // A key whose lease passed and that no one took is still completed. + time = 2000; + store.complete('b', 'unknown', { fingerprint: 'fb', receipt: sent('rb') }, 1000); + assert.strictEqual(store.reserve('b', 'fb', 100).record.receipt.messageId, 'rb'); assert.throws(() => createMemoryIdempotencyStore({ maxEntries: 0 }), /maxEntries/); }); + +test('Idempotency: a late complete or release under a stale token leaves the key alone', () => { + let time = 0; + const store = createMemoryIdempotencyStore({ now: () => time }); + const stale = store.reserve('k', 'f', 100).token; + time = 100; + const owner = store.reserve('k', 'f', 100).token; + store.complete('k', stale, { fingerprint: 'f', receipt: sent('stale') }, 1000); + store.release('k', stale); + assert.deepStrictEqual(store.reserve('k', 'f', 100), { + record: { state: 'pending', fingerprint: 'f' }, + }); + store.complete('k', owner, { fingerprint: 'f', receipt: sent('owner') }, 1000); + store.complete('k', stale, { fingerprint: 'f', receipt: sent('stale') }, 1000); + store.release('k', stale); + // A done key is not released, by its own token either. + store.release('k', owner); + assert.strictEqual(store.reserve('k', 'f', 100).record.receipt.messageId, 'owner'); +}); + +test('Idempotency: the memory store makes room with expired keys, then done ones, never pending', () => { + let time = 0; + const store = createMemoryIdempotencyStore({ maxEntries: 2, now: () => time }); + store.reserve('pending', 'fp', 1000); + const { token } = store.reserve('done', 'fd', 1000); + store.complete('done', token, { fingerprint: 'fd', receipt: sent('rd') }, 5000); + // Three keys for two places: the done one goes, though the pending one is + // older; unemail dropped the pending one, and its message could go twice. + store.reserve('new', 'fn', 1000); + const pending = { record: { state: 'pending', fingerprint: 'fp' } }; + assert.deepStrictEqual(store.reserve('pending', 'fp', 1000), pending); + // While more sends than maxEntries are in flight, it holds every key. + assert.notStrictEqual(store.reserve('done', 'fd', 1000).token, undefined); + assert.deepStrictEqual(store.reserve('pending', 'fp', 1000), pending); + assert.strictEqual(store.reserve('new', 'fn', 1000).record.state, 'pending'); + // Expired keys go before a done one still within its time. + const kept = createMemoryIdempotencyStore({ maxEntries: 2, now: () => time }); + kept.reserve('old', 'fo', 100); + const keptToken = kept.reserve('kept', 'fk', 1000).token; + kept.complete('kept', keptToken, { fingerprint: 'fk', receipt: sent('rk') }, 5000); + time = 100; + kept.reserve('third', 'ft', 1000); + assert.strictEqual(kept.reserve('kept', 'fk', 1000).record.receipt.messageId, 'rk'); +}); From 00c350deb13c1dde63f4e336d8988aabef4e1147 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:32:49 +0300 Subject: [PATCH 061/143] fix: an unsupported category, which ./pool fails over with A message a transport cannot send as it is, a field it cannot express or a limit of its provider (a second tag on Postmark, more recipients than Mailjet takes, scheduledAt on SMTP, a message over the SMTP server's SIZE), was refused as validation, or rejected, and ./pool stops at a refused message, so a pool lost a message another of its transports would have sent. The refusal of scheduledAt was a validation failure only by the word "Invalid" in its text. The new category `unsupported` is not retried and does not count against the provider in ./breaker, but ./pool fails over with it; validation and rejected keep stopping it, for a message no transport would take. refusalFailure, refuseScheduledAt, the JMAP and SMTP refusals and a raw message that is not UTF-8 for a JSON API give it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 19 +++-- docs/messages/compose.md | 4 +- docs/transports/azure.md | 2 +- docs/transports/cloudflare.md | 4 +- docs/transports/custom.md | 27 ++++++ docs/transports/lettermint.md | 2 +- docs/transports/mailjet.md | 4 +- docs/transports/mailtrap.md | 2 +- docs/transports/plunk.md | 4 +- docs/transports/pool.md | 9 +- docs/transports/postmark.md | 4 +- docs/transports/scaleway.md | 2 +- docs/transports/sendmail.md | 2 +- docs/transports/ses.md | 2 +- docs/transports/smtp.md | 4 +- docs/transports/sparkpost.md | 2 +- docs/transports/zeptomail.md | 2 +- index.d.ts | 17 +++- scripts/size-budgets.json | 6 +- src/cloudflare/message-converter.js | 7 +- src/internal/raw.js | 8 +- src/internal/refusal.js | 8 +- src/internal/schedule.js | 14 ++-- src/jmap/jmap-transport.js | 2 +- src/lettermint/lettermint-transport.js | 7 +- src/lettermint/message-converter.js | 5 +- src/mailtrap/mailtrap-transport.js | 6 +- src/plunk/message-converter.js | 3 +- src/plunk/plunk-transport.js | 6 +- src/pool/index.d.ts | 5 ++ src/pool/pool-transport.js | 6 +- src/sendmail/sendmail-transport.js | 3 + src/smtp/smtp-transport.js | 8 +- tests/types/index.test-d.ts | 12 +++ tests/unit/api.test.js | 9 +- tests/unit/azure.test.js | 7 +- tests/unit/cloudflare.test.js | 10 ++- tests/unit/decorators.test.js | 75 +++++++++++++++++ tests/unit/lettermint-transport.test.js | 4 + tests/unit/mailjet.test.js | 7 +- tests/unit/message-fields.test.js | 83 ++++++++++++++++--- tests/unit/postmark.test.js | 6 +- tests/unit/scaleway.test.js | 7 +- tests/unit/sendmail.test.js | 4 +- tests/unit/smtp-transport.integration.test.js | 4 +- tests/unit/sparkpost.test.js | 2 +- tests/unit/zeptomail.test.js | 7 +- 47 files changed, 344 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dedf8fc..dc46def 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,11 +36,13 @@ additive but the defaults listed below. difference. - **The default DKIM `h=` signs `List-Unsubscribe` and `List-Unsubscribe-Post`** too, as RFC 8058 asks. -- **Failures are classified more precisely:** a message a provider's limits refuse before any - request is a non-retryable `validation` failure, where it was `unknown`; and a TLS certificate - refused in the handshake is a non-retryable `configuration` failure at the first attempt, where - SMTP reported it as `unknown` and the HTTP transports retried it and reported a retryable - `network` failure (`deliveryUnknown` on Deno). +- **Failures are classified more precisely:** a message a transport cannot send as it is (a + provider's limit, a field it cannot express, `scheduledAt` where nothing schedules, a message + over the SMTP server's `SIZE`) is a non-retryable failure of the new category `unsupported`, + where it was `unknown`, `validation` or `rejected`, and `./pool` fails over with it to a + transport that can send it; and a TLS certificate refused in the handshake is a non-retryable + `configuration` failure at the first attempt, where SMTP reported it as `unknown` and the HTTP + transports retried it and reported a retryable `network` failure (`deliveryUnknown` on Deno). - **On Deno, give the HTTP transports a `fetch` that keeps no idle connection** (`Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`): Deno's own sends a request again when a reused connection closes before its answer. See @@ -57,8 +59,8 @@ additive but the defaults listed below. whitespace, controls or angle brackets is refused, and one-click needs an `https` URL. - **`scheduledAt`, `tracking` and `metadata` on the message,** from unemail's model: when the provider should deliver it, open and click tracking for this message, and values the provider - returns in its events. A transport that cannot schedule refuses the message with a - `validation` receipt rather than sending it at once. Resend, SendGrid, Mailgun, Maileroo, + returns in its events. A transport that cannot schedule refuses the message with an + `unsupported` receipt rather than sending it at once, and `./pool` fails over with it. Resend, SendGrid, Mailgun, Maileroo, Brevo, MailerSend, SparkPost and SMTP2GO schedule; ten providers take `tracking`, seventeen `metadata` (each page says how). - **`./attachments`:** `fromPath`, `fromUrl`, `fromDataUri`, `fromStream` and `inlineDataUrls`, @@ -149,7 +151,8 @@ additive but the defaults listed below. for an untrusted CA. It is now a non-retryable `configuration` failure at the first attempt, on every transport and runtime. - **A message refused by a provider's limits before any request was `unknown`;** it is a - non-retryable `validation` failure. + non-retryable `unsupported` failure, which `./pool` fails over with, as a message no transport + could send (`validation`, `rejected`) stops it. - **A pooled SMTP connection was reused for any number of messages,** while servers limit the messages of one connection (`maxMessages`, above). diff --git a/docs/messages/compose.md b/docs/messages/compose.md index 61d975b..6bf1585 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -259,8 +259,8 @@ const message = createMessage({ - `scheduledAt` is when the provider should deliver the message. A transport that cannot schedule it, SMTP and JMAP among them, refuses the message with a - non-retryable `validation` receipt rather than sending it at once, so `./pool` - does not fail over with it and `./retry` does not retry it. + non-retryable `unsupported` receipt rather than sending it at once, so `./pool` + fails over to a transport that can schedule and `./retry` does not retry it. - `tracking` turns open and click tracking on or off for this message, over the configuration of the transport, where the provider takes it per message. - `metadata` holds values the provider keeps with the message and returns in its diff --git a/docs/transports/azure.md b/docs/transports/azure.md index cf568a1..74e9b32 100644 --- a/docs/transports/azure.md +++ b/docs/transports/azure.md @@ -66,7 +66,7 @@ the environment: pass it, or `endpoint` and `accessKey`, as options. resource has on but not the reverse. `tracking` sets it; a message that asks for opens without clicks, or the reverse, is refused. - Azure has no tags, which are not sent, as over SMTP; it cannot schedule a - message, so `scheduledAt` is refused as `validation`. + message, so `scheduledAt` is refused as `unsupported`. - An inline attachment goes with its Content-ID as `contentId`. - A request may be at most 10 MB, its attachments in base64 included; a larger one is refused before it is sent. diff --git a/docs/transports/cloudflare.md b/docs/transports/cloudflare.md index 6c1b63e..387db30 100644 --- a/docs/transports/cloudflare.md +++ b/docs/transports/cloudflare.md @@ -60,7 +60,7 @@ if (receipt.successful) console.log(receipt.messageId); // the message_id lists, and answers another with a 400. - Cloudflare has no tags or per-message tracking, which are not sent, as over SMTP; it cannot schedule a message, so `scheduledAt` is refused as - `validation`. + `unsupported`. - A message has at most 50 recipients across `to`, `cc` and `bcc`; more are refused before the request. @@ -77,7 +77,7 @@ When none was accepted it fails as `rejected`, not retryable. A 200 with `sendRaw()` sends a ready MIME message through `send_raw`, routed by its envelope. The API takes it as a JSON string, so its bytes must be UTF-8; others, -and an envelope without a sender, are refused as `validation` before any +and an envelope without a sender, are refused as `unsupported` before any request. ## Batches diff --git a/docs/transports/custom.md b/docs/transports/custom.md index dbb2b9e..9e73c72 100644 --- a/docs/transports/custom.md +++ b/docs/transports/custom.md @@ -353,6 +353,33 @@ async send(message) { This approach provides predictable error handling and allows users to handle errors consistently across all transports. +### Classify a failure + +The `category` of a failure tells the decorators what to do with it, so give it +with `createFailedReceipt` or `createReceiptError` of the root entry, together +with `retryable`, rather than leaving both to the words of the message: + +| Category | Meaning | `./pool` | +| --- | --- | --- | +| `validation`, `rejected` | The message itself is refused: an invalid address, a suppressed recipient. No transport would take it. | stops | +| `unsupported` | This transport cannot send the message as it is: a field it cannot express, such as `scheduledAt` or a second tag, or a limit of its provider, such as recipients or size. Another transport may. | fails over | +| `auth`, `configuration` | The transport is set up wrongly. | fails over | +| `rate-limit`, `network`, `timeout`, `server-error`, `service-unavailable` | The provider cannot take it now; usually retryable. | fails over | +| `unknown` | Anything else. | fails over | + +```js +if (message.scheduledAt != null) { + return createFailedReceipt('My service cannot schedule a message.', { + provider: this.id, + category: 'unsupported', + retryable: false, + }); +} +``` + +`./retry` retries what is `retryable`, and `./breaker` counts only `network`, +`timeout`, `server-error` and `service-unavailable` against the provider. + ### Use the platform, add no dependency @alexify/mailer runs on Node.js 22 or newer and has no runtime dependencies. A diff --git a/docs/transports/lettermint.md b/docs/transports/lettermint.md index da2fb93..6bf1d56 100644 --- a/docs/transports/lettermint.md +++ b/docs/transports/lettermint.md @@ -101,7 +101,7 @@ sending the message. A message's own `metadata` goes in `metadata` too, its keys over those of the transport, and its `tracking` sets `track_opens` and `track_clicks` over the transport's `settings`. Lettermint cannot schedule a message, so `scheduledAt` -is refused as `validation`. +is refused as `unsupported`, as is a second tag. ## Batch sending diff --git a/docs/transports/mailjet.md b/docs/transports/mailjet.md index 72a1160..1e04793 100644 --- a/docs/transports/mailjet.md +++ b/docs/transports/mailjet.md @@ -61,8 +61,8 @@ if (receipt.successful) console.log(receipt.messageId); // the MessageUUID `List-Id`, `X-Mailer`, `Date` and others are refused before the request, so a message with `list.id` cannot go through Mailjet. - Mailjet takes one reply-to address, a subject of at most 255 characters and 50 - recipients a message; it cannot schedule a message. Each is refused with a - failed receipt. + recipients a message; it cannot schedule a message. Each is refused with an + `unsupported` receipt, which `./pool` fails over with. ## Batches and errors diff --git a/docs/transports/mailtrap.md b/docs/transports/mailtrap.md index 14200ec..e93a743 100644 --- a/docs/transports/mailtrap.md +++ b/docs/transports/mailtrap.md @@ -123,7 +123,7 @@ const message = createMessage({ A message's own `metadata` goes in `custom_variables` too, its keys over those of the configuration. Mailtrap cannot schedule a message, so `scheduledAt` is -refused as `validation`, and it sets tracking per domain, so a message's +refused as `unsupported`, and it sets tracking per domain, so a message's `tracking` is not sent. ## Batch sending diff --git a/docs/transports/plunk.md b/docs/transports/plunk.md index ad482f5..9a29415 100644 --- a/docs/transports/plunk.md +++ b/docs/transports/plunk.md @@ -157,8 +157,8 @@ const transport = new PlunkTransport({ apiKey: process.env.PLUNK_API_KEY, concur ## Scheduling, metadata and tracking -Plunk takes none of the three. A message with `scheduledAt` is refused as a -`validation` failure rather than sent at once, and its `tracking` and +Plunk takes none of the three. A message with `scheduledAt` is refused as an +`unsupported` failure rather than sent at once, and its `tracking` and `metadata` are not sent, as over SMTP. ## Advanced features and reliability diff --git a/docs/transports/pool.md b/docs/transports/pool.md index b188f10..443b0a2 100644 --- a/docs/transports/pool.md +++ b/docs/transports/pool.md @@ -243,8 +243,13 @@ providers. Structured errors use each child transport's provider `id`, so fallback failures remain tied to the transport that produced them. Failover moves on after a failure of the transport, such as an authentication -error, a rate limit or an outage, and stops in two cases where another -transport could not help: +error, a rate limit or an outage, and after a message the transport cannot send +as it is, an error of category `unsupported`: a field it cannot express, such +as `scheduledAt` on SMTP or a second tag on Postmark, or a limit of its +provider, such as recipients, attachments or the server's SIZE. A pool of SMTP +and Resend sends a scheduled message through Resend whichever comes first; when +every transport refuses it so, the receipt is `unsupported` and not retryable. +Failover stops in two cases where another transport could not help: - **The message itself was refused**, an error of category `validation` or `rejected`: another provider would refuse it too. The receipt is not diff --git a/docs/transports/postmark.md b/docs/transports/postmark.md index 6a2d4a4..bb9ea71 100644 --- a/docs/transports/postmark.md +++ b/docs/transports/postmark.md @@ -57,8 +57,8 @@ if (receipt.successful) console.log(receipt.messageId); // Postmark's MessageID - List-* fields, priority and threading go in `Headers`. - An inline attachment goes with `cid:` before its Content-ID, as Postmark wants it. -- Postmark cannot schedule a message, so `scheduledAt` is refused with a - `validation` receipt, and it has no idempotency key. +- Postmark cannot schedule a message, so `scheduledAt` is refused with an + `unsupported` receipt, as is a second tag, and it has no idempotency key. ## Errors diff --git a/docs/transports/scaleway.md b/docs/transports/scaleway.md index 1f02d1d..0fecff8 100644 --- a/docs/transports/scaleway.md +++ b/docs/transports/scaleway.md @@ -57,7 +57,7 @@ if (receipt.successful) console.log(receipt.messageId); // the id of the email header fields, List-* and the threading fields. - Scaleway has no tags or per-message tracking, which are not sent, as over SMTP. It cannot schedule a message, so `scheduledAt` is refused as - `validation`. + `unsupported`. - It has no Content-ID for an attachment, so an inline one is refused rather than sent as a file the HTML cannot refer to. - A request may be at most 2 MB, its attachments in base64 included; a larger diff --git a/docs/transports/sendmail.md b/docs/transports/sendmail.md index 87831dd..38039c2 100644 --- a/docs/transports/sendmail.md +++ b/docs/transports/sendmail.md @@ -69,7 +69,7 @@ wrote to stderr (its first 4 KiB, addresses replaced by `[REDACTED]`): An abort of the `signal` kills the process. A message whose content cannot be read (an attachment that fails) kills it with SIGINT, so that no part of it is -delivered. `scheduledAt` is refused, as sendmail cannot schedule. +delivered. `scheduledAt` is refused as `unsupported`, as sendmail cannot schedule. ## Raw messages and verification diff --git a/docs/transports/ses.md b/docs/transports/ses.md index 9bcb02b..5908024 100644 --- a/docs/transports/ses.md +++ b/docs/transports/ses.md @@ -346,7 +346,7 @@ same name. A key and its value must then be ASCII letters, digits, `_` or `-`, fewer than 256 characters, and a key neither `category` nor `priority`; any other is refused before the request. Tracking is set by the configuration set, so a message's `tracking` is not sent, and SES cannot schedule a message, so -`scheduledAt` is refused as `validation`. +`scheduledAt` is refused as `unsupported`. ## Advanced configuration and reliability diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index 9b00030..d7bb0ef 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -282,7 +282,9 @@ server advertises it. The transport adds the encoded message size in octets to uploaded. If the server advertises a fixed maximum, @alexify/mailer returns a failed receipt without -sending `MAIL FROM` when the message exceeds that limit. A bare `SIZE` +sending `MAIL FROM` when the message exceeds that limit, of category `unsupported` +(code `smtp.message-size-exceeded`): not retried, and failed over by +[`./pool`](/transports/pool) to a transport whose limit the message is under. A bare `SIZE` capability, or `SIZE 0`, means that no fixed maximum was advertised, so @alexify/mailer still declares the message size without rejecting it locally. Servers that do not advertise `SIZE` retain the existing SMTP flow. diff --git a/docs/transports/sparkpost.md b/docs/transports/sparkpost.md index 2e192de..5b0b28d 100644 --- a/docs/transports/sparkpost.md +++ b/docs/transports/sparkpost.md @@ -77,7 +77,7 @@ spent sending limit a `rate-limit` one. `sendRaw()` sends a ready MIME message as `email_rfc822`, to the recipients of its envelope. The API takes it as a JSON string, so its bytes must be UTF-8; others -are refused as `validation`, before any request. +are refused as `unsupported`, before any request. ```js const receipt = await transport.sendRaw({ diff --git a/docs/transports/zeptomail.md b/docs/transports/zeptomail.md index 499230e..fa8b358 100644 --- a/docs/transports/zeptomail.md +++ b/docs/transports/zeptomail.md @@ -62,7 +62,7 @@ if (receipt.successful) console.log(receipt.messageId); // the request_id - ZeptoMail takes 500 addresses in each of `to`, `cc` and `bcc`, 60 attachments and a subject of 500 characters; more are refused before the request. It cannot schedule a message, so `scheduledAt` is refused as - `validation`. + `unsupported`. ZeptoMail answers 201 with a `request_id`, the receipt's id; an answer without one is reported as `deliveryUnknown`. The code in the details of an error says diff --git a/index.d.ts b/index.d.ts index 6086c99..f9d4252 100644 --- a/index.d.ts +++ b/index.d.ts @@ -489,8 +489,9 @@ export interface Message { readonly calendar?: CalendarContent; /** * When the provider should deliver the message. A transport that cannot - * schedule it refuses the message with a non-retryable `validation` - * receipt rather than sending it at once. + * schedule it refuses the message with a non-retryable `unsupported` + * receipt rather than sending it at once, and `./pool` fails over to one + * that can. */ readonly scheduledAt?: Date; /** @@ -680,8 +681,8 @@ export interface MessageConstructor { /** * When the provider should deliver the message. A transport that cannot * schedule it (SMTP, JMAP, and every provider without a scheduling API) - * refuses the message with a non-retryable `validation` receipt rather - * than sending it at once. + * refuses the message with a non-retryable `unsupported` receipt rather + * than sending it at once, and `./pool` fails over to one that can. */ readonly scheduledAt?: Date; /** @@ -993,6 +994,13 @@ export declare function comparePriority(a: Priority, b: Priority): number; /** * A machine-readable category for an email delivery failure. + * + * `validation` and `rejected` say that the message itself was refused: no + * transport would take it, so `./pool` stops failing over. `unsupported` + * says that this transport cannot send the message as it is, a field it + * cannot express (`scheduledAt`, a second tag) or a limit of its provider + * (recipients, attachments): another transport may, so `./pool` fails over + * with it. Neither is retried. */ export type ReceiptErrorCategory = | 'auth' @@ -1001,6 +1009,7 @@ export type ReceiptErrorCategory = | 'timeout' | 'validation' | 'rejected' + | 'unsupported' | 'server-error' | 'service-unavailable' | 'configuration' diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 5d4afb1..6ca8438 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -11,9 +11,9 @@ "./resend": 6.5, "./sendgrid": 6.5, "./mailgun": 6.5, - "./mailtrap": 6.5, + "./mailtrap": 7, "./maileroo": 6.5, - "./lettermint": 6, + "./lettermint": 6.5, "./plunk": 6, "./postmark": 7, "./brevo": 7, @@ -23,7 +23,7 @@ "./zeptomail": 7, "./smtp2go": 7.5, "./azure": 8, - "./scaleway": 7, + "./scaleway": 7.5, "./cloudflare": 8, "./pool": 3, "./retry": 3, diff --git a/src/cloudflare/message-converter.js b/src/cloudflare/message-converter.js index 2de0c75..e528cc3 100644 --- a/src/cloudflare/message-converter.js +++ b/src/cloudflare/message-converter.js @@ -11,7 +11,6 @@ const { toAsciiAddress } = require('../core/address.js'); const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); const { rawMessageText } = require('../internal/raw.js'); -const { RawMessageValidationError } = require('../core/raw-message.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -103,12 +102,12 @@ const convertMessage = async (message, signal) => { * * @param raw The envelope and bytes. * @returns The request. - * @throws {RawMessageValidationError} If the envelope has no sender, or the - * bytes are not UTF-8. + * @throws {RefusalError} If the envelope has no sender, or the bytes are not + * UTF-8. */ const convertRawMessage = (raw) => { if (raw.envelope.from === null) { - throw new RawMessageValidationError('Cloudflare needs a sender in the envelope.', 'envelope'); + throw new RefusalError('Cloudflare needs a sender in the envelope.'); } return { from: raw.envelope.from, diff --git a/src/internal/raw.js b/src/internal/raw.js index 80fb3be..73080f3 100644 --- a/src/internal/raw.js +++ b/src/internal/raw.js @@ -9,6 +9,7 @@ const { const { toAsciiAddress } = require('../core/address.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { joinChunks } = require('./body.js'); +const { RefusalError } = require('./refusal.js'); const UTF8 = new TextDecoder('utf-8', { fatal: true }); @@ -54,19 +55,20 @@ const rawMessageFailure = (provider, error) => : undefined; /** - * The text of a raw message, for an API that takes it as a JSON string. + * The text of a raw message, for an API that takes it as a JSON string. Bytes + * that are not UTF-8 are refused as `unsupported`: SMTP sends them as they are. * * @param bytes The bytes of the message. * @param label The provider's name, for the error. * @returns The text. - * @throws {RawMessageValidationError} If the bytes are not UTF-8. + * @throws {RefusalError} If the bytes are not UTF-8. * @internal */ const rawMessageText = (bytes, label) => { try { return UTF8.decode(bytes); } catch { - throw new RawMessageValidationError(`${label} takes a raw message only as UTF-8.`, 'content'); + throw new RefusalError(`${label} takes a raw message only as UTF-8.`); } }; diff --git a/src/internal/refusal.js b/src/internal/refusal.js index 3468a1b..edf420b 100644 --- a/src/internal/refusal.js +++ b/src/internal/refusal.js @@ -15,9 +15,9 @@ const { isDeliveryUnknown } = require('./delivery-unknown.js'); class RefusalError extends RangeError {} /** - * The receipt of a refusal before any request, a non-retryable `validation` - * failure, as for a message a transport cannot schedule; or `undefined` for - * another error. + * The receipt of a refusal before any request, a non-retryable `unsupported` + * failure: this transport cannot send the message as it is, and another one + * may, so `./pool` fails over with it; or `undefined` for another error. * * @param provider The transport. * @param error What was thrown. @@ -26,7 +26,7 @@ class RefusalError extends RangeError {} */ const refusalFailure = (provider, error) => error instanceof RefusalError - ? createFailedReceipt(error.message, { provider, category: 'validation', retryable: false }) + ? createFailedReceipt(error.message, { provider, category: 'unsupported', retryable: false }) : undefined; /** diff --git a/src/internal/schedule.js b/src/internal/schedule.js index 24009ae..dd29405 100644 --- a/src/internal/schedule.js +++ b/src/internal/schedule.js @@ -3,17 +3,19 @@ // take rather than dropping it (src/core/define.ts); a message scheduled for // tomorrow and sent now is the worse of the two failures. +const { RefusalError } = require('./refusal.js'); + /** - * The reason a transport gives for a message it cannot schedule. "Invalid" - * makes the receipt a non-retryable validation failure, so `./pool` does not - * fail over with it and `./retry` does not retry it. + * The reason a transport gives for a message it cannot schedule. Its receipt + * is a non-retryable `unsupported` failure, so `./retry` does not retry it and + * `./pool` fails over to a transport that can schedule. * * @param provider The transport that refuses it. * @returns The error message. * @internal */ const schedulingRefusal = (provider) => - `Invalid scheduledAt: ${provider} cannot schedule a message, so it is refused rather than sent at once.`; + `Unsupported scheduledAt: ${provider} cannot schedule a message, so it is refused rather than sent at once.`; /** * Refuses a message that asks for a later delivery its transport cannot @@ -21,11 +23,11 @@ const schedulingRefusal = (provider) => * * @param message The message. * @param provider The transport that would send it. - * @throws {TypeError} If the message carries `scheduledAt`. + * @throws {RefusalError} If the message carries `scheduledAt`. * @internal */ const refuseScheduledAt = (message, provider) => { - if (message.scheduledAt != null) throw new TypeError(schedulingRefusal(provider)); + if (message.scheduledAt != null) throw new RefusalError(schedulingRefusal(provider)); }; module.exports = { refuseScheduledAt, schedulingRefusal }; diff --git a/src/jmap/jmap-transport.js b/src/jmap/jmap-transport.js index 3025917..2b02878 100644 --- a/src/jmap/jmap-transport.js +++ b/src/jmap/jmap-transport.js @@ -987,7 +987,7 @@ const executeSubmission = async (client, apiUrl, request, signal) => { // this transport does not ask for. const refuseScheduled = () => createJmapFailure(schedulingRefusal('jmap'), undefined, { - category: 'validation', + category: 'unsupported', code: 'jmap.scheduling_unsupported', retryable: false, }); diff --git a/src/lettermint/lettermint-transport.js b/src/lettermint/lettermint-transport.js index d147fb8..a41b74a 100644 --- a/src/lettermint/lettermint-transport.js +++ b/src/lettermint/lettermint-transport.js @@ -11,7 +11,7 @@ const { const { convertMessage, generateIdempotencyKey } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); -const { certificateFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -245,8 +245,9 @@ const responseToReceipt = (response) => { }; const createLettermintFailure = (message, error) => { - const certificate = certificateFailure('lettermint', message, error); - if (certificate !== undefined) return certificate; + const known = + refusalFailure('lettermint', error) ?? certificateFailure('lettermint', message, error); + if (known !== undefined) return known; if (error instanceof LettermintApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/lettermint/message-converter.js b/src/lettermint/message-converter.js index 3212505..89fb19e 100644 --- a/src/lettermint/message-converter.js +++ b/src/lettermint/message-converter.js @@ -6,6 +6,7 @@ const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); const { convertAttachments, formatAddresses, @@ -37,12 +38,12 @@ const PRIORITIES = { high: '1', normal: '3', low: '5' }; * @param config The resolved Lettermint configuration. * @returns JSON object ready for Lettermint API submission. * @throws {TypeError} If the message carries an invalid message identifier. - * @throws {RangeError} If the message has more than one tag. + * @throws {RefusalError} If the message has more than one tag, or `scheduledAt`. */ const convertMessage = async (message, config, signal) => { refuseScheduledAt(message, 'lettermint'); if (message.tags.length > 1) { - throw new RangeError('Lettermint supports at most one tag per message.'); + throw new RefusalError('Lettermint supports at most one tag per message.'); } const attachments = withCalendarAttachment(message); diff --git a/src/mailtrap/mailtrap-transport.js b/src/mailtrap/mailtrap-transport.js index 0e4ac8d..ff9455b 100644 --- a/src/mailtrap/mailtrap-transport.js +++ b/src/mailtrap/mailtrap-transport.js @@ -12,7 +12,7 @@ const { const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); -const { certificateFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -251,8 +251,8 @@ const toReceipt = (response, options) => { }; const createMailtrapFailure = (message, error) => { - const certificate = certificateFailure('mailtrap', message, error); - if (certificate !== undefined) return certificate; + const known = refusalFailure('mailtrap', error) ?? certificateFailure('mailtrap', message, error); + if (known !== undefined) return known; if (error instanceof MailtrapResponseError) { return createFailedReceipt(message, { deliveryUnknown: true, diff --git a/src/plunk/message-converter.js b/src/plunk/message-converter.js index 00b728f..3fd6937 100644 --- a/src/plunk/message-converter.js +++ b/src/plunk/message-converter.js @@ -6,6 +6,7 @@ const { readAttachmentContent } = require('../core/attachment.js'); const { toBase64 } = require('../internal/base64.js'); const { collectCustomHeaders, withCalendarAttachment } = require('../internal/provider.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { RefusalError } = require('../internal/refusal.js'); /** * Converts a Upyo message to Plunk API format. @@ -76,7 +77,7 @@ const convertMessage = async (message, _config, signal) => { // invitation occupies a slot, going over the limit would silently drop one of // the caller's own files, so it is refused instead. if (message.calendar != null && sourceAttachments.length > 5) { - throw new RangeError( + throw new RefusalError( 'Plunk supports at most five attachments, and a calendar message needs ' + 'one of them for the invitation.', ); diff --git a/src/plunk/plunk-transport.js b/src/plunk/plunk-transport.js index 53770eb..22bb202 100644 --- a/src/plunk/plunk-transport.js +++ b/src/plunk/plunk-transport.js @@ -9,7 +9,7 @@ const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); -const { certificateFailure } = require('../internal/refusal.js'); +const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Plunk transport implementation for sending emails via Plunk API. @@ -226,8 +226,8 @@ class PlunkTransport { } const createPlunkFailure = (message, error) => { - const certificate = certificateFailure('plunk', message, error); - if (certificate !== undefined) return certificate; + const known = refusalFailure('plunk', error) ?? certificateFailure('plunk', message, error); + if (known !== undefined) return known; if (error instanceof PlunkApiError) { return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), diff --git a/src/pool/index.d.ts b/src/pool/index.d.ts index 9ac03d4..119ab13 100644 --- a/src/pool/index.d.ts +++ b/src/pool/index.d.ts @@ -164,6 +164,11 @@ export interface ResolvedTransportEntry { * replacement for any single transport. It distributes messages across multiple * underlying transports based on the configured strategy. * + * It fails over after a failure of a transport (an outage, a rate limit) or of + * what it can express (`unsupported`: `scheduledAt` on SMTP, a limit of the + * provider), and stops at a refused message (`validation`, `rejected`) or an + * unknown delivery. + * * @example Round-robin load balancing * ```typescript * import { PoolTransport } from "@alexify/mailer/pool"; diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 75041a9..229c140 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -168,7 +168,8 @@ class PoolTransport { const attemptedIndices = new Set(); // Failover stops when another transport could only deliver the message a // second time (delivery unknown) or would refuse it too (the message - // itself was rejected). + // itself was rejected). A message one transport cannot send as it is + // (`unsupported`) goes on to the next, which may. const failure = { errorMessages: [], errors: [], @@ -499,7 +500,8 @@ const isAbortError = (error) => { }; // A rejection of the message itself, as opposed to one of the transport -// (authentication, rate limit, outage), which another transport may not share. +// (authentication, rate limit, outage) or of what it can express +// (`unsupported`), which another transport may not share. const hasMessageRejection = (errors) => { if (errors == null) return false; for (const error of errors) { diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index eb8473a..aee6df0 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -22,6 +22,7 @@ const { RawMessageValidationError, } = require('../core/raw-message.js'); const { createFailedReceipt } = require('../core/receipt.js'); +const { refusalFailure } = require('../internal/refusal.js'); const { createLog } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); @@ -52,6 +53,8 @@ const isCallerAbort = (error, signal) => signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); const failureOf = (error) => { + const refusal = refusalFailure('sendmail', error); + if (refusal !== undefined) return refusal; const message = error instanceof Error ? error.message : String(error); if (error instanceof RawMessageValidationError) { return createFailedReceipt(message, { provider: 'sendmail', category: 'validation' }); diff --git a/src/smtp/smtp-transport.js b/src/smtp/smtp-transport.js index 859acd3..bf6c1f0 100644 --- a/src/smtp/smtp-transport.js +++ b/src/smtp/smtp-transport.js @@ -34,6 +34,7 @@ const { parseEnhancedSmtpStatusCode } = require('./smtp-status-code.js'); const { TLSSocket } = require('node:tls'); const { warnDegradation } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); +const { RefusalError } = require('../internal/refusal.js'); /** * SMTP transport implementation for sending emails via SMTP protocol. @@ -841,8 +842,11 @@ const createSmtpFailure = (message, error) => { }; // Failures found before or without a reply of the server, which no retry -// fixes: the code and category of each, and what the receipt tells of it. +// fixes: the code and category of each, and what the receipt tells of it. A +// message over the server's SIZE, or one it cannot schedule, is `unsupported`: +// another transport of a pool may send it. const LOCAL_FAILURES = [ + { type: RefusalError, code: 'smtp.unsupported', category: 'unsupported' }, { type: Smtp8BitMimeUnsupportedError, code: 'smtp.8bitmime-unsupported', @@ -860,7 +864,7 @@ const LOCAL_FAILURES = [ { type: SmtpMessageSizeError, code: 'smtp.message-size-exceeded', - category: 'rejected', + category: 'unsupported', details: (error) => ({ actualSize: error.actualSize, maximumSize: error.maximumSize.toString(), diff --git a/tests/types/index.test-d.ts b/tests/types/index.test-d.ts index dee4def..ad34b29 100644 --- a/tests/types/index.test-d.ts +++ b/tests/types/index.test-d.ts @@ -19,6 +19,7 @@ import { type DeliveryEvent, type DeliveryEventType, type ReceiptError, + type ReceiptErrorCategory, type RetryEvent, type Transport, type VerifiableTransport, @@ -119,6 +120,17 @@ const unknownDelivery = createFailedReceipt('Connection closed after DATA', { if (!unknownDelivery.successful) { expectType(unknownDelivery.deliveryUnknown); } +// A message this transport cannot send as it is, which a pool fails over with. +const unsupported = createFailedReceipt('Postmark takes one tag per message.', { + provider: 'postmark', + category: 'unsupported', + retryable: false, +}); +if (!unsupported.successful) { + expectAssignable(unsupported.errors?.[0]?.category); +} +expectAssignable('unsupported'); +expectError(createReceiptError('x', { category: 'not-supported' })); const transport: Transport<'smtp'> = { id: 'smtp', diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js index d4bb7f1..03b87ae 100644 --- a/tests/unit/api.test.js +++ b/tests/unit/api.test.js @@ -134,14 +134,15 @@ test('API: the receipt of a failure carries what the error says', () => { assert.strictEqual(checkCredential('k', 'apiKey', 'DemoTransport'), 'k'); }); -test('API: a refusal of the converter is a non-retryable validation failure', () => { +test('API: a refusal of the converter is a non-retryable unsupported failure', () => { const refused = providerFailure('x', new RefusalError('X takes at most 5 tags.')); - assert.strictEqual(refused.errors[0].category, 'validation'); + assert.strictEqual(refused.errors[0].category, 'unsupported'); + assert.strictEqual(refused.errors[0].code, 'unsupported'); assert.strictEqual(refused.retryable, false); assert.strictEqual(refused.errorMessages[0], 'X takes at most 5 tags.'); const other = providerFailure('x', new Error('boom')); - assert.notStrictEqual(other.errors[0].category, 'validation'); + assert.notStrictEqual(other.errors[0].category, 'unsupported'); // Another RangeError, such as a body over its limit, keeps its own receipt. const range = providerFailure('x', new RangeError('The body is over 4 MiB.')); - assert.notStrictEqual(range.errors[0].category, 'validation'); + assert.notStrictEqual(range.errors[0].category, 'unsupported'); }); diff --git a/tests/unit/azure.test.js b/tests/unit/azure.test.js index 8f967cf..798ecd3 100644 --- a/tests/unit/azure.test.js +++ b/tests/unit/azure.test.js @@ -222,7 +222,7 @@ test('Azure: tracking, size and a schedule are refused before a request', async const requests = stubFetch(t, () => accepted()); const refusals = [ [message({ tracking: { opens: true, clicks: false } }), /on or off together/], - [message({ scheduledAt: new Date(NOW + 1000) }), /Invalid scheduledAt: azure/], + [message({ scheduledAt: new Date(NOW + 1000) }), /Unsupported scheduledAt: azure/], [ message({ attachments: { @@ -237,7 +237,10 @@ test('Azure: tracking, size and a schedule are refused before a request', async ], ]; for (const [refused, error] of refusals) { - assert.match((await transport().send(refused)).errorMessages[0], error); + const receipt = await transport().send(refused); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); } assert.strictEqual(requests.length, 0); }); diff --git a/tests/unit/cloudflare.test.js b/tests/unit/cloudflare.test.js index 0832f25..808f0a6 100644 --- a/tests/unit/cloudflare.test.js +++ b/tests/unit/cloudflare.test.js @@ -187,10 +187,13 @@ test('Cloudflare: its limits and a schedule are refused before a request', async const refusals = [ [message({ to: many, cc: many }), /at most 50 recipients/], [message({ replyTo: ['a@example.com', 'b@example.com'] }), /one reply-to/], - [message({ scheduledAt: new Date() }), /Invalid scheduledAt: cloudflare/], + [message({ scheduledAt: new Date() }), /Unsupported scheduledAt: cloudflare/], ]; for (const [refused, error] of refusals) { - assert.match((await transport().send(refused)).errorMessages[0], error); + const receipt = await transport().send(refused); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); } assert.strictEqual(requests.length, 0); }); @@ -213,13 +216,14 @@ test('Cloudflare: sendRaw sends the MIME text through send_raw', async (t) => { envelope: { from: null, to: ['bob@example.com'] }, content: new TextEncoder().encode(text), }); - assert.strictEqual(bounce.errors[0].category, 'validation'); + assert.strictEqual(bounce.errors[0].category, 'unsupported'); assert.match(bounce.errorMessages[0], /needs a sender in the envelope/); const latin1 = await transport().sendRaw({ envelope: { from: 'shop@example.com', to: ['bob@example.com'] }, content: new Uint8Array([...new TextEncoder().encode('Subject: x\r\n\r\n'), 0xe9, 13, 10]), }); assert.match(latin1.errorMessages[0], /Cloudflare takes a raw message only as UTF-8/); + assert.strictEqual(latin1.errors[0].category, 'unsupported'); stubFetch(t, () => json({ success: false, errors: [{ code: 10001 }] }, 400)); const failed = await transport().sendRaw({ envelope: { from: 'shop@example.com', to: ['bob@example.com'] }, diff --git a/tests/unit/decorators.test.js b/tests/unit/decorators.test.js index 35c735b..25254f3 100644 --- a/tests/unit/decorators.test.js +++ b/tests/unit/decorators.test.js @@ -6,7 +6,10 @@ const { CircuitBreakerTransport } = require('../../src/breaker/index.js'); const { HooksTransport } = require('../../src/hooks/index.js'); const { IdempotencyTransport } = require('../../src/idempotency/index.js'); const { PoolTransport } = require('../../src/pool/index.js'); +const { PostmarkTransport } = require('../../src/postmark/index.js'); +const { RetryTransport } = require('../../src/retry/index.js'); const { failed, fakeTransport, sent } = require('../helpers/decorators/fake.js'); +const { json, stubFetch } = require('../helpers/providers/fetch.js'); const message = (fields = {}) => createMessage({ @@ -61,3 +64,75 @@ test('Decorators: idempotency outside a pool reserves a message once for all its assert.strictEqual(steady.calls.length, 1); assert.deepStrictEqual(heard, ['s0']); }); + +const postmark = () => new PostmarkTransport({ serverToken: 'server-token', retries: 0 }); + +test('Decorators: a pool fails over from a transport that cannot send the message as it is', async (t) => { + const requests = stubFetch(t, () => json({ MessageID: 'pm1', ErrorCode: 0 })); + const other = fakeTransport((_m, index) => sent(`other${index}`)); + other.id = 'other'; + const pool = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: postmark(), priority: 2 }, + { transport: other, priority: 1 }, + ], + }); + // Postmark cannot schedule a message nor take two tags: the other sends both. + const scheduled = await pool.send(message({ scheduledAt: new Date(Date.now() + 3600_000) })); + const tagged = await pool.send(message({ tags: ['welcome', 'onboarding'] })); + assert.deepStrictEqual( + [scheduled.successful, scheduled.messageId, tagged.successful, tagged.messageId], + [true, 'other0', true, 'other1'], + ); + assert.strictEqual(other.calls.length, 2); + assert.strictEqual(requests.length, 0); + // A message Postmark can send goes through it. + assert.strictEqual((await pool.send(message())).messageId, 'pm1'); + + // When every transport refuses it so, the receipt is unsupported, not retried. + const refusing = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: postmark(), priority: 2 }, + { transport: postmark(), priority: 1 }, + ], + }); + const refused = await refusing.send(message({ tags: ['welcome', 'onboarding'] })); + assert.strictEqual(refused.successful, false); + assert.strictEqual(refused.retryable, false); + assert.strictEqual(refused.attempts, 2); + assert.deepStrictEqual( + refused.errors.map((error) => [error.provider, error.category, error.retryable]), + [ + ['postmark', 'unsupported', false], + ['postmark', 'unsupported', false], + ], + ); + assert.strictEqual(requests.length, 1); +}); + +test('Decorators: retry does not retry an unsupported message, nor does a breaker count it', async (t) => { + const requests = stubFetch(t, () => json({ MessageID: 'pm1', ErrorCode: 0 })); + const inner = postmark(); + let attempts = 0; + const counted = { + id: 'postmark', + send: (sentMessage, options) => { + attempts++; + return inner.send(sentMessage, options); + }, + async *sendMany() {}, + }; + const retry = new RetryTransport(counted, { maxAttempts: 3, initialDelay: 1 }); + const receipt = await retry.send(message({ tags: ['welcome', 'onboarding'] })); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(attempts, 1); + + const breaker = new CircuitBreakerTransport(postmark(), { failureThreshold: 1 }); + const refused = await breaker.send(message({ scheduledAt: new Date(Date.now() + 3600_000) })); + assert.strictEqual(refused.errors[0].category, 'unsupported'); + assert.strictEqual(breaker.state, 'closed'); + assert.strictEqual((await breaker.send(message())).successful, true); + assert.strictEqual(requests.length, 1); +}); diff --git a/tests/unit/lettermint-transport.test.js b/tests/unit/lettermint-transport.test.js index 9e6f280..deb2c2b 100644 --- a/tests/unit/lettermint-transport.test.js +++ b/tests/unit/lettermint-transport.test.js @@ -822,6 +822,10 @@ describe('LettermintTransport - sendMany', { concurrency: false }, () => { assert.deepEqual(receipts[1].errorMessages, [ 'Lettermint supports at most one tag per message.', ]); + // Changed after the port: a refusal another transport may not + // share is `unsupported`, which a pool fails over with. + assert.equal(receipts[1].errors[0].category, 'unsupported'); + assert.equal(receipts[1].retryable, false); } }, ); diff --git a/tests/unit/mailjet.test.js b/tests/unit/mailjet.test.js index 6fa6430..cde05b7 100644 --- a/tests/unit/mailjet.test.js +++ b/tests/unit/mailjet.test.js @@ -185,10 +185,13 @@ test('Mailjet: what Mailjet sets itself, and its limits, are refused before a re [message({ headers: { 'X-Mailer': 'Mine 1.0' } }), /sets X-Mailer itself/], [message({ subject: 'x'.repeat(256) }), /subject of at most 255/], [message({ replyTo: ['a@example.com', 'b@example.com'] }), /one reply-to/], - [message({ scheduledAt: new Date() }), /Invalid scheduledAt: mailjet/], + [message({ scheduledAt: new Date() }), /Unsupported scheduledAt: mailjet/], ]; for (const [refused, error] of refusals) { - assert.match((await transport().send(refused)).errorMessages[0], error); + const receipt = await transport().send(refused); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); } assert.strictEqual(requests.length, 0); }); diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index 71eb029..be71f7d 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -1,6 +1,7 @@ // Not ported: `scheduledAt`, `tracking` and `metadata` of a message (from // unemail), checked by createMessage, and the refusal of a scheduled message by -// every transport that cannot schedule it, before anything is sent. +// every transport that cannot schedule it, before anything is sent, as +// `unsupported`, which `./pool` fails over with. const { test } = require('node:test'); const assert = require('node:assert'); const { createMessage } = require('../../index.js'); @@ -76,8 +77,8 @@ test('Fields: an invalid scheduledAt, tracking or metadata is refused', () => { const assertRefused = (receipt) => { assert.strictEqual(receipt.successful, false); assert.strictEqual(receipt.retryable, false); - assert.strictEqual(receipt.errors[0].category, 'validation'); - assert.match(receipt.errorMessages[0], /Invalid scheduledAt: \w+ cannot schedule a message/); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.match(receipt.errorMessages[0], /Unsupported scheduledAt: \w+ cannot schedule a message/); }; test('Fields: SMTP refuses a scheduled message before it connects', async () => { @@ -126,6 +127,50 @@ for (const [name, make] of Object.entries(providers)) { }); } +const INVITATION = [ + 'BEGIN:VCALENDAR', + 'VERSION:2.0', + 'METHOD:REQUEST', + 'BEGIN:VEVENT', + 'UID:1@example.com', + 'SUMMARY:Lunch', + 'END:VEVENT', + 'END:VCALENDAR', + '', +].join('\r\n'); + +test('Fields: what a provider cannot carry is unsupported: a second Lettermint tag, a sixth Plunk file', async (t) => { + const fetch = t.mock.method(globalThis, 'fetch', async () => { + throw new Error('no request expected'); + }); + const files = []; + for (let i = 0; i < 5; i++) { + const content = new Uint8Array(1); + files.push({ + filename: `f${i}.txt`, + contentType: 'text/plain', + inline: false, + contentId: '', + content, + }); + } + const refusals = [ + [providers.lettermint(), { tags: ['a', 'b'] }, /at most one tag/], + [ + providers.plunk(), + { calendar: { method: 'REQUEST', content: INVITATION }, attachments: files }, + /at most five attachments/, + ], + ]; + for (const [transport, fields, error] of refusals) { + const receipt = await transport.send(createMessage({ ...base, ...fields })); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); + } + assert.strictEqual(fetch.mock.callCount(), 0); +}); + test('Fields: Resend sends a scheduled message alone, with scheduled_at', async (t) => { const bodies = []; t.mock.method(globalThis, 'fetch', async (input, init) => { @@ -187,7 +232,7 @@ test('Fields: Resend sends metadata as tags, alone, and refuses what a tag canno ]); for (const receipt of receipts.slice(1)) { assert.match(receipt.errorMessages[0], /Resend takes metadata as tags/); - assert.strictEqual(receipt.errors[0].category, 'validation'); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); assert.strictEqual(receipt.retryable, false); } }); @@ -227,7 +272,7 @@ test('Fields: SendGrid sends send_at, the tracking of the message and custom_arg createMessage({ ...base, scheduledAt: new Date(now() + 73 * 3600_000) }), ); assert.match(late.errorMessages[0], /at most 72 hours ahead/); - assert.strictEqual(late.errors[0].category, 'validation'); + assert.strictEqual(late.errors[0].category, 'unsupported'); assert.strictEqual(bodies.length, 2); }); @@ -315,7 +360,7 @@ test('Fields: SES sends metadata as tags, over the default ones, and refuses wha for (const metadata of [{ 'order.id': '1' }, { note: 'two words' }, { category: 'x' }]) { const refused = await transport.send(createMessage({ ...base, metadata })); assert.match(refused.errorMessages[0], /SES takes metadata as tags/); - assert.strictEqual(refused.errors[0].category, 'validation'); + assert.strictEqual(refused.errors[0].category, 'unsupported'); } assert.strictEqual(bodies.length, 1); }); @@ -361,7 +406,7 @@ test('Fields: Maileroo sends scheduled_at, one tracking switch and metadata as t for (const [fields, error] of refusals) { const refused = await transport.send(createMessage({ ...base, ...fields })); assert.match(refused.errorMessages[0], error); - assert.strictEqual(refused.errors[0].category, 'validation'); + assert.strictEqual(refused.errors[0].category, 'unsupported'); } assert.strictEqual(bodies.length, 2); }); @@ -464,7 +509,7 @@ test('Fields: JMAP refuses a scheduled message, and keeps the order of a batch', for (const receipt of allRefused) assertRefused(receipt); }); -test('Fields: the pool does not fail over with a refusal, and retry does not retry it', async (t) => { +test('Fields: the pool fails over with a refusal to a transport that schedules, and retry does not retry it', async (t) => { t.mock.method(globalThis, 'fetch', async () => { throw new Error('no request expected'); }); @@ -476,8 +521,26 @@ test('Fields: the pool does not fail over with a refusal, and retry does not ret { transport: mock, priority: 1 }, ], }); - assertRefused(await pool.send(scheduled())); - assert.strictEqual(mock.getSentMessagesCount(), 0); + // Changed: a refusal of scheduling was a validation failure, which stopped + // failover, so the message was lost although another transport took it. + assert.strictEqual((await pool.send(scheduled())).successful, true); + assert.deepStrictEqual(mock.getSentMessages()[0].scheduledAt, tomorrow); + const unable = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: providers.ses(), priority: 2 }, + { transport: providers.plunk(), priority: 1 }, + ], + }); + const refused = await unable.send(scheduled()); + assertRefused(refused); + assert.deepStrictEqual( + refused.errors.map((error) => [error.provider, error.category]), + [ + ['ses', 'unsupported'], + ['plunk', 'unsupported'], + ], + ); let attempts = 0; const counted = { send: async (message, options) => { diff --git a/tests/unit/postmark.test.js b/tests/unit/postmark.test.js index 095f6cb..13f0caf 100644 --- a/tests/unit/postmark.test.js +++ b/tests/unit/postmark.test.js @@ -209,9 +209,13 @@ test('Postmark: 500 a batch, a failed request fails its batch, a missing answer test('Postmark: a scheduled message and two tags are refused before any request', async (t) => { const requests = stubFetch(t, () => ok()); const scheduled = await transport().send(message({ scheduledAt: new Date() })); - assert.match(scheduled.errorMessages[0], /Invalid scheduledAt: postmark/); + assert.match(scheduled.errorMessages[0], /Unsupported scheduledAt: postmark/); const tagged = await transport().send(message({ tags: ['a', 'b'] })); assert.match(tagged.errorMessages[0], /one tag per message/); + for (const receipt of [scheduled, tagged]) { + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); + } assert.strictEqual(requests.length, 0); }); diff --git a/tests/unit/scaleway.test.js b/tests/unit/scaleway.test.js index c8c2b6e..2aa98c5 100644 --- a/tests/unit/scaleway.test.js +++ b/tests/unit/scaleway.test.js @@ -142,7 +142,7 @@ test('Scaleway: an inline part, a schedule and 2 MB are refused before a request }), /cannot send an inline attachment/, ], - [message({ scheduledAt: new Date() }), /Invalid scheduledAt: scaleway/], + [message({ scheduledAt: new Date() }), /Unsupported scheduledAt: scaleway/], [ message({ attachments: { @@ -157,7 +157,10 @@ test('Scaleway: an inline part, a schedule and 2 MB are refused before a request ], ]; for (const [refused, error] of refusals) { - assert.match((await transport().send(refused)).errorMessages[0], error); + const receipt = await transport().send(refused); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); } assert.strictEqual(requests.length, 0); }); diff --git a/tests/unit/sendmail.test.js b/tests/unit/sendmail.test.js index a34c255..3ebb1eb 100644 --- a/tests/unit/sendmail.test.js +++ b/tests/unit/sendmail.test.js @@ -298,7 +298,9 @@ test('Sendmail: a scheduled message is refused, sendMany sends in turn, DKIM sig }, }); const scheduled = await transport.send(message({ scheduledAt: new Date() })); - assert.match(scheduled.errorMessages[0], /Invalid scheduledAt: sendmail cannot schedule/); + assert.match(scheduled.errorMessages[0], /Unsupported scheduledAt: sendmail cannot schedule/); + assert.strictEqual(scheduled.errors[0].category, 'unsupported'); + assert.strictEqual(scheduled.retryable, false); const receipts = []; for await (const receipt of transport.sendMany([message(), message()])) receipts.push(receipt); assert.deepStrictEqual( diff --git a/tests/unit/smtp-transport.integration.test.js b/tests/unit/smtp-transport.integration.test.js index 4a6f6b2..24cab19 100644 --- a/tests/unit/smtp-transport.integration.test.js +++ b/tests/unit/smtp-transport.integration.test.js @@ -753,7 +753,9 @@ describe('SmtpTransport Integration Tests', () => { assert.match(receipt.errorMessages[0], /Message size \d+ octets/); assert.equal(receipt.retryable, false); assert.equal(receipt.errors?.[0]?.code, 'smtp.message-size-exceeded'); - assert.equal(receipt.errors?.[0]?.category, 'rejected'); + // Changed after the port: `rejected` stopped the failover of a pool to + // a transport whose limit the message is under. + assert.equal(receipt.errors?.[0]?.category, 'unsupported'); const providerDetails = receipt.errors?.[0]?.providerDetails; if (providerDetails == null || typeof providerDetails !== 'object') { assert.fail('Expected provider details for the SIZE rejection.'); diff --git a/tests/unit/sparkpost.test.js b/tests/unit/sparkpost.test.js index cc1c5fb..4a23c9a 100644 --- a/tests/unit/sparkpost.test.js +++ b/tests/unit/sparkpost.test.js @@ -201,7 +201,7 @@ test('SparkPost: sendRaw sends the message as email_rfc822 to its envelope', asy content: new Uint8Array([...new TextEncoder().encode('Subject: x\r\n\r\n'), 0xe9, 13, 10]), }); assert.strictEqual(latin1.successful, false); - assert.strictEqual(latin1.errors[0].category, 'validation'); + assert.strictEqual(latin1.errors[0].category, 'unsupported'); assert.match(latin1.errorMessages[0], /only as UTF-8/); const invalid = await transport().sendRaw({ envelope: { from: 'shop@example.com', to: [] }, diff --git a/tests/unit/zeptomail.test.js b/tests/unit/zeptomail.test.js index 56b7a35..6d9892a 100644 --- a/tests/unit/zeptomail.test.js +++ b/tests/unit/zeptomail.test.js @@ -173,10 +173,13 @@ test('ZeptoMail: its limits, a second tag and a schedule are refused before a re [message({ attachments: files }), /at most 60 attachments/], [message({ subject: 'x'.repeat(501) }), /subject of at most 500/], [message({ tags: ['a', 'b'] }), /one tag per message/], - [message({ scheduledAt: new Date() }), /Invalid scheduledAt: zeptomail/], + [message({ scheduledAt: new Date() }), /Unsupported scheduledAt: zeptomail/], ]; for (const [refused, error] of refusals) { - assert.match((await transport().send(refused)).errorMessages[0], error); + const receipt = await transport().send(refused); + assert.match(receipt.errorMessages[0], error); + assert.strictEqual(receipt.errors[0].category, 'unsupported'); + assert.strictEqual(receipt.retryable, false); } assert.strictEqual(requests.length, 0); }); From 94724b0b45415a1aee3821d45daeb82313fafa11 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:34:34 +0300 Subject: [PATCH 062/143] feat(webhooks): the Message-ID of an event, and its recipient by domain A receipt of unknown delivery, which the plan of reconciliation from the events is for, has no messageId of the provider, the one field an event gave to find its send by. An event carries rfcMessageId, the Message-ID the message was sent with, where the provider tells it: SES's commonHeaders, SendGrid's smtp-id, Mailgun's id, and the header fields the events of Scaleway, SMTP2GO and MailerSend carry. Its recipient has its domain in lower case, so that an address reads the same in the events of every provider, as a suppression list will want. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 8 ++++- index.d.ts | 13 ++++++++- src/webhooks/mailersend.js | 2 ++ src/webhooks/mailgun.js | 2 ++ src/webhooks/request.js | 45 +++++++++++++++++++++++++++-- src/webhooks/scaleway.js | 9 +++++- src/webhooks/sendgrid.js | 2 ++ src/webhooks/ses.js | 1 + src/webhooks/smtp2go.js | 2 ++ tests/unit/webhooks-mailgun.test.js | 1 + tests/unit/webhooks-request.test.js | 43 +++++++++++++++++++++++++++ 11 files changed, 123 insertions(+), 5 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 8b056fd..d51eef0 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -70,7 +70,8 @@ as many events. | `type` | `accepted`, `delivered`, `deferred`, `bounced`, `rejected`, `complained`, `unsubscribed`, `opened`, `clicked` or `unknown`. | | `id` | The id of the event, the same each time the provider sends it again. | | `messageId` | The provider's id of the message, the `messageId` of its receipt. | -| `recipient` | The recipient, when the event names one. | +| `rfcMessageId` | The Message-ID the message was sent with, when the event carries it (SES, SendGrid, Mailgun, and the providers whose events carry the header fields of the email). | +| `recipient` | The recipient, when the event names one, its domain in lower case. | | `occurredAt` | When it happened. | | `bounce` | For a bounce: `kind` (`hard`, `soft` or `undetermined`), `code` and `diagnostic`. | | `reason` | Why, for a rejection, a deferral or a bounce. | @@ -82,6 +83,11 @@ as many events. suppressed address; `deferred` one it keeps trying; `unknown` an event the model has no name for, kept with its `raw` record. +A receipt of unknown delivery (`deliveryUnknown`) has no `messageId` of the +provider to find its events by. Give the message a `messageId` of its own, or +send it through `./pool` or `./retry`, which pin one, and find its events by +`rfcMessageId`, where the provider tells it. + ## Time and replays A signed request is taken only within `toleranceMs` of now, and only once, by diff --git a/index.d.ts b/index.d.ts index f9d4252..bbc89d2 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1346,7 +1346,18 @@ export interface DeliveryEvent { * event finds the send it belongs to. */ readonly messageId?: string; - /** The recipient, when the event names one. */ + /** + * The Message-ID of the message as it was sent, when the event carries it + * (SES, SendGrid, Mailgun, and the providers whose events carry the header + * fields of the email). A receipt of unknown delivery has no `messageId` + * of the provider: give the message a `messageId`, or pass it through + * `./pool` or `./retry`, which pin one, and find its events by this. + */ + readonly rfcMessageId?: string; + /** + * The recipient, when the event names one, its domain in lower case so + * that an address reads the same in the events of every provider. + */ readonly recipient?: string; /** When it happened, as the provider tells it. */ readonly occurredAt?: Date; diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js index 141e578..697c057 100644 --- a/src/webhooks/mailersend.js +++ b/src/webhooks/mailersend.js @@ -11,6 +11,7 @@ const { createEvent, dateOf, isRecord, + headerValueOf, metadataFromHeaders, parseJsonOf, requireSecret, @@ -63,6 +64,7 @@ const eventOf = (payload) => { type, id: `${stringOf(data.id) ?? stringOf(payload.created_at)}:${payload.type}`, messageId, + rfcMessageId: headerValueOf(email.headers ?? data.headers, 'message-id'), recipient, occurredAt: dateOf(data.created_at ?? payload.created_at), bounce, diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js index e98ef1a..b63c92f 100644 --- a/src/webhooks/mailgun.js +++ b/src/webhooks/mailgun.js @@ -63,6 +63,8 @@ const eventOf = (data, token) => { type: failure?.type ?? TYPES.get(data.event) ?? 'unknown', id: stringOf(data.id) ?? token, messageId: messageIdOf(data.message), + // Mailgun's id of a message is its Message-ID. + rfcMessageId: messageIdOf(data.message), recipient: stringOf(data.recipient), occurredAt: dateOf(data.timestamp), bounce: failure?.bounce, diff --git a/src/webhooks/request.js b/src/webhooks/request.js index cdb3dbd..fa8fc23 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -435,7 +435,24 @@ const dateOf = (value) => { return date !== undefined && Number.isFinite(date.getTime()) ? date : undefined; }; -const OPTIONAL_FIELDS = ['messageId', 'recipient', 'occurredAt', 'bounce', 'reason', 'url']; +const OPTIONAL_FIELDS = [ + 'messageId', + 'rfcMessageId', + 'recipient', + 'occurredAt', + 'bounce', + 'reason', + 'url', +]; + +// An address with its domain in lower case, as a domain has no case: the +// same recipient reads the same in the events of every provider, for a list +// of suppressed addresses to be kept by it. The local part is kept as it is. +const recipientOf = (value) => { + if (typeof value !== 'string') return value; + const at = value.lastIndexOf('@'); + return at < 0 ? value : value.slice(0, at + 1) + value.slice(at + 1).toLowerCase(); +}; /** * An event, its optional fields left out when absent. @@ -445,7 +462,8 @@ const OPTIONAL_FIELDS = ['messageId', 'recipient', 'occurredAt', 'bounce', 'reas const createEvent = (fields) => { const event = { provider: fields.provider, type: fields.type, id: fields.id }; for (const name of OPTIONAL_FIELDS) { - if (fields[name] !== undefined && fields[name] !== '') event[name] = fields[name]; + const value = name === 'recipient' ? recipientOf(fields[name]) : fields[name]; + if (value !== undefined && value !== '') event[name] = value; } event.tags = fields.tags ?? []; event.metadata = fields.metadata ?? Object.create(null); @@ -573,6 +591,28 @@ const jsonRecordOf = (text) => { } }; +/** + * The value of a header field among the headers an event carries, its name + * in any case: a record, or a list of `{ name | key, value }`. + * + * @internal + */ +const headerValueOf = (headers, name) => { + const wanted = name.toLowerCase(); + if (Array.isArray(headers)) { + for (const header of headers) { + const key = header?.name ?? header?.key; + if (typeof key === 'string' && key.toLowerCase() === wanted) return stringOf(header.value); + } + return undefined; + } + if (!isRecord(headers)) return undefined; + for (const key in headers) { + if (key.toLowerCase() === wanted) return stringOf(headers[key]); + } + return undefined; +}; + const METADATA_HEADER = /^x-metadata-(.+)$/i; /** @@ -666,6 +706,7 @@ module.exports = { deliveryOf, equalBytes, headerOf, + headerValueOf, hmacOf, isRecord, jsonRecordOf, diff --git a/src/webhooks/scaleway.js b/src/webhooks/scaleway.js index 3b32898..faa80d1 100644 --- a/src/webhooks/scaleway.js +++ b/src/webhooks/scaleway.js @@ -4,7 +4,13 @@ // sently and unemail have no Scaleway webhook. const { X509Certificate } = require('node:crypto'); -const { createEvent, dateOf, metadataFromHeaders, stringOf } = require('./request.js'); +const { + createEvent, + dateOf, + headerValueOf, + metadataFromHeaders, + stringOf, +} = require('./request.js'); const { certificateError, createSnsVerifier, isHttpsUrlOf } = require('./sns.js'); const DEFAULT_TOLERANCE_MS = 60 * 60 * 1000; @@ -51,6 +57,7 @@ const eventOf = (record, snsMessageId) => { type, id: stringOf(record.id) ?? snsMessageId, messageId: stringOf(record.email_id), + rfcMessageId: headerValueOf(record.email_headers, 'message-id'), recipient: stringOf(record.email_to), occurredAt: dateOf(record.created_at), bounce, diff --git a/src/webhooks/sendgrid.js b/src/webhooks/sendgrid.js index 6fffe00..eaa8fb8 100644 --- a/src/webhooks/sendgrid.js +++ b/src/webhooks/sendgrid.js @@ -124,6 +124,8 @@ const eventOf = (record, index, timestamp) => { type, id: stringOf(record.sg_event_id) ?? `${timestamp}:${index}`, messageId: messageIdOf(record), + // The Message-ID of the message as it was sent. + rfcMessageId: stringOf(record['smtp-id']), recipient: stringOf(record.email), occurredAt: dateOf(record.timestamp), bounce, diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index a5bba37..7cef6e4 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -171,6 +171,7 @@ const eventsOf = (record, snsMessageId) => { type: described.type, id: `${snsMessageId}:${index}`, messageId: stringOf(record.mail?.messageId), + rfcMessageId: stringOf(record.mail?.commonHeaders?.messageId), recipient: recipient.recipient, occurredAt: dateOf(described.at ?? record.mail?.timestamp), bounce: recipient.bounce, diff --git a/src/webhooks/smtp2go.js b/src/webhooks/smtp2go.js index 35597ab..5731447 100644 --- a/src/webhooks/smtp2go.js +++ b/src/webhooks/smtp2go.js @@ -11,6 +11,7 @@ const { dateOf, deliveryOf, isRecord, + headerValueOf, metadataFromHeaders, resolveCredentialOptions, stringOf, @@ -65,6 +66,7 @@ const eventOf = (record) => { type, id: stringOf(record.id) ?? `${record.event}:${messageId}:${recipient}:${at}`, messageId, + rfcMessageId: headerValueOf(record, 'message-id'), recipient, occurredAt: dateOf(at), bounce, diff --git a/tests/unit/webhooks-mailgun.test.js b/tests/unit/webhooks-mailgun.test.js index e4c4a36..3cb3c67 100644 --- a/tests/unit/webhooks-mailgun.test.js +++ b/tests/unit/webhooks-mailgun.test.js @@ -61,6 +61,7 @@ test('Webhooks Mailgun: a signed event, its message id that of the receipt', asy type: 'delivered', id: 'mg-event-1', messageId: '<20270115.1@mg.example.com>', + rfcMessageId: '<20270115.1@mg.example.com>', recipient: 'bob@example.com', occurredAt: new Date(1_799_999_990_500), tags: ['orders'], diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 84c89a6..857a3d5 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -293,3 +293,46 @@ test('Webhooks: a refusal says what to answer, a certificate not read asks for a await delivery.release(); assert.strictEqual(delivery.events.length, 1); }); + +test('Webhooks: an event carries the Message-ID it was sent with, and its recipient by domain', async () => { + // Before: an event had only the provider's id of the message, which a + // receipt of unknown delivery lacks, and the recipient as each provider + // wrote it. + const { createEvent, headerValueOf } = require('../../src/webhooks/request.js'); + const event = createEvent({ + provider: 'x', + type: 'delivered', + id: '1', + rfcMessageId: '', + recipient: 'Bob.Smith@Example.COM', + }); + assert.strictEqual(event.rfcMessageId, ''); + assert.strictEqual(event.recipient, 'Bob.Smith@example.com'); + assert.strictEqual(headerValueOf([{ key: 'Message-ID', value: '' }], 'message-id'), ''); + assert.strictEqual( + headerValueOf([{ name: 'message-id', value: '' }], 'Message-ID'), + '', + ); + assert.strictEqual(headerValueOf({ 'Message-Id': '' }, 'message-id'), ''); + assert.strictEqual(headerValueOf([{ key: 'Subject', value: 's' }], 'message-id'), undefined); + assert.strictEqual(headerValueOf('text', 'message-id'), undefined); + const served = sns.certificateFetch(); + const ses = createSesWebhookVerifier({ + topicArns: [sns.TOPIC], + fetch: served.fetch, + now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: null, + }); + const message = sns.notification({ + eventType: 'Delivery', + mail: { + messageId: 'ses-1', + destination: ['Carol@Example.org'], + commonHeaders: { messageId: '' }, + }, + delivery: { recipients: ['Carol@Example.org'] }, + }); + const { events } = await ses.verify({ body: JSON.stringify(message) }); + assert.strictEqual(events[0].rfcMessageId, ''); + assert.strictEqual(events[0].recipient, 'Carol@example.org'); +}); From 70f01457ab876750c74c31966cf9a8c5411b7b26 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:35:03 +0300 Subject: [PATCH 063/143] chore(size): the webhooks budget for the delivery and correlation fields Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 6ca8438..39e4b74 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4.5, "./render": 3, - "./webhooks": 13, + "./webhooks": 13.5, "./smtp": 23, "./sendmail": 10.5, "./jmap": 13, From 502c62d59eaa933575a9e8253a3cee65e299c2ad Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:35:31 +0300 Subject: [PATCH 064/143] fix(mailtrap): read Retry-After as every other provider does Mailtrap kept a parser of its own, which the fix of bug 29 did not reach: a Retry-After of 99999999999 seconds came back as about 3,170 years, for a queue to schedule, where every other provider reads it as a day; it took fractions of seconds, and a date past as 0. It uses parseRetryAfter of the core. Co-Authored-By: Claude Opus 5.5 --- src/mailtrap/http-client.js | 12 +----------- tests/unit/mailtrap.test.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/mailtrap/http-client.js b/src/mailtrap/http-client.js index 07f09c9..3d0d0c9 100644 --- a/src/mailtrap/http-client.js +++ b/src/mailtrap/http-client.js @@ -1,6 +1,7 @@ // Ported from upyo v0.7.0 (packages/mailtrap/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +const { parseRetryAfter } = require('../core/receipt.js'); const { requestWithRetry } = require('../internal/http.js'); const { createLog } = require('../internal/log.js'); const { readText } = require('../internal/body.js'); @@ -213,17 +214,6 @@ const withAttempts = (error, attempts) => { return Object.assign(error, { attempts }); }; -const parseRetryAfter = (header) => { - if (header == null || header === '') return undefined; - const asSeconds = Number(header); - if (Number.isFinite(asSeconds) && asSeconds >= 0) { - return Math.round(asSeconds * 1000); - } - const asDate = Date.parse(header); - if (Number.isNaN(asDate)) return undefined; - return Math.max(0, asDate - Date.now()); -}; - const inferSandboxRetryAfter = (message, statusCode, sandbox) => { if (sandbox && statusCode === 429 && message.toLowerCase().includes(sandboxRateLimitMessage)) { return sandboxRateLimitWindowMilliseconds; diff --git a/tests/unit/mailtrap.test.js b/tests/unit/mailtrap.test.js index 0800c09..e27ac6d 100644 --- a/tests/unit/mailtrap.test.js +++ b/tests/unit/mailtrap.test.js @@ -165,3 +165,16 @@ test('Mailtrap: a 200 that is not JSON is sent once, of unknown delivery', async assert.strictEqual(receipt.deliveryUnknown, true); assert.strictEqual(receipt.retryable, false); }); + +test('Mailtrap: a Retry-After past a day is read as a day, as for the other providers', async (t) => { + // Before: Mailtrap had a parser of its own, without the bound of the + // others, so 99999999999 seconds came back as about 3,170 years. + t.mock.method( + globalThis, + 'fetch', + async () => new Response('', { status: 429, headers: { 'Retry-After': '99999999999' } }), + ); + const receipt = await new MailtrapTransport({ ...config, retries: 0 }).send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.errors[0].retryAfterMilliseconds, 24 * 60 * 60 * 1000); +}); From c9b15b8262d120247a827dab0c646ebb1536208a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:37:22 +0300 Subject: [PATCH 065/143] fix: settle the messages a batch is left with when its caller stops The decorators' sendMany heard of each message either by its receipt or by the error of the wrapped sendMany. A transport that reads ahead holds messages it has been handed when the caller stops reading, or answers fewer than it read, and neither came for those: an idempotency key stayed reserved, a hook never heard of its send's end, and a circuit breaker whose probe was among them kept probing, refusing every send for good. The gate calls `abandon` for each of them. The breaker frees its probe; hooks call onError with an AbortError; idempotency keeps the key as of unknown delivery, `idempotency.abandoned`, for the lease alone, as the message may have gone. Co-Authored-By: Claude Opus 5.5 --- src/breaker/breaker-transport.js | 5 +++++ src/hooks/hooks-transport.js | 9 ++++++++ src/hooks/index.d.ts | 6 ++++- src/idempotency/idempotency-transport.js | 22 +++++++++++++++++-- src/idempotency/index.d.ts | 7 ++++-- src/internal/gate.js | 14 ++++++++++-- tests/unit/breaker.test.js | 28 ++++++++++++++++++++++++ tests/unit/gate.test.js | 27 +++++++++++++++++++++++ tests/unit/hooks.test.js | 15 +++++++++++++ tests/unit/idempotency.test.js | 23 +++++++++++++++++++ 10 files changed, 149 insertions(+), 7 deletions(-) diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 6c40525..390c3be 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -109,6 +109,11 @@ class CircuitBreakerTransport { return receipt; }, fail: (error, admitted) => this.#thrown(error, options?.signal, admitted.probe), + // Nothing is learned of a message left unanswered, but a probe frees + // its place, or the circuit would refuse every send for good. + abandon: (admitted) => { + if (admitted.probe) this.#probing = false; + }, }); } diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js index 4d3fdcf..7702b07 100644 --- a/src/hooks/hooks-transport.js +++ b/src/hooks/hooks-transport.js @@ -104,6 +104,9 @@ class HooksTransport { return receipt; }, fail: (error, admitted) => this.#error(error, admitted.message, admitted.context), + // A message the batch was left before it was answered ends with an + // error, so that each send that began hears of its end. + abandon: (admitted) => this.#error(abandonedError(), admitted.message, admitted.context), }); } @@ -203,4 +206,10 @@ class HooksTransport { } } +const abandonedError = () => + new DOMException( + 'The batch was left before the message was answered; it may have been sent.', + 'AbortError', + ); + module.exports = { HooksTransport }; diff --git a/src/hooks/index.d.ts b/src/hooks/index.d.ts index d940966..f8c9960 100644 --- a/src/hooks/index.d.ts +++ b/src/hooks/index.d.ts @@ -58,7 +58,11 @@ export interface Hooks { * the context of its send (that of the batch for `sendMany()`). */ readonly onRetry?: (event: RetryEvent, context: HookContext) => void; - /** When the wrapped transport throws, before the error goes on. */ + /** + * When the wrapped transport throws, before the error goes on; and, with an + * `AbortError`, for a message of a batch the caller stopped reading before + * its answer, which a transport reading ahead may have sent. + */ readonly onError?: ( error: unknown, message: Message | RawMessage, diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index f303b94..5de8c1b 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -100,6 +100,7 @@ class IdempotencyTransport { return receipt; }, fail: (error, admitted) => this.#thrown(admitted.entry, error, options?.signal), + abandon: (admitted) => this.#abandon(admitted.entry), }); } @@ -199,7 +200,7 @@ class IdempotencyTransport { // A sent or possibly sent message keeps its key with its receipt; one that // failed to go frees it for another attempt. - async #settle(entry, receipt) { + async #settle(entry, receipt, ttlMs = this.config.ttlMs) { if (entry.key === undefined) return; try { if (receipt.successful || receipt.deliveryUnknown === true) { @@ -207,7 +208,7 @@ class IdempotencyTransport { entry.key, entry.token, { fingerprint: entry.fingerprint, receipt }, - this.config.ttlMs, + ttlMs, ); } else { await this.config.store.release(entry.key, entry.token); @@ -218,6 +219,23 @@ class IdempotencyTransport { } } + // A message handed to a transport that reads ahead, and left before it was + // answered, may have gone or not: its key is kept as of unknown delivery + // for the lease alone, so that a send of it in the next minutes is not + // made twice, and one after them goes. + async #abandon(entry) { + const receipt = createFailedReceipt( + 'The batch was left before the message was answered; whether it went is not known.', + { + provider: this.id, + category: 'unknown', + deliveryUnknown: true, + code: 'idempotency.abandoned', + }, + ); + await this.#settle(entry, receipt, this.config.leaseMs); + } + // A send the caller aborted may have gone, so its key is kept as of unknown // delivery; one that threw otherwise, a bug before any request most likely, // frees it. diff --git a/src/idempotency/index.d.ts b/src/idempotency/index.d.ts index fe680ec..2f713f1 100644 --- a/src/idempotency/index.d.ts +++ b/src/idempotency/index.d.ts @@ -156,8 +156,11 @@ export declare function createMemoryIdempotencyStore( * delivery (code `idempotency.in_flight`); the same key for another message * is refused (`idempotency.key_reused`). A send that failed frees its key, one * of unknown delivery keeps it; a key stays reserved for `leaseMs` at most, - * its receipt is kept for `ttlMs`. A store that fails fails the message it - * was asked about alone (`idempotency.store_failed`, retryable). The + * its receipt is kept for `ttlMs`. A message of a batch the caller stopped + * reading before its answer, which a transport reading ahead may have sent, + * keeps its key as of unknown delivery (`idempotency.abandoned`) for `leaseMs` + * alone. A store that fails fails the message it was asked about alone + * (`idempotency.store_failed`, retryable). The * receipts it makes, except `key_reused` (`validation`), are of the category * `unknown`, which a circuit breaker does not count against the provider. * diff --git a/src/internal/gate.js b/src/internal/gate.js index a4a8f35..51fafff 100644 --- a/src/internal/gate.js +++ b/src/internal/gate.js @@ -49,12 +49,14 @@ const iteratorOf = (messages) => * runs, a run ending at a message answered in place, and each receipt goes * through `gate.settle(receipt, admitted)` before it is yielded. When the * wrapped `sendMany()` throws, `gate.fail(error, admitted)` hears of each - * message still waiting for its receipt. + * message still waiting for its receipt; when the caller stops reading, or + * the wrapped `sendMany()` ends short, `gate.abandon(admitted)` does, as a + * transport that reads ahead may have sent them or not. * * @param transport The wrapped transport. * @param messages The messages, sync or async. * @param options The caller's options, passed on. - * @param gate `admit`, and optional `settle` and `fail`. + * @param gate `admit`, and optional `settle`, `fail` and `abandon`. * @returns The receipts, one per message, in order. * @internal */ @@ -107,16 +109,24 @@ const feedRun = async function* (run, gate) { }; const sendRun = async function* (transport, run, options, gate) { + let failed = false; try { for await (const receipt of transport.sendMany(feedRun(run, gate), options)) { const admitted = run.fed.shift(); yield gate.settle === undefined ? receipt : await gate.settle(receipt, admitted); } } catch (error) { + failed = true; if (gate.fail !== undefined) { while (run.fed.size > 0) await gate.fail(error, run.fed.shift()); } throw error; + } finally { + // The caller stopped reading, or the transport answered fewer messages + // than it was handed: those it was handed are neither settled nor failed. + if (!failed && gate.abandon !== undefined) { + while (run.fed.size > 0) await gate.abandon(run.fed.shift()); + } } }; diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js index eb4805c..cb9ddf6 100644 --- a/tests/unit/breaker.test.js +++ b/tests/unit/breaker.test.js @@ -256,3 +256,31 @@ test('Breaker: a listener that throws is a warning, and the options are checked' assert.strictEqual(defaults.config.resetTimeoutMs, 30_000); assert.strictEqual(defaults.id, 'fake'); }); + +test('Breaker: a probe a batch left before its answer frees its place', async () => { + // Before: a probe admitted while the wrapped transport read ahead, and left + // unanswered when the caller stopped reading, kept the circuit probing: it + // refused every send for good. + const time = clock(); + const inner = fakeTransport( + (_message, index) => (index === 0 ? failed('network') : sent(`m${index}`)), + { readAhead: 1 }, + ); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 1, + now: time.now, + }); + const iterator = breaker + .sendMany([message('a'), message('b'), message('c')]) + [Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).value.successful, false); + // The circuit is open; once its timeout passes, the next message read is the probe. + time.advance(1); + assert.strictEqual((await iterator.next()).value.successful, true); + await iterator.return(); + assert.strictEqual(breaker.state, 'half-open'); + const receipt = await breaker.send(message('d')); + assert.strictEqual(receipt.successful, true); + assert.strictEqual(breaker.state, 'closed'); +}); diff --git a/tests/unit/gate.test.js b/tests/unit/gate.test.js index ad80bbb..83f124a 100644 --- a/tests/unit/gate.test.js +++ b/tests/unit/gate.test.js @@ -63,3 +63,30 @@ test('Gate: the queue keeps its order across its compaction', () => { queue.push(5000); assert.strictEqual(queue.shift(), 4000); }); + +test('Gate: what a transport reading ahead was handed is abandoned on a break', async () => { + // Before: neither settle nor fail heard of it, so a decorator kept its + // state for it: a reserved key, a probe, a hook never called. + const inner = fakeTransport((message) => sent(String(message)), { readAhead: 3 }); + const abandoned = []; + const gate = { + admit: (n) => ({ message: n }), + settle: (receipt) => receipt, + abandon: (admitted) => { + abandoned.push(admitted.message); + }, + }; + for await (const receipt of gatedSendMany(inner, [1, 2, 3, 4, 5, 6], undefined, gate)) { + if (receipt.messageId === '1') break; + } + assert.deepStrictEqual(abandoned, [2, 3, 4]); + // A transport that answers fewer messages than it read abandons the rest. + const short = { + async *sendMany(messages) { + for await (const message of messages) if (message === 2) return; + }, + }; + abandoned.length = 0; + assert.deepStrictEqual(await collect(gatedSendMany(short, [1, 2], undefined, gate)), []); + assert.deepStrictEqual(abandoned, [1, 2]); +}); diff --git a/tests/unit/hooks.test.js b/tests/unit/hooks.test.js index 41aa2e9..b33b348 100644 --- a/tests/unit/hooks.test.js +++ b/tests/unit/hooks.test.js @@ -220,3 +220,18 @@ test('Hooks: sendRaw, verify, dispose and the options', async () => { /already/, ); }); + +test('Hooks: a message a batch left before its answer ends with onError', async () => { + // Before: neither afterSend nor onError heard of it. + const errors = []; + const hooks = new HooksTransport(fakeTransport(undefined, { readAhead: 2 }), { + onError: (error, sentMessage) => errors.push([error.name, sentMessage.subject]), + }); + for await (const receipt of hooks.sendMany([message('a'), message('b'), message('c')])) { + if (receipt.successful) break; + } + assert.deepStrictEqual(errors, [ + ['AbortError', 'b'], + ['AbortError', 'c'], + ]); +}); diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js index ac7a868..3897f42 100644 --- a/tests/unit/idempotency.test.js +++ b/tests/unit/idempotency.test.js @@ -473,3 +473,26 @@ test('Idempotency: the memory store makes room with expired keys, then done ones kept.reserve('third', 'ft', 1000); assert.strictEqual(kept.reserve('kept', 'fk', 1000).record.receipt.messageId, 'rk'); }); + +test('Idempotency: a key a batch left before its answer is held for the lease alone', async () => { + // Before: the key of a message handed to a transport that reads ahead, and + // left unanswered on a break, stayed pending, answered in_flight, until it + // expired. + let time = 0; + const store = createMemoryIdempotencyStore({ now: () => time }); + const inner = fakeTransport(undefined, { readAhead: 1 }); + const idempotent = new IdempotencyTransport(inner, { store, leaseMs: 1000 }); + for await (const receipt of idempotent.sendMany([ + message({ idempotencyKey: 'k-1' }), + message({ idempotencyKey: 'k-2', subject: 'Second' }), + ])) { + if (receipt.successful) break; + } + const again = await idempotent.send(message({ idempotencyKey: 'k-2', subject: 'Second' })); + assert.strictEqual(again.successful, false); + assert.strictEqual(again.errors[0].code, 'idempotency.abandoned'); + assert.strictEqual(again.deliveryUnknown, true); + time = 1000; + const later = await idempotent.send(message({ idempotencyKey: 'k-2', subject: 'Second' })); + assert.strictEqual(later.successful, true); +}); From 947e8a659d29586ef20c364c85883ba2536c1af7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:37:31 +0300 Subject: [PATCH 066/143] test: a generator in the gate test that yields, as lint asks Co-Authored-By: Claude Opus 5.5 --- tests/unit/gate.test.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/gate.test.js b/tests/unit/gate.test.js index 83f124a..edebdc8 100644 --- a/tests/unit/gate.test.js +++ b/tests/unit/gate.test.js @@ -83,7 +83,11 @@ test('Gate: what a transport reading ahead was handed is abandoned on a break', // A transport that answers fewer messages than it read abandons the rest. const short = { async *sendMany(messages) { - for await (const message of messages) if (message === 2) return; + for await (const message of messages) { + if (message === 2) return; + // Never reached: the transport ends before it answers anything. + if (message === 0) yield sent('none'); + } }, }; abandoned.length = 0; From a348b1cfbaa10d1cd1789ad723319be5845a6c67 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:37:39 +0300 Subject: [PATCH 067/143] test: the iterator of the breaker test through a variable, as lint asks Co-Authored-By: Claude Opus 5.5 --- tests/unit/breaker.test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js index cb9ddf6..a5ee6cd 100644 --- a/tests/unit/breaker.test.js +++ b/tests/unit/breaker.test.js @@ -271,9 +271,8 @@ test('Breaker: a probe a batch left before its answer frees its place', async () resetTimeoutMs: 1, now: time.now, }); - const iterator = breaker - .sendMany([message('a'), message('b'), message('c')]) - [Symbol.asyncIterator](); + const receipts = breaker.sendMany([message('a'), message('b'), message('c')]); + const iterator = receipts[Symbol.asyncIterator](); assert.strictEqual((await iterator.next()).value.successful, false); // The circuit is open; once its timeout passes, the next message read is the probe. time.advance(1); From 44e9fca2064c5a67e29c2b9890fae88d9d23ca50 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:39:07 +0300 Subject: [PATCH 068/143] fix(breaker): a probe has a deadline; results count in their period A probe that never answered, a send that hangs, kept the circuit probing, refusing every send for good. One that has not answered within probeTimeoutMs (resetTimeoutMs by default) is taken for a failure and the circuit opens again. A result counted in whatever closed period the circuit was in when it came, so a send made before the circuit opened and closed again could open it anew; each change of state starts a period, and a result counts only in its own. An error of the wrapped sendMany counted once per message it held; it is one failure. An isFailure that threw rejected the send and lost its receipt; it is a warning, and the receipt is judged by default. Co-Authored-By: Claude Opus 5.5 --- docs/transports/breaker.md | 11 ++-- src/breaker/breaker-transport.js | 86 ++++++++++++++++++++++++-------- src/breaker/config.js | 6 +++ src/breaker/index.d.ts | 11 +++- tests/unit/breaker.test.js | 82 ++++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 25 deletions(-) diff --git a/docs/transports/breaker.md b/docs/transports/breaker.md index f0117c3..9ee3760 100644 --- a/docs/transports/breaker.md +++ b/docs/transports/breaker.md @@ -29,11 +29,13 @@ const transport = new CircuitBreakerTransport(new SesTransport(sesConfig), { `breaker.open`) whose `retryAfterMilliseconds` is the time left. - **half-open:** after `resetTimeoutMs`. One send goes through as a probe; the others are answered as while open until it ends. Its success closes the - circuit, its failure opens it again. A probe the caller aborts decides - nothing. + circuit, its failure opens it again, and so does a probe that has not + answered within `probeTimeoutMs`. A probe the caller aborts, or that a batch + is left with when its caller stops reading, decides nothing. -A result counts only in the state it was sent in: a late one of a send made -before the circuit opened changes nothing. The state is that of the instance; +A result counts only in the period of the circuit it was sent in: a late one of +a send made before the circuit opened, or before it closed again, changes +nothing. A batch whose `sendMany()` throws is one failure, not one per message. The state is that of the instance; `breaker.state` reads it. ## With a pool and retries @@ -60,6 +62,7 @@ const pool = new PoolTransport({ | --- | --- | --- | | `failureThreshold` | `5` | The failures in a row that open the circuit. | | `resetTimeoutMs` | `30000` | How long it stays open before a probe. | +| `probeTimeoutMs` | `resetTimeoutMs` | How long a probe may take before it counts as a failure. | | `isFailure` | the categories above | Whether a receipt counts as a failure. | | `onStateChange` | | Hears each change of state; one that throws becomes a process warning (`MAILER_BREAKER_LISTENER_FAILED`). | | `now` | `Date.now` | The clock. | diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 390c3be..a1dbb21 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -8,7 +8,9 @@ // server error, unavailable), not a refused message or a caller's abort; and // an open circuit answers with a retryable `service-unavailable` receipt // carrying the time left, which `./pool` fails over with and `./retry` waits -// for. +// for; a probe that does not answer within `probeTimeoutMs` is a failure, a +// result counts only in the period of the circuit it was sent in, and a +// batch that throws counts once. const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); @@ -45,6 +47,10 @@ class CircuitBreakerTransport { #failures = 0; #openedAt = 0; #probing = false; + #probeAt = 0; + // Grows at each change of state: a result counts only in the period it + // was sent in, not in one the circuit came to since. + #period = 0; /** * @param transport The transport to guard. @@ -67,6 +73,15 @@ class CircuitBreakerTransport { return this.#state; } + #now() { + try { + return this.config.now(); + } catch (error) { + warnThrown('The now of CircuitBreakerTransport', error, 'MAILER_BREAKER_LISTENER_FAILED'); + return Date.now(); + } + } + /** * Sends one message, unless the circuit is open. * @@ -99,20 +114,27 @@ class CircuitBreakerTransport { * once, and one while it is half-open goes as its probe. */ sendMany(messages, options) { + // One error of the wrapped sendMany fails every message it held: it is one + // failure of the provider, not as many as the messages. + let counted; return gatedSendMany(this.wrappedTransport, messages, options, { admit: (message) => { const entry = this.#enter(); - return entry.refusal === undefined ? { message, probe: entry.probe } : entry.refusal; + return entry.refusal === undefined ? { message, ...entry } : entry.refusal; }, settle: (receipt, admitted) => { - this.#record(this.#counts(receipt), admitted.probe); + this.#record(this.#counts(receipt), admitted); return receipt; }, - fail: (error, admitted) => this.#thrown(error, options?.signal, admitted.probe), + fail: (error, admitted) => { + if (error === counted && !admitted.probe) return; + counted = error; + this.#thrown(error, options?.signal, admitted); + }, // Nothing is learned of a message left unanswered, but a probe frees // its place, or the circuit would refuse every send for good. abandon: (admitted) => { - if (admitted.probe) this.#probing = false; + if (admitted.probe && admitted.period === this.#period) this.#probing = false; }, }); } @@ -147,19 +169,27 @@ class CircuitBreakerTransport { try { receipt = await send(options); } catch (error) { - this.#thrown(error, options?.signal, entry.probe); + this.#thrown(error, options?.signal, entry); throw error; } - this.#record(this.#counts(receipt), entry.probe); + this.#record(this.#counts(receipt), entry); return receipt; } #remaining() { - return this.#openedAt + this.config.resetTimeoutMs - this.config.now(); + return this.#openedAt + this.config.resetTimeoutMs - this.#now(); } - // Whether a send may go now, and whether it is the probe. + // Whether a send may go now, whether it is the probe, and the period of the + // circuit it goes in. #enter() { + if (this.#state === 'half-open' && this.#probing) { + // A probe that has not answered in time failed: the circuit opens again. + if (this.#now() - this.#probeAt >= this.config.probeTimeoutMs) { + this.#probing = false; + this.#open(); + } + } if (this.#state === 'open') { const remaining = this.#remaining(); if (remaining > 0) return { refusal: this.#refusal(remaining) }; @@ -168,9 +198,10 @@ class CircuitBreakerTransport { if (this.#state === 'half-open') { if (this.#probing) return { refusal: this.#refusal(undefined) }; this.#probing = true; - return { probe: true }; + this.#probeAt = this.#now(); + return { probe: true, period: this.#period }; } - return { probe: false }; + return { probe: false, period: this.#period }; } #refusal(remaining) { @@ -188,23 +219,37 @@ class CircuitBreakerTransport { return { receipt }; } + // An isFailure that throws is a warning, and the receipt is judged as it + // would be without it, rather than lost with the send. #counts(receipt) { - return (this.config.isFailure ?? isProviderFailure)(receipt) === true; + if (this.config.isFailure === undefined) return isProviderFailure(receipt); + try { + return this.config.isFailure(receipt) === true; + } catch (error) { + warnThrown( + 'The isFailure of CircuitBreakerTransport', + error, + 'MAILER_BREAKER_LISTENER_FAILED', + ); + return isProviderFailure(receipt); + } } - #thrown(error, signal, probe) { + #thrown(error, signal, entry) { if (isAbortOf(error, signal)) { // The caller gave up: nothing is learned of the provider. - if (probe) this.#probing = false; + if (entry.probe && entry.period === this.#period) this.#probing = false; return; } - this.#record(true, probe); + this.#record(true, entry); } - // A result counts only in the state it was sent in: a late one of a send - // made before the circuit opened decides nothing. - #record(failed, probe) { - if (probe) { + // A result counts only in the period of the circuit it was sent in: a late + // one of a send made before the circuit opened, or closed again, or of a + // probe that ran out of time, decides nothing. + #record(failed, entry) { + if (entry.period !== this.#period) return; + if (entry.probe) { this.#probing = false; if (failed) this.#open(); else this.#close(); @@ -220,7 +265,7 @@ class CircuitBreakerTransport { } #open() { - this.#openedAt = this.config.now(); + this.#openedAt = this.#now(); this.#failures = 0; this.#transition('open'); } @@ -234,6 +279,7 @@ class CircuitBreakerTransport { if (this.#state === state) return; const previous = this.#state; this.#state = state; + this.#period++; try { this.config.onStateChange?.(state, previous); } catch (error) { diff --git a/src/breaker/config.js b/src/breaker/config.js index 3ee90ed..8c75e7c 100644 --- a/src/breaker/config.js +++ b/src/breaker/config.js @@ -19,6 +19,12 @@ const { const createBreakerConfig = (config = {}) => ({ failureThreshold: checkPositiveInteger(config.failureThreshold ?? 5, 'failureThreshold'), resetTimeoutMs: checkPositiveNumber(config.resetTimeoutMs ?? 30000, 'resetTimeoutMs'), + // A probe that has not answered by then is taken for a failure, so that a + // send that hangs cannot hold the circuit half-open for good. + probeTimeoutMs: checkPositiveNumber( + config.probeTimeoutMs ?? config.resetTimeoutMs ?? 30000, + 'probeTimeoutMs', + ), isFailure: checkOptionalFunction(config.isFailure, 'isFailure'), onStateChange: checkOptionalFunction(config.onStateChange, 'onStateChange'), now: checkOptionalFunction(config.now, 'now') ?? Date.now, diff --git a/src/breaker/index.d.ts b/src/breaker/index.d.ts index c91d15a..ed44de9 100644 --- a/src/breaker/index.d.ts +++ b/src/breaker/index.d.ts @@ -34,12 +34,20 @@ export interface CircuitBreakerConfig { * @default 30000 */ readonly resetTimeoutMs?: number; + /** + * How long a probe may take before it counts as a failure, the circuit + * opening again, in milliseconds. + * @default resetTimeoutMs + */ + readonly probeTimeoutMs?: number; /** * Whether a receipt counts as a failure of the provider. By default a * failed receipt counts when one of its errors is of the category * `network`, `timeout`, `server-error` or `service-unavailable`; a refused * message, a rate limit or a caller's abort says nothing of its health. - * A thrown error, other than the caller's abort, always counts. + * A thrown error, other than the caller's abort, always counts, once for + * a batch. One that throws is a process warning, and the receipt is + * judged as by default. */ readonly isFailure?: (receipt: Receipt) => boolean; /** @@ -57,6 +65,7 @@ export interface CircuitBreakerConfig { export interface ResolvedCircuitBreakerConfig { readonly failureThreshold: number; readonly resetTimeoutMs: number; + readonly probeTimeoutMs: number; readonly isFailure?: (receipt: Receipt) => boolean; readonly onStateChange?: (state: CircuitState, previous: CircuitState) => void; readonly now: () => number; diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js index a5ee6cd..244ec05 100644 --- a/tests/unit/breaker.test.js +++ b/tests/unit/breaker.test.js @@ -283,3 +283,85 @@ test('Breaker: a probe a batch left before its answer frees its place', async () assert.strictEqual(receipt.successful, true); assert.strictEqual(breaker.state, 'closed'); }); + +test('Breaker: a probe that does not answer in time fails, and a new one goes later', async () => { + // Before: a probe that hung held the circuit half-open, refusing every send. + const time = clock(); + let hang = true; + const inner = fakeTransport((_message, index) => { + if (index === 0) return failed('network'); + return hang ? new Promise(() => {}) : sent(`m${index}`); + }); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 100, + probeTimeoutMs: 50, + now: time.now, + }); + await breaker.send(message()); + time.advance(100); + breaker.send(message()); // the probe, which hangs + time.advance(49); + assert.strictEqual((await breaker.send(message())).errors[0].code, 'breaker.open'); + time.advance(1); + const reopened = await breaker.send(message()); + assert.strictEqual(reopened.errors[0].code, 'breaker.open'); + assert.strictEqual(reopened.errors[0].retryAfterMilliseconds, 100); + hang = false; + time.advance(100); + assert.strictEqual((await breaker.send(message())).successful, true); + assert.strictEqual(breaker.state, 'closed'); +}); + +test('Breaker: a late result counts only in the period of the circuit it was sent in', async () => { + // Before: a failure of a send made before the circuit opened and closed + // again counted in the new closed period. + const time = clock(); + let failLate; + const inner = fakeTransport((_message, index) => { + if (index === 0) return new Promise((resolve) => (failLate = () => resolve(failed('network')))); + return index === 1 ? failed('network') : sent(`m${index}`); + }); + const breaker = new CircuitBreakerTransport(inner, { + failureThreshold: 1, + resetTimeoutMs: 10, + now: time.now, + }); + const late = breaker.send(message('slow')); + await breaker.send(message('fails')); + assert.strictEqual(breaker.state, 'open'); + time.advance(10); + await breaker.send(message('probe')); + assert.strictEqual(breaker.state, 'closed'); + failLate(); + await late; + assert.strictEqual(breaker.state, 'closed'); +}); + +test('Breaker: a batch that throws is one failure, and an isFailure that throws a warning', async (t) => { + // Before: each message the batch held counted, and a throwing isFailure + // turned the send into a rejection. + const boom = new Error('connection lost'); + const inner = fakeTransport(() => ({ throw: boom }), { readAhead: 3 }); + const breaker = new CircuitBreakerTransport(inner, { failureThreshold: 2 }); + await assert.rejects( + collect(breaker.sendMany([message(), message(), message(), message()])), + /connection lost/, + ); + assert.strictEqual(breaker.state, 'closed'); + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push(options.code)); + const judged = new CircuitBreakerTransport( + fakeTransport(() => failed('network')), + { + failureThreshold: 1, + isFailure: () => { + throw new Error('cannot judge'); + }, + }, + ); + const receipt = await judged.send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(judged.state, 'open'); + assert.deepStrictEqual(warnings, ['MAILER_BREAKER_LISTENER_FAILED']); +}); From 4a7aeb5675c529b9618528cff74f7d3c7d3c0e1d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:41:15 +0300 Subject: [PATCH 069/143] fix(sendmail): a timeout, a bound on the processes, no success unread A sendmail that hung held the send for good, unless the caller passed a signal; and every concurrent send started its process. A process has `timeout` (60 s) to exit, then is stopped, the receipt a timeout of unknown delivery once it had the whole message; no more than `maxConcurrent` (10) run at once, a send waiting for a place, which an abort leaves; and an abort or a timeout sends SIGKILL after a grace of five seconds when SIGTERM is not heeded. An exit with 0 was a success whatever had been written: a command that closed its input early and said it was done had queued a part of the message, or none. It is a failure of unknown delivery. Co-Authored-By: Claude Opus 5.5 --- docs/transports/sendmail.md | 14 +++- scripts/size-budgets.json | 2 +- src/sendmail/config.js | 22 +++++- src/sendmail/index.d.ts | 14 ++++ src/sendmail/sendmail-transport.js | 121 +++++++++++++++++++++++++---- tests/unit/sendmail.test.js | 95 ++++++++++++++++++++++ 6 files changed, 249 insertions(+), 19 deletions(-) diff --git a/docs/transports/sendmail.md b/docs/transports/sendmail.md index 38039c2..453d0c0 100644 --- a/docs/transports/sendmail.md +++ b/docs/transports/sendmail.md @@ -47,6 +47,8 @@ command line only: the message has no Bcc field for sendmail to strip. | `path` | `'sendmail'` | The command, a path or a name on PATH. | | `args` | | Arguments in place of `-f `: ` -i -- `. | | `newline` | `'lf'` | `'lf'`, which every sendmail takes, or `'crlf'`. | +| `timeout` | `60000` | How long a process may take; it is stopped then (SIGTERM, then SIGKILL), of unknown delivery once it had the whole message. 0 or `Infinity` for none. | +| `maxConcurrent` | `10` | The most processes at once; a send waits for a place. | | `spawn` | `child_process.spawn` | Starts the process: pass your own to run sendmail in a container or as another user. | | `dkim` | | A DKIM configuration, as for [SMTP](/transports/smtp#dkim-signing). | | `generateId` | | Makes the left half of a Message-ID, and the id of a raw receipt. | @@ -54,8 +56,8 @@ command line only: the message has no Bcc field for sendmail to strip. ## Receipts -A send succeeds when sendmail exits with code 0, and the receipt carries the -Message-ID of the message. Any other end is a failed receipt, with what sendmail +A send succeeds when sendmail exits with code 0 having read the whole message, +and the receipt carries the Message-ID of the message. Any other end is a failed receipt, with what sendmail wrote to stderr (its first 4 KiB, addresses replaced by `[REDACTED]`): | Exit | Category | Retryable | @@ -66,9 +68,13 @@ wrote to stderr (its first 4 KiB, addresses replaced by `[REDACTED]`): | 64, 77, 78, 126, 127, command not found | `configuration` | no | | any other code | `server-error` | no | | killed by a signal | `server-error` | before it had the whole message; after, `deliveryUnknown` | +| past `timeout` | `timeout` | before it had the whole message; after, `deliveryUnknown` | +| 0 before it read the whole message | `server-error` | no, `deliveryUnknown` | -An abort of the `signal` kills the process. A message whose content cannot be -read (an attachment that fails) kills it with SIGINT, so that no part of it is +An abort of the `signal` stops the process, SIGTERM then SIGKILL five seconds +on, and its place among the `maxConcurrent` is free once it has exited. A +message whose content cannot be read (an attachment that fails) kills it with +SIGINT, so that no part of it is delivered. `scheduledAt` is refused as `unsupported`, as sendmail cannot schedule. ## Raw messages and verification diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 39e4b74..a6ccd66 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -5,7 +5,7 @@ "./render": 3, "./webhooks": 13.5, "./smtp": 23, - "./sendmail": 10.5, + "./sendmail": 11.5, "./jmap": 13, "./ses": 8, "./resend": 6.5, diff --git a/src/sendmail/config.js b/src/sendmail/config.js index 7eafb85..1859a00 100644 --- a/src/sendmail/config.js +++ b/src/sendmail/config.js @@ -2,7 +2,8 @@ // options of the sendmail transport, `path`, `args` and `newline`. // https://github.com/nodemailer/nodemailer — MIT-0, Copyright 2011-2023 Andris Reinman // Changed in the port: checked when the transport is made, with `spawn` passed -// in, LF line endings by default, and DKIM from ./mime. +// in, LF line endings by default, DKIM from ./mime, and a timeout and a bound +// on the processes at once, which nodemailer had not. const { checkDkimConfig } = require('../mime/dkim/types.js'); @@ -12,6 +13,23 @@ const throwTypeError = (message) => { throw new TypeError(message); }; +// 0 and Infinity mean none, as for the other transports. +const checkTimeout = (value) => { + if (value === undefined) return 60_000; + if (typeof value !== 'number' || Number.isNaN(value) || value < 0) { + throwTypeError(`timeout must be a non-negative number: ${value}`); + } + return value === 0 ? Number.POSITIVE_INFINITY : value; +}; + +const checkMaxConcurrent = (value) => { + if (value === undefined) return 10; + if (value !== Number.POSITIVE_INFINITY && (!Number.isSafeInteger(value) || value < 1)) { + throwTypeError(`maxConcurrent must be a positive whole number or Infinity: ${value}`); + } + return value; +}; + // An argument goes to the process as it is; only a NUL cannot. const isArgument = (value) => typeof value === 'string' && !value.includes('\0'); @@ -47,6 +65,8 @@ const createSendmailConfig = (config = {}) => { path, args: checkArgs(config.args), newline, + timeout: checkTimeout(config.timeout), + maxConcurrent: checkMaxConcurrent(config.maxConcurrent), spawn: config.spawn, dkim: config.dkim, generateId: config.generateId, diff --git a/src/sendmail/index.d.ts b/src/sendmail/index.d.ts index b2c2c7c..40eb6df 100644 --- a/src/sendmail/index.d.ts +++ b/src/sendmail/index.d.ts @@ -52,6 +52,20 @@ export interface SendmailConfig { * @default "lf" */ readonly newline?: 'lf' | 'crlf'; + /** + * How long a process may take, in milliseconds, 0 or `Infinity` for none: + * it is then stopped (SIGTERM, then SIGKILL five seconds on), and the + * receipt is a `timeout`, of unknown delivery once the whole message was + * written. + * @default 60000 + */ + readonly timeout?: number; + /** + * The most processes run at once; a send waits for a place, first come + * first served. + * @default 10 + */ + readonly maxConcurrent?: number; /** * Starts the process, `child_process.spawn` by default: pass your own to run * sendmail in a container or under another user. diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index aee6df0..fe0530f 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -8,7 +8,10 @@ // alone, where nodemailer took a process killed by a signal (code null) for // success and handed the code of `close` on as an error; what sendmail writes // to stderr is read, bounded, into the failure; line endings are LF by -// default; an abort kills the process; and `spawn` is passed in. +// default; an abort or the timeout kills the process, with SIGKILL after a +// grace; no more than `maxConcurrent` processes run at once; an exit with 0 +// before the whole message was read is of unknown delivery; and `spawn` is +// passed in. const childProcess = require('node:child_process'); const { once } = require('node:events'); @@ -27,11 +30,15 @@ const { createLog } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); +const { abortReason } = require('../internal/sleep.js'); +const { Fifo } = require('../internal/gate.js'); const { composeMessage } = require('../mime/index.js'); const { createSendmailConfig } = require('./config.js'); const MAX_STDERR_BYTES = 4096; const CR = 0x0d; +// How long a process asked to stop with SIGTERM has before SIGKILL. +const KILL_GRACE_MS = 5000; // sysexits.h: the category of a failure by the exit status of sendmail, and // whether a later attempt may pass. A status sendmail gives says it did not @@ -133,27 +140,46 @@ const collectStderr = (stream) => { return () => Buffer.concat(chunks, size).toString('utf8').trim(); }; +// Asks the process to stop, and makes it if it does not within the grace. +const terminate = (child) => { + child.kill('SIGTERM'); + const timer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS); + timer.unref?.(); + child.once('close', () => clearTimeout(timer)); +}; + /** - * Runs sendmail on a message and waits for it to exit. + * Runs sendmail on a message and waits for it to exit, or for `timeout`. * - * @returns `{ code, signal, stderr, written, error, contentError }`. + * @returns `{ code, signal, stderr, written, timedOut, error, contentError }`. */ -const run = (child, chunks, newline, abort) => +const run = (child, chunks, newline, abort, timeout) => new Promise((resolve, reject) => { - const outcome = { written: false }; + const outcome = { written: false, timedOut: false }; const stderrOf = collectStderr(child.stderr); child.stdin?.on('error', () => {}); const onAbort = () => { - child.kill(); - reject(abort.reason); + terminate(child); + reject(abortReason(abort)); }; abort?.addEventListener('abort', onAbort, { once: true }); - child.once('error', (error) => { + const timer = + timeout === Number.POSITIVE_INFINITY + ? undefined + : setTimeout(() => { + outcome.timedOut = true; + terminate(child); + }, timeout); + const done = () => { + clearTimeout(timer); abort?.removeEventListener('abort', onAbort); + }; + child.once('error', (error) => { + done(); resolve({ ...outcome, error }); }); child.once('close', (code, killSignal) => { - abort?.removeEventListener('abort', onAbort); + done(); resolve({ ...outcome, code, signal: killSignal, stderr: stderrOf() }); }); writeMessage(child.stdin, chunks, newline).then( @@ -180,10 +206,26 @@ const receiptOf = (result, path, messageId) => { }); } if (result.contentError !== undefined) return failureOf(result.contentError); + const said = result.stderr === '' ? '' : `: ${redactAddresses(result.stderr)}`; + if (result.timedOut) { + // Stopped once it had the whole message, it may have queued it. + return createFailedReceipt(`sendmail did not exit in time${said}`, { + provider: 'sendmail', + category: 'timeout', + retryable: !result.written, + deliveryUnknown: result.written, + }); + } if (result.code === 0 && result.signal == null) { - return { successful: true, messageId, provider: 'sendmail' }; + if (result.written) return { successful: true, messageId, provider: 'sendmail' }; + // It took a part of the message, or none, and said it was done: what it + // queued, if anything, is not the message. + return createFailedReceipt(`sendmail exited before it read the whole message${said}`, { + provider: 'sendmail', + category: 'server-error', + deliveryUnknown: true, + }); } - const said = result.stderr === '' ? '' : `: ${redactAddresses(result.stderr)}`; if (result.code == null) { // Killed once it had the whole message, it may have queued it. return createFailedReceipt(`sendmail was killed by ${result.signal}${said}`, { @@ -227,6 +269,8 @@ const findCommand = async (command) => { class SendmailTransport { id = 'sendmail'; #log; + #running = 0; + #waiting = new Fifo(); constructor(config = {}) { this.config = createSendmailConfig(config); @@ -311,19 +355,34 @@ class SendmailTransport { async deliver(envelope, chunks, messageId, signal) { let args; - let child; try { args = argsOf(this.config, envelope); + } catch (error) { + return failureOf(error); + } + await this.#acquire(signal); + let child; + try { const spawn = this.config.spawn ?? childProcess.spawn; child = spawn(this.config.path, args, { stdio: ['pipe', 'ignore', 'pipe'], windowsHide: true, }); } catch (error) { + this.#release(); return failureOf(error); } + // The place is the process's until it has exited, an aborted one too. + let released = false; + const release = () => { + if (released) return; + released = true; + this.#release(); + }; + child.once('close', release); + child.once('error', release); this.#log.write('debug', () => ({ recipients: envelope.to.length }), 'sendmail started'); - const result = await run(child, chunks, this.config.newline, signal); + const result = await run(child, chunks, this.config.newline, signal, this.config.timeout); const receipt = receiptOf(result, this.config.path, messageId); this.#log.write( 'debug', @@ -332,6 +391,42 @@ class SendmailTransport { ); return receipt; } + + // Waits for a place among the `maxConcurrent` processes, first come first + // served; an abort leaves the queue. + #acquire(signal) { + if (this.#running < this.config.maxConcurrent) { + this.#running++; + return undefined; + } + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(abortReason(signal)); + return; + } + const waiter = { resolve, cancelled: false, signal, onAbort: undefined }; + if (signal !== undefined) { + waiter.onAbort = () => { + waiter.cancelled = true; + reject(abortReason(signal)); + }; + signal.addEventListener('abort', waiter.onAbort, { once: true }); + } + this.#waiting.push(waiter); + }); + } + + // Hands the place on to the next waiter, or frees it. + #release() { + while (this.#waiting.size > 0) { + const next = this.#waiting.shift(); + if (next.cancelled) continue; + next.signal?.removeEventListener('abort', next.onAbort); + next.resolve(); + return; + } + this.#running--; + } } module.exports = { SendmailTransport }; diff --git a/tests/unit/sendmail.test.js b/tests/unit/sendmail.test.js index 3ebb1eb..df88afa 100644 --- a/tests/unit/sendmail.test.js +++ b/tests/unit/sendmail.test.js @@ -388,3 +388,98 @@ test('Sendmail: the options are checked when the transport is made', () => { } assert.strictEqual(new SendmailTransport().id, 'sendmail'); }); + +test('Sendmail: a process that does not exit in time is stopped, of unknown delivery', async () => { + // Before: a sendmail that hung held the send for good, unless the caller + // passed a signal. + const { spawn, calls } = fakeSpawn({ hang: true }); + const transport = new SendmailTransport({ spawn, timeout: 20 }); + const receipt = await transport.send(message()); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.errors[0].category, 'timeout'); + assert.strictEqual(receipt.deliveryUnknown, true); + assert.deepStrictEqual(calls[0].child.killed, ['SIGTERM']); + assert.throws(() => new SendmailTransport({ timeout: -1 }), /timeout/); + assert.throws(() => new SendmailTransport({ maxConcurrent: 0 }), /maxConcurrent/); +}); + +test('Sendmail: a process that ignores SIGTERM gets SIGKILL after a grace', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + t.after(() => t.mock.timers.reset()); + const children = []; + const spawn = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.killed = []; + child.kill = (signal) => { + child.killed.push(signal); + if (signal === 'SIGKILL') setImmediate(() => child.emit('close', null, signal)); + return true; + }; + children.push(child); + return child; + }; + const controller = new AbortController(); + const sending = new SendmailTransport({ spawn }).send(message(), { signal: controller.signal }); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(new Error('stop')); + await assert.rejects(sending, /stop/); + assert.deepStrictEqual(children[0].killed, ['SIGTERM']); + t.mock.timers.tick(5000); + assert.deepStrictEqual(children[0].killed, ['SIGTERM', 'SIGKILL']); +}); + +test('Sendmail: an exit with 0 before the whole message was read is not a success', async () => { + // Before: code 0 was success whatever had been written. + const spawn = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = () => true; + // It closes its input at once, and exits with 0. + setImmediate(() => { + child.stdin.destroy(); + setImmediate(() => child.emit('close', 0, null)); + }); + return child; + }; + const big = message({ content: { text: 'x'.repeat(4 * 1024 * 1024) } }); + const receipt = await new SendmailTransport({ spawn }).send(big); + assert.strictEqual(receipt.successful, false); + assert.strictEqual(receipt.deliveryUnknown, true); + assert.match(receipt.errorMessages[0], /before it read the whole message/); +}); + +test('Sendmail: no more than maxConcurrent processes run at once', async () => { + // Before: every concurrent send spawned its process. + let running = 0; + let most = 0; + const spawn = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = () => true; + running++; + most = Math.max(most, running); + child.stdin.resume(); + child.stdin.on('end', () => + setTimeout(() => { + running--; + child.emit('close', 0, null); + }, 5), + ); + return child; + }; + const transport = new SendmailTransport({ spawn, maxConcurrent: 2 }); + const receipts = await Promise.all([1, 2, 3, 4, 5].map(() => transport.send(message()))); + assert.ok(receipts.every((receipt) => receipt.successful)); + assert.strictEqual(most, 2); + // A send that waits for a place leaves the queue when it is aborted. + const controller = new AbortController(); + const held = [transport.send(message()), transport.send(message())]; + const queued = transport.send(message(), { signal: controller.signal }); + controller.abort(new Error('gone')); + await assert.rejects(queued, /gone/); + await Promise.all(held); +}); From 729bf6ca503d358978f75c43b04f2e6a158ffc5c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:43:22 +0300 Subject: [PATCH 070/143] feat(otel, logger): tell a failure of unknown delivery apart A receipt the provider may have taken, deliveryUnknown, is the one a sender reconciles from the provider's events instead of sending again, but ./otel counted it as any failure, and ./logger kept the flag inside the receipt of its line. The span and the count of messages carry mailer.delivery_unknown, and the email.failed line deliveryUnknown. Co-Authored-By: Claude Opus 5.5 --- docs/transports/logger.md | 5 +++++ docs/transports/otel.md | 7 +++++-- src/logger/logger-transport.js | 2 ++ src/otel/metrics.js | 8 +++++--- src/otel/opentelemetry-transport.js | 10 ++++++++-- tests/unit/logger.test.js | 24 ++++++++++++++++++++++++ tests/unit/otel.test.js | 26 +++++++++++++++++++++++++- 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/docs/transports/logger.md b/docs/transports/logger.md index 2e12d92..2a79618 100644 --- a/docs/transports/logger.md +++ b/docs/transports/logger.md @@ -158,6 +158,11 @@ count, and priority. Completion events additionally include duration and receipt error details. Instead of a category, give the transport a child logger with the bindings you want (`logger.child({ component: 'mail' })`). +A failed receipt of unknown delivery, which the provider may have taken, adds +`deliveryUnknown: true` to the fields of its `email.failed` line, so that the +messages to reconcile from the provider's events, rather than send again, can be +found by it. + ## Recording message content Messages are excluded from logs by default, and so are the addresses that the diff --git a/docs/transports/otel.md b/docs/transports/otel.md index c53fe0f..70d4e46 100644 --- a/docs/transports/otel.md +++ b/docs/transports/otel.md @@ -365,7 +365,7 @@ Prometheus turns into its own (`mailer_email_attempts_total`): |---|---|---|---| | `mailer.email.attempts` | counter | `{attempt}` | `mailer.transport.name`, `mailer.operation.name` (`send`, `send_batch`), `mailer.status` (`success`, `failure`, `partial_failure`), `error.type` on a failure | | `mailer.email.duration` | histogram | `s` | as `mailer.email.attempts` | -| `mailer.email.messages` | counter | `{message}` | `mailer.transport.name`, `email.priority`, `mailer.status` | +| `mailer.email.messages` | counter | `{message}` | `mailer.transport.name`, `email.priority`, `mailer.status`, and `mailer.delivery_unknown` (`true`) on a failure the provider may have taken | | `mailer.email.retries` | counter | `{retry}` | `mailer.transport.name`, `error.type` | | `mailer.email.message.size` | histogram | `By` | `mailer.transport.name`, `email.priority`, `email.content.type` | | `mailer.email.attachments` | histogram | `{attachment}` | as `mailer.email.message.size` | @@ -373,7 +373,10 @@ Prometheus turns into its own (`mailer_email_attempts_total`): A message is counted once in `mailer.email.messages`, when its receipt comes back; a send operation, of one message or a whole batch, once in `mailer.email.attempts`. -`error.type` is the category of the failure, from `errorClassifier`. The retries +`error.type` is the category of the failure, from `errorClassifier`. A failure +of unknown delivery, which the provider may have taken, carries +`mailer.delivery_unknown` on its span and its count of messages: those are the +messages to reconcile from the provider's events, not to send again. The retries are those of the wrapped transport, reported through `onRetry`; a span shows them as `retry` events and in `mailer.retry.count`. diff --git a/src/logger/logger-transport.js b/src/logger/logger-transport.js index 7393d30..71827eb 100644 --- a/src/logger/logger-transport.js +++ b/src/logger/logger-transport.js @@ -280,6 +280,8 @@ class LoggerTransport { errorMessages: failure.errorMessages, errors: failure.errors, retryable: failure.retryable, + // The provider may have taken it: to reconcile, not to send again. + ...(failure.deliveryUnknown === true && { deliveryUnknown: true }), provider: failure.provider ?? this.id, attempts: failure.attempts, receipt: failure, diff --git a/src/otel/metrics.js b/src/otel/metrics.js index b27854a..12622a9 100644 --- a/src/otel/metrics.js +++ b/src/otel/metrics.js @@ -97,13 +97,15 @@ class MetricsCollector { /** * Records the receipt of one message, once, with the priority of the - * message and whether it was sent. + * message, whether it was sent, and, for a failure the provider may have + * taken, `mailer.delivery_unknown`: the messages to reconcile. */ - recordMessageOutcome(transport, priority, successful) { + recordMessageOutcome(transport, priority, receipt) { this.messageCounter.add(1, { 'mailer.transport.name': transport, 'email.priority': priority, - 'mailer.status': successful ? 'success' : 'failure', + 'mailer.status': receipt.successful ? 'success' : 'failure', + ...(receipt.deliveryUnknown === true && { 'mailer.delivery_unknown': true }), }); } diff --git a/src/otel/opentelemetry-transport.js b/src/otel/opentelemetry-transport.js index 8e91fa3..0c3aa70 100644 --- a/src/otel/opentelemetry-transport.js +++ b/src/otel/opentelemetry-transport.js @@ -209,7 +209,7 @@ class OpenTelemetryTransport { // Delegate to wrapped transport const receipt = await deliver(this.withRetryRecording(options, span)); const duration = (performance.now() - startTime) / 1000; // Convert to seconds - this.metricsCollector?.recordMessageOutcome(this.transportName, priority, receipt.successful); + this.metricsCollector?.recordMessageOutcome(this.transportName, priority, receipt); // Record success/failure if (receipt.successful) { @@ -218,6 +218,7 @@ class OpenTelemetryTransport { } else { const errorCategory = this.classifyReceipt(receipt); span?.recordFailure(receipt.errorMessages); + if (receipt.deliveryUnknown === true) span?.setAttributes(DELIVERY_UNKNOWN); this.metricsCollector?.recordAttempt( this.transportName, 'send', @@ -298,7 +299,8 @@ class OpenTelemetryTransport { pending.length = 0; head = 0; } - metrics?.recordMessageOutcome(transportName, priority, receipt.successful); + metrics?.recordMessageOutcome(transportName, priority, receipt); + if (receipt.deliveryUnknown === true) span?.setAttributes(DELIVERY_UNKNOWN); if (receipt.successful) { successCount++; @@ -487,6 +489,10 @@ class OpenTelemetryTransport { } } +// A failure the provider may have taken: the message is to be reconciled +// from its events, not sent again. +const DELIVERY_UNKNOWN = { 'mailer.delivery_unknown': true }; + // Keeps the messages of a failed receipt while fewer than the bound are kept, // and counts all of them. const keepErrorMessages = (kept, count, receipt) => { diff --git a/tests/unit/logger.test.js b/tests/unit/logger.test.js index 99d11c0..e263502 100644 --- a/tests/unit/logger.test.js +++ b/tests/unit/logger.test.js @@ -186,3 +186,27 @@ test('Logger: generateId makes the message ID of a log-only receipt', async () = const transport = new LoggerTransport({ logger, generateId: () => '01JLOG' }); assert.strictEqual((await transport.send(message())).messageId, 'logger-01JLOG'); }); + +test('Logger: a failure of unknown delivery says so at the top of its line', async () => { + // Before: it was only inside the receipt of the line. + const lines = []; + const logger = { ...methods('debug', 'info', 'warn'), error: (fields) => lines.push(fields) }; + const unknown = { + successful: false, + errorMessages: ['The request timed out after it went out.'], + deliveryUnknown: true, + }; + const transport = new LoggerTransport({ + transport: { id: 'p', send: async () => unknown }, + logger, + }); + await transport.send(message()); + assert.strictEqual(lines[0].event, 'email.failed'); + assert.strictEqual(lines[0].deliveryUnknown, true); + const known = new LoggerTransport({ + transport: { id: 'p', send: async () => ({ successful: false, errorMessages: ['no'] }) }, + logger, + }); + await known.send(message()); + assert.strictEqual('deliveryUnknown' in lines[1], false); +}); diff --git a/tests/unit/otel.test.js b/tests/unit/otel.test.js index d04bf6a..307815c 100644 --- a/tests/unit/otel.test.js +++ b/tests/unit/otel.test.js @@ -6,7 +6,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { OpenTelemetryTransport } = require('../../src/otel/index.js'); const { MockMeterProvider, MockTracerProvider } = require('../helpers/otel/mock-otel-providers.js'); -const { createTestMessage } = require('../helpers/otel/test-config.js'); +const { createTestMessage, extractSpans } = require('../helpers/otel/test-config.js'); const providers = () => ({ tracerProvider: new MockTracerProvider(), @@ -453,3 +453,27 @@ test('OTel: random decides which operations get a span', async () => { assert.strictEqual(tracerProvider.getAllSpans().length, spans); } }); + +test('OTel: a failure of unknown delivery is told apart, on the span and the counter', async () => { + // Before: it was a failure like any other, though it is the one to + // reconcile from the provider's events rather than to send again. + const unknown = { + successful: false, + errorMessages: ['The connection closed after the end of DATA.'], + errors: [{ message: 'lost', category: 'network', retryable: false }], + deliveryUnknown: true, + }; + const otel = providers(); + const transport = new OpenTelemetryTransport(plain({}, unknown), otel); + await transport.send(createTestMessage()); + for await (const receipt of transport.sendMany([createTestMessage()])) assert.ok(receipt); + const spans = extractSpans(otel.tracerProvider); + assert.strictEqual(spans.length, 2); + for (const span of spans) assert.strictEqual(span.attributes['mailer.delivery_unknown'], true); + const counted = []; + for (const metric of otel.meterProvider.getAllRecords()) { + if (metric.name !== 'mailer.email.messages') continue; + for (const record of metric.records) counted.push(record.attributes['mailer.delivery_unknown']); + } + assert.deepStrictEqual(counted, [true, true]); +}); From 7964976f80bd91d2b20ac60ef19e1e7f7e475e56 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:47:54 +0300 Subject: [PATCH 071/143] fix(postmark, mailjet): split a batch by its bytes as well Postmark's /email/batch took 500 messages whatever their files, past the 50 MB Postmark takes a request, which refused the whole batch, and one failure of the request failed, or left of unknown delivery, all 500. Mailjet's requests were split by recipients alone, past its 15 MB. A batch is split by the bytes of its JSON as well, each request failing on its own. Co-Authored-By: Claude Opus 5.5 --- docs/transports/mailjet.md | 4 +-- docs/transports/postmark.md | 5 +-- scripts/size-budgets.json | 4 +-- src/internal/batch.js | 10 +++++- src/mailjet/mailjet-transport.js | 19 +++++++++--- src/postmark/postmark-transport.js | 49 ++++++++++++++++++++++-------- tests/unit/mailjet.test.js | 25 +++++++++++++++ tests/unit/postmark.test.js | 30 ++++++++++++++++++ 8 files changed, 122 insertions(+), 24 deletions(-) diff --git a/docs/transports/mailjet.md b/docs/transports/mailjet.md index 1e04793..59dee23 100644 --- a/docs/transports/mailjet.md +++ b/docs/transports/mailjet.md @@ -67,8 +67,8 @@ if (receipt.successful) console.log(receipt.messageId); // the MessageUUID ## Batches and errors `Messages` is Mailjet's batch: `sendMany()` sends requests of at most 50 -recipients, as Mailjet counts them over the whole request, reading its input a -batch at a time. When one message of a request fails, Mailjet answers 400 with +recipients, as Mailjet counts them over the whole request, and 15 MB, reading +its input a batch at a time. When one message of a request fails, Mailjet answers 400 with the answers of all of them, so each message gets the receipt of its own answer: those that went succeed, the others fail with the code Mailjet gives (`mailjet.send-0003`) and the fields it names. diff --git a/docs/transports/postmark.md b/docs/transports/postmark.md index bb9ea71..1edc690 100644 --- a/docs/transports/postmark.md +++ b/docs/transports/postmark.md @@ -71,8 +71,9 @@ the message is `deliveryUnknown`. ## Batches -`sendMany()` sends up to 500 messages a request through `/email/batch`, reading -its input a batch at a time. Each message has the receipt of its own answer, so an +`sendMany()` sends up to 500 messages and 50 MB a request through `/email/batch`, +reading its input a batch at a time: messages with files go in as many requests +as their size takes, each of which fails, or is of unknown delivery, on its own. Each message has the receipt of its own answer, so an inactive recipient fails its message alone; a message that cannot be converted fails alone too. diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index a6ccd66..45fdd8f 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -15,10 +15,10 @@ "./maileroo": 6.5, "./lettermint": 6.5, "./plunk": 6, - "./postmark": 7, + "./postmark": 7.5, "./brevo": 7, "./mailjet": 8, - "./mailersend": 7, + "./mailersend": 7.5, "./sparkpost": 8, "./zeptomail": 7, "./smtp2go": 7.5, diff --git a/src/internal/batch.js b/src/internal/batch.js index def24af..f2d4846 100644 --- a/src/internal/batch.js +++ b/src/internal/batch.js @@ -83,4 +83,12 @@ const sendConcurrently = async function* (messages, limit, send, signal) { } }; -module.exports = { chunks, readItems, sendConcurrently }; +/** + * The bytes a value takes as JSON, with the comma that follows it in an + * array: what it adds to the body of a batch request. + * + * @internal + */ +const jsonBytes = (value) => Buffer.byteLength(JSON.stringify(value)) + 1; + +module.exports = { chunks, jsonBytes, readItems, sendConcurrently }; diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index 8968c30..c9acb23 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -8,7 +8,7 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); -const { chunks } = require('../internal/batch.js'); +const { chunks, jsonBytes } = require('../internal/batch.js'); const { ProviderApiError, convertBatch, @@ -22,6 +22,9 @@ const { createMailjetConfig } = require('./config.js'); const { createMailjetClient } = require('./http-client.js'); const { MAX_RECIPIENTS, convertMessage, recipientCount } = require('./message-converter.js'); +// Mailjet takes 15 MB a request at most, attachments included. +const MAX_REQUEST_BYTES = 15 * 1000 * 1000; + // Mailjet names every problem of a message at once, with the fields it is // about. const describeErrors = (errors) => { @@ -134,8 +137,8 @@ class MailjetTransport { } /** - * Sends messages in requests of up to 50 recipients, each message with the - * receipt of its own answer. + * Sends messages in requests of up to 50 recipients and 15 MB, each message + * with the receipt of its own answer. * * @param messages The messages. * @param options Optional abort signal and retry callback. @@ -154,16 +157,22 @@ class MailjetTransport { const receipts = []; let run = []; let recipients = 0; + let bytes = 16; for (let i = 0; i < batch.length; i++) { if (typeof converted.slots[i] !== 'number') continue; const count = recipientCount(batch[i]); - if (run.length > 0 && recipients + count > MAX_RECIPIENTS) { + const entry = converted.payloads[converted.slots[i]]; + const size = jsonBytes(entry); + const full = recipients + count > MAX_RECIPIENTS || bytes + size > MAX_REQUEST_BYTES; + if (run.length > 0 && full) { for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); run = []; recipients = 0; + bytes = 16; } - run.push(converted.payloads[converted.slots[i]]); + run.push(entry); recipients += count; + bytes += size; } if (run.length > 0) { for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index 598f054..d455b8e 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -9,7 +9,7 @@ // sently's JSON parse before the status check is gone with the shared client. const { createFailedReceipt } = require('../core/receipt.js'); -const { chunks } = require('../internal/batch.js'); +const { chunks, jsonBytes } = require('../internal/batch.js'); const { convertBatch, isCallerAbort, mergeBatch, providerFailure } = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { verifyByRequest } = require('../internal/verify.js'); @@ -19,6 +19,10 @@ const { convertMessage } = require('./message-converter.js'); // The most messages one request of `/email/batch` takes. const MAX_BATCH_SIZE = 500; +// Postmark takes a batch of 50 MB at most, attachments included: a batch of +// messages with files goes in as many requests as that takes, each failing, +// or of unknown delivery, on its own rather than with all 500. +const MAX_BATCH_BYTES = 50 * 1000 * 1000; // The receipt of an answer of Postmark: an `ErrorCode` other than 0 fails it, // though it came in a 200. @@ -81,8 +85,9 @@ class PostmarkTransport { } /** - * Sends messages with `/email/batch`, 500 a request, reading the input a - * batch at a time; each message has the receipt of its own answer. + * Sends messages with `/email/batch`, 500 and 50 MB a request at most, + * reading the input a batch at a time; each message has the receipt of its + * own answer. * * @param messages The messages. * @param options Optional abort signal and retry callback. @@ -102,19 +107,39 @@ class PostmarkTransport { yield* mergeBatch(converted.slots, () => undefined); continue; } - let answers; - let failure; - try { - const answer = await this.httpClient.sendBatch(converted.payloads, options); - answers = Array.isArray(answer.json) ? answer.json : []; - } catch (error) { - if (isCallerAbort(error, signal)) throw error; - failure = providerFailure('postmark', error); + const receipts = []; + let run = []; + let bytes = 2; + for (const payload of converted.payloads) { + const size = jsonBytes(payload); + if (run.length > 0 && bytes + size > MAX_BATCH_BYTES) { + for (const receipt of await this.#sendRun(run, options)) receipts.push(receipt); + run = []; + bytes = 2; + } + run.push(payload); + bytes += size; } - yield* mergeBatch(converted.slots, (index) => failure ?? receiptOf(answers[index])); + for (const receipt of await this.#sendRun(run, options)) receipts.push(receipt); + yield* mergeBatch(converted.slots, (index) => receipts[index]); } } + // Sends the payloads of one request: a receipt each, in order. + async #sendRun(payloads, options) { + const receipts = []; + try { + const answer = await this.httpClient.sendBatch(payloads, options); + const answers = Array.isArray(answer.json) ? answer.json : []; + for (let i = 0; i < payloads.length; i++) receipts.push(receiptOf(answers[i])); + } catch (error) { + if (isCallerAbort(error, options?.signal)) throw error; + const failure = providerFailure('postmark', error); + for (let i = 0; i < payloads.length; i++) receipts.push(failure); + } + return receipts; + } + /** * Verifies the server token by reading its server (`GET /server`). * diff --git a/tests/unit/mailjet.test.js b/tests/unit/mailjet.test.js index cde05b7..5580654 100644 --- a/tests/unit/mailjet.test.js +++ b/tests/unit/mailjet.test.js @@ -225,3 +225,28 @@ test('Mailjet: verify reads the profile, the options are checked, the secret nev controller.abort(new Error('stop')); await assert.rejects(made.send(message(), { signal: controller.signal }), /stop/); }); + +test('Mailjet: a batch past 15 MB goes in as many requests as that takes', async (t) => { + // Before: only the recipients split a batch, so messages with files went in + // one request past the 15 MB Mailjet takes. + const requests = stubFetch(t, (request) => + json({ Messages: request.json.Messages.map((_entry, index) => success(`u-${index}`)) }), + ); + const file = () => ({ + filename: 'big.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: new Uint8Array(8 * 1000 * 1000), + }); + const receipts = []; + for await (const receipt of transport().sendMany([ + message({ attachments: [file()] }), + message({ attachments: [file()] }), + ])) { + receipts.push(receipt); + } + assert.ok(receipts.every((receipt) => receipt.successful)); + assert.strictEqual(requests.length, 2); + for (const request of requests) assert.ok(Buffer.byteLength(request.body) <= 15 * 1000 * 1000); +}); diff --git a/tests/unit/postmark.test.js b/tests/unit/postmark.test.js index 13f0caf..4a31aba 100644 --- a/tests/unit/postmark.test.js +++ b/tests/unit/postmark.test.js @@ -252,3 +252,33 @@ test('Postmark: an abort rejects with its reason', async (t) => { const sending = transport().sendMany([message()], { signal: controller.signal }); await assert.rejects(sending.next(), /stop/); }); + +test('Postmark: a batch past 50 MB goes in as many requests as that takes', async (t) => { + // Before: 500 messages went in one request whatever their files, which + // Postmark refuses past 50 MB, and one failure failed them all. + const requests = stubFetch(t, (request) => + json(request.json.map((_payload, index) => ({ ErrorCode: 0, MessageID: `pm-${index}` }))), + ); + const file = () => ({ + filename: 'big.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: new Uint8Array(20 * 1000 * 1000), + }); + const receipts = []; + for await (const receipt of transport().sendMany([ + message({ attachments: [file()] }), + message({ attachments: [file()] }), + message(), + ])) { + receipts.push(receipt); + } + assert.ok(receipts.every((receipt) => receipt.successful)); + assert.strictEqual(requests.length, 2); + assert.deepStrictEqual( + requests.map((request) => request.json.length), + [1, 2], + ); + for (const request of requests) assert.ok(Buffer.byteLength(request.body) <= 50 * 1000 * 1000); +}); From b95da8b956aec8a36fc4f70aa5d7e8bfefd67f36 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:49:57 +0300 Subject: [PATCH 072/143] test: the delivery and redaction contracts over every provider of 1.1 The matrices of delivery.test.js (a 2xx unread, a body that never ends, 4 MiB, a redirect, Retry-After, a lost request, onRetry, the fetch given) and secrets.test.js held the providers ported from upyo alone; each of the ten providers of 1.1 checked a part of the same in its own file. They run over all eighteen, and secrets are looked for by their values, in the transports of 1.1, sendmail with DKIM and the five decorators, so that a field of a new name cannot leak unnoticed. Co-Authored-By: Claude Opus 5.5 --- tests/unit/delivery.test.js | 74 +++++++++++++++++++++++++++++++++++-- tests/unit/secrets.test.js | 62 +++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 4 deletions(-) diff --git a/tests/unit/delivery.test.js b/tests/unit/delivery.test.js index b8e9f9b..36e745b 100644 --- a/tests/unit/delivery.test.js +++ b/tests/unit/delivery.test.js @@ -12,6 +12,16 @@ const { ResendTransport } = require('../../src/resend/index.js'); const { SendGridTransport } = require('../../src/sendgrid/index.js'); const { SesTransport } = require('../../src/ses/index.js'); const { JmapTransport } = require('../../src/jmap/index.js'); +const { AzureTransport } = require('../../src/azure/index.js'); +const { BrevoTransport } = require('../../src/brevo/index.js'); +const { CloudflareTransport } = require('../../src/cloudflare/index.js'); +const { MailerSendTransport } = require('../../src/mailersend/index.js'); +const { MailjetTransport } = require('../../src/mailjet/index.js'); +const { PostmarkTransport } = require('../../src/postmark/index.js'); +const { ScalewayTransport } = require('../../src/scaleway/index.js'); +const { Smtp2goTransport } = require('../../src/smtp2go/index.js'); +const { SparkPostTransport } = require('../../src/sparkpost/index.js'); +const { ZeptoMailTransport } = require('../../src/zeptomail/index.js'); const message = () => createMessage({ @@ -21,10 +31,12 @@ const message = () => content: { text: 'Hi' }, }); -// One retry: a provider that resends shows two requests, as Resend and Lettermint do with -// the idempotency key they always send. +// One retry: a provider that resends shows two requests, as Resend, Lettermint, Brevo and +// Azure do with the idempotency key they always send. const options = { retries: 1 }; -const expectedRequests = (name) => (name === 'Resend' || name === 'Lettermint' ? 2 : 1); +const WITH_KEYS = new Set(['Resend', 'Lettermint', 'Brevo', 'Azure']); +const expectedRequests = (name) => (WITH_KEYS.has(name) ? 2 : 1); +const AZURE_KEY = Buffer.from('azure-access-key').toString('base64'); const providers = [ [ 'SES', @@ -53,6 +65,60 @@ const providers = [ ], ['Plunk', 200, (extra) => new PlunkTransport({ ...options, ...extra, apiKey: 'key' })], ]; +// The providers of 1.1, on the shared client of src/internal/api.js: the same +// contracts, every one of them. +const providersOf11 = [ + [ + 'Postmark', + 200, + (extra) => new PostmarkTransport({ ...options, ...extra, serverToken: 'token' }), + ], + ['Brevo', 201, (extra) => new BrevoTransport({ ...options, ...extra, apiKey: 'key' })], + [ + 'Mailjet', + 200, + (extra) => new MailjetTransport({ ...options, ...extra, apiKey: 'a', apiSecret: 'b' }), + ], + ['MailerSend', 202, (extra) => new MailerSendTransport({ ...options, ...extra, apiKey: 'key' })], + ['SparkPost', 200, (extra) => new SparkPostTransport({ ...options, ...extra, apiKey: 'key' })], + ['ZeptoMail', 201, (extra) => new ZeptoMailTransport({ ...options, ...extra, token: 'token' })], + ['SMTP2GO', 200, (extra) => new Smtp2goTransport({ ...options, ...extra, apiKey: 'key' })], + [ + 'Azure', + 202, + (extra) => + new AzureTransport({ + ...options, + ...extra, + connectionString: `endpoint=https://contoso.communication.azure.com/;accesskey=${AZURE_KEY}`, + }), + ], + [ + 'Scaleway', + 200, + (extra) => + new ScalewayTransport({ + ...options, + ...extra, + secretKey: 'key', + projectId: '11111111-2222-4333-8444-555555555555', + }), + ], + [ + 'Cloudflare', + 200, + (extra) => + new CloudflareTransport({ + ...options, + ...extra, + accountId: '023e105f4ecef8ad9ca31a8372d0c353', + apiToken: 'token', + }), + ], +]; +// The clients ported from upyo, which expose sendMessage of their own. +const portedProviders = providers.slice(); +for (const provider of providersOf11) providers.push(provider); // A 2xx whose body cannot be read: the connection broke after the headers. const unreadable = (status) => @@ -218,7 +284,7 @@ test('Delivery: the failure of fetch stays the cause of the provider error', asy t.mock.method(globalThis, 'fetch', async () => { throw refused; }); - for (const [name, , create] of providers) { + for (const [name, , create] of portedProviders) { const body = name === 'Mailgun' ? new FormData() : {}; const error = await create({ retries: 0 }) .httpClient.sendMessage(body, undefined) diff --git a/tests/unit/secrets.test.js b/tests/unit/secrets.test.js index 78cd1d6..1085555 100644 --- a/tests/unit/secrets.test.js +++ b/tests/unit/secrets.test.js @@ -115,3 +115,65 @@ test('Secrets: redaction copies arrays and records, keeps built-in values, names assert.strictEqual(copy.empty, null); assert.deepStrictEqual(redactSecrets({ password: undefined }), { password: undefined }); }); + +test('Secrets: the transports and decorators of 1.1 show no credentials either', () => { + // The redaction goes by the names of the fields; this goes by the values, + // so that a field of a new name does not leak unnoticed. + const { AzureTransport } = require('../../src/azure/index.js'); + const { BrevoTransport } = require('../../src/brevo/index.js'); + const { CloudflareTransport } = require('../../src/cloudflare/index.js'); + const { MailerSendTransport } = require('../../src/mailersend/index.js'); + const { MailjetTransport } = require('../../src/mailjet/index.js'); + const { PostmarkTransport } = require('../../src/postmark/index.js'); + const { ScalewayTransport } = require('../../src/scaleway/index.js'); + const { SendmailTransport } = require('../../src/sendmail/index.js'); + const { Smtp2goTransport } = require('../../src/smtp2go/index.js'); + const { SparkPostTransport } = require('../../src/sparkpost/index.js'); + const { ZeptoMailTransport } = require('../../src/zeptomail/index.js'); + const { CircuitBreakerTransport } = require('../../src/breaker/index.js'); + const { HooksTransport } = require('../../src/hooks/index.js'); + const { IdempotencyTransport } = require('../../src/idempotency/index.js'); + const { RateLimitTransport } = require('../../src/ratelimit/index.js'); + const { TeeTransport } = require('../../src/tee/index.js'); + const base64Secret = Buffer.from(SECRET).toString('base64'); + const dkim = { + signatures: [ + { signingDomain: 'example.com', selector: 's', privateKey: TEST_DKIM_PRIVATE_KEY }, + ], + }; + const providers = [ + new PostmarkTransport({ serverToken: SECRET }), + new BrevoTransport({ apiKey: SECRET }), + new MailjetTransport({ apiKey: 'public', apiSecret: SECRET }), + new MailerSendTransport({ apiKey: SECRET }), + new SparkPostTransport({ apiKey: SECRET }), + new ZeptoMailTransport({ token: SECRET }), + new Smtp2goTransport({ apiKey: SECRET }), + new AzureTransport({ + endpoint: 'https://contoso.communication.azure.com', + accessKey: base64Secret, + }), + new AzureTransport({ + connectionString: `endpoint=https://contoso.communication.azure.com/;accesskey=${base64Secret}`, + }), + new ScalewayTransport({ secretKey: SECRET, projectId: '11111111-2222-4333-8444-555555555555' }), + new CloudflareTransport({ accountId: '023e105f4ecef8ad9ca31a8372d0c353', apiToken: SECRET }), + new SendmailTransport({ dkim }), + ]; + const decorated = [ + new CircuitBreakerTransport(new PostmarkTransport({ serverToken: SECRET })), + new RateLimitTransport(new BrevoTransport({ apiKey: SECRET }), { limit: 1 }), + new IdempotencyTransport(new SparkPostTransport({ apiKey: SECRET })), + new HooksTransport(new ZeptoMailTransport({ token: SECRET })), + new TeeTransport(new Smtp2goTransport({ apiKey: SECRET }), { + mirrors: [new CloudflareTransport({ accountId: 'a'.repeat(32), apiToken: SECRET })], + }), + ]; + for (const transport of [...providers, ...decorated]) { + for (const shown of [inspect(transport, { depth: Infinity }), JSON.stringify(transport)]) { + for (const secret of [SECRET, base64Secret, KEY_LINE]) { + assert.ok(!shown.includes(secret), `${transport.constructor.name}: ${shown}`); + } + } + } +}); From 1b01155d462c5c45eb9ffa626bde36cc591c717f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:52:28 +0300 Subject: [PATCH 073/143] refactor: check every clock option, keep the loops of pool and retry private The configs of azure, sendgrid, mailersend and smtp2go took any `now`, which failed at the first send that read it, where the others refuse one that is not a function when the transport is made. PoolTransport.deliver and RetryTransport.withRetries, which the port made public methods that no d.ts declares, are private. Co-Authored-By: Claude Opus 5.5 --- src/azure/config.js | 3 ++- src/mailersend/config.js | 3 ++- src/pool/pool-transport.js | 6 +++--- src/retry/retry-transport.js | 6 +++--- src/sendgrid/config.js | 3 ++- src/smtp2go/config.js | 3 ++- tests/unit/internal.test.js | 29 +++++++++++++++++++++++++++++ 7 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/azure/config.js b/src/azure/config.js index c1262c1..2eb66b6 100644 --- a/src/azure/config.js +++ b/src/azure/config.js @@ -11,6 +11,7 @@ const { checkRequestOptions, checkSecureUrl, } = require('../internal/config.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { isAccessKey, parseConnectionString } = require('./sign.js'); const API_VERSION = /^\d{4}-\d{2}-\d{2}(-preview)?$/; @@ -68,7 +69,7 @@ const createAzureConfig = (config = {}) => { fetch: config.fetch, generateId: config.generateId, logger: config.logger, - now: config.now ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? Date.now, }; checkConcurrency(resolved.concurrency); checkRequestOptions(resolved.timeout, resolved.retries); diff --git a/src/mailersend/config.js b/src/mailersend/config.js index 864103b..fd775da 100644 --- a/src/mailersend/config.js +++ b/src/mailersend/config.js @@ -3,6 +3,7 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind const { checkCredential } = require('../internal/api.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { checkConcurrency, checkOptionalBoolean, @@ -32,7 +33,7 @@ const createMailerSendConfig = (config = {}) => { fetch: config.fetch, generateId: config.generateId, logger: config.logger, - now: config.now ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? Date.now, }; checkConcurrency(resolved.concurrency); checkRequestOptions(resolved.timeout, resolved.retries); diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 229c140..97cef92 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -100,7 +100,7 @@ class PoolTransport { */ async send(message, options) { const pinned = pinMessageId(message, this.config.generateId); - return this.deliver( + return this.#deliver( pinned, this.config.transports, (transport, sendOptions) => transport.send(pinned, sendOptions), @@ -128,7 +128,7 @@ class PoolTransport { entries.push({ ...entry, enabled: canSend, selector: undefined }); } if (enabled === 0) return refuseRaw(this); - return this.deliver( + return this.#deliver( message, entries, (transport, sendOptions) => transport.sendRaw(message, sendOptions), @@ -164,7 +164,7 @@ class PoolTransport { * Sends a message through the transports the strategy picks, one after * another until one sends it or failover has to stop. */ - async deliver(message, entries, sendOnce, options) { + async #deliver(message, entries, sendOnce, options) { const attemptedIndices = new Set(); // Failover stops when another transport could only deliver the message a // second time (delivery unknown) or would refuse it too (the message diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index 4611dd8..1d74d68 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -64,7 +64,7 @@ class RetryTransport { options?.signal?.throwIfAborted(); const pinned = pinMessageId(message, this.config.generateId); - return this.withRetries( + return this.#withRetries( (sendOptions) => this.wrappedTransport.send(pinned, sendOptions), options, ); @@ -84,7 +84,7 @@ class RetryTransport { async sendRaw(message, options) { options?.signal?.throwIfAborted(); if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); - return this.withRetries( + return this.#withRetries( (sendOptions) => this.wrappedTransport.sendRaw(message, sendOptions), options, ); @@ -107,7 +107,7 @@ class RetryTransport { * @param options Optional transport options. * @returns The final delivery receipt. */ - async withRetries(sendOnce, options) { + async #withRetries(sendOnce, options) { let lastThrownError; for (let attempt = 1; attempt <= this.config.maxAttempts; attempt++) { options?.signal?.throwIfAborted(); diff --git a/src/sendgrid/config.js b/src/sendgrid/config.js index 40038c1..702587a 100644 --- a/src/sendgrid/config.js +++ b/src/sendgrid/config.js @@ -2,6 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { checkConcurrency, checkRequestOptions, checkSecureUrl } = require('../internal/config.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { checkValidateSsl } = require('../internal/validate-ssl.js'); /** @@ -32,7 +33,7 @@ const createSendGridConfig = (config) => { openTracking: config.openTracking ?? true, subscriptionTracking: config.subscriptionTracking ?? false, googleAnalytics: config.googleAnalytics ?? false, - now: config.now ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? Date.now, }; checkRequestOptions(resolved.timeout, resolved.retries); checkConcurrency(resolved.concurrency); diff --git a/src/smtp2go/config.js b/src/smtp2go/config.js index 1828564..ea25ea4 100644 --- a/src/smtp2go/config.js +++ b/src/smtp2go/config.js @@ -3,6 +3,7 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind const { checkCredential } = require('../internal/api.js'); +const { checkOptionalFunction } = require('../internal/options.js'); const { checkConcurrency, checkOptionalBoolean, @@ -49,7 +50,7 @@ const createSmtp2goConfig = (config = {}) => { fetch: config.fetch, generateId: config.generateId, logger: config.logger, - now: config.now ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? Date.now, }; checkConcurrency(resolved.concurrency); checkRequestOptions(resolved.timeout, resolved.retries); diff --git a/tests/unit/internal.test.js b/tests/unit/internal.test.js index 982ad4e..c828803 100644 --- a/tests/unit/internal.test.js +++ b/tests/unit/internal.test.js @@ -546,3 +546,32 @@ test('Internal: every provider that took validateSsl refuses false', () => { assert.ok(new Transport(config)); } }); + +test('Config: a clock that is not a function is refused where a provider takes one', () => { + // Before: azure, sendgrid, mailersend and smtp2go took any `now`, and + // failed at the first send that read it. + const { AzureTransport } = require('../../src/azure/index.js'); + const { MailerSendTransport } = require('../../src/mailersend/index.js'); + const { SendGridTransport } = require('../../src/sendgrid/index.js'); + const { Smtp2goTransport } = require('../../src/smtp2go/index.js'); + const key = Buffer.from('k').toString('base64'); + const makers = [ + () => + new AzureTransport({ endpoint: 'https://a.communication.azure.com', accessKey: key, now: 1 }), + () => new MailerSendTransport({ apiKey: 'k', now: 1 }), + () => new SendGridTransport({ apiKey: 'k', now: 1 }), + () => new Smtp2goTransport({ apiKey: 'k', now: 1 }), + ]; + for (const make of makers) assert.throws(make, /now must be a function/); +}); + +test('Decorators: pool and retry keep their loops to themselves', () => { + const { PoolTransport } = require('../../src/pool/index.js'); + const { RetryTransport } = require('../../src/retry/index.js'); + const inner = { id: 'x', send: async () => ({ successful: true, messageId: 'm' }) }; + assert.strictEqual( + 'deliver' in new PoolTransport({ strategy: 'priority', transports: [{ transport: inner }] }), + false, + ); + assert.strictEqual('withRetries' in new RetryTransport(inner), false); +}); From bc6f9bc9cf2ff819af92e87ccfa5055cc8cc16d4 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:53:59 +0300 Subject: [PATCH 074/143] feat(pool): verifyAll, how each transport did verify() of a pool passes when any of its transports verifies, which is what sending needs, but a key a provider revoked did not show while another still sent. verifyAll() verifies every enabled transport at once and gives each one's status, verified, failed with its error, or unverifiable, for a health check. The note that JMAP does not call onRetry is gone: it calls it for the reads it retries. The budgets of ./sendgrid and ./azure follow the check of their clock. Co-Authored-By: Claude Opus 5.5 --- index.d.ts | 3 ++- scripts/size-budgets.json | 4 ++-- src/pool/index.d.ts | 22 +++++++++++++++++- src/pool/pool-transport.js | 37 ++++++++++++++++++++++++++++++ tests/types/pool.test-d.ts | 5 +++++ tests/unit/pool.test.js | 46 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 4 deletions(-) diff --git a/index.d.ts b/index.d.ts index bbc89d2..60cbce3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1272,7 +1272,8 @@ export interface TransportOptions { * provider that retries its own requests (`retries`), and by the `./retry` * decorator. The decorators pass it on to the transport they wrap, so one * callback hears the retries of every level. An error it throws rejects - * the send. The JMAP transport does not call it yet. + * the send. The JMAP transport calls it for the reads it retries, such as + * its discovery; it never sends a submission again. */ readonly onRetry?: (event: RetryEvent) => void; } diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 45fdd8f..74b8ae9 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -9,7 +9,7 @@ "./jmap": 13, "./ses": 8, "./resend": 6.5, - "./sendgrid": 6.5, + "./sendgrid": 7, "./mailgun": 6.5, "./mailtrap": 7, "./maileroo": 6.5, @@ -22,7 +22,7 @@ "./sparkpost": 8, "./zeptomail": 7, "./smtp2go": 7.5, - "./azure": 8, + "./azure": 8.5, "./scaleway": 7.5, "./cloudflare": 8, "./pool": 3, diff --git a/src/pool/index.d.ts b/src/pool/index.d.ts index 119ab13..3dd4780 100644 --- a/src/pool/index.d.ts +++ b/src/pool/index.d.ts @@ -156,6 +156,20 @@ export interface ResolvedTransportEntry { * @throws {Error} If the configuration is invalid. */ +/** + * How one transport of a pool did in {@link PoolTransport.verifyAll}. + */ +export interface PoolVerification { + /** The id of the transport. */ + readonly id: string; + /** Its place in the pool's `transports`. */ + readonly index: number; + /** `verified`, `failed`, or `unverifiable` for one without `verify()`. */ + readonly status: 'verified' | 'failed' | 'unverifiable'; + /** Why it failed. */ + readonly error?: unknown; +} + /** * Pool transport that combines multiple transports with various load balancing * and failover strategies. @@ -251,9 +265,15 @@ export declare class PoolTransport */ sendRaw(message: RawMessage, options?: TransportOptions): Promise>; /** - * Verifies the enabled transports that can verify: it resolves when one of them verifies, and throws an `AggregateError` of their errors when none does; it has nothing to check when none can verify. + * Verifies the enabled transports that can verify: it resolves when one of them verifies, and throws an `AggregateError` of their errors when none does; it has nothing to check when none can verify. {@link PoolTransport.verifyAll} tells how each did. */ verify(options?: TransportOptions): Promise; + /** + * Verifies every enabled transport, all at once, and tells how each did, + * for a health check: a key a provider revoked shows while the others + * still send. It throws only for the caller's abort. + */ + verifyAll(options?: TransportOptions): Promise; /** * Sends multiple email messages using the pool strategy. * diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 97cef92..68996fd 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -136,10 +136,47 @@ class PoolTransport { ); } + /** + * Verifies every enabled transport, all at once, and tells how each did: + * `verified`, `failed` with its error, or `unverifiable` for one without + * `verify()`. It never throws for a transport, so that a key a provider + * revoked shows while the others still send. + * + * @param options Optional abort signal. + * @returns `{ id, index, status, error? }` for each enabled transport. + */ + async verifyAll(options) { + options?.signal?.throwIfAborted(); + const pending = []; + const results = []; + for (let index = 0; index < this.config.transports.length; index++) { + const entry = this.config.transports[index]; + if (!entry.enabled) continue; + const result = { id: entry.transport.id, index, status: 'unverifiable' }; + results.push(result); + if (!isVerifiableTransport(entry.transport)) continue; + pending.push( + (async () => { + try { + await entry.transport.verify(options); + result.status = 'verified'; + } catch (error) { + result.status = 'failed'; + result.error = error; + } + })(), + ); + } + await Promise.all(pending); + options?.signal?.throwIfAborted(); + return results; + } + /** * Verifies the enabled transports that can verify, all at once: the pool * can send when one of them can. It throws an `AggregateError` of their * errors when none verifies, and has nothing to check when none can verify. + * {@link PoolTransport.verifyAll} tells how each did. */ async verify(options) { options?.signal?.throwIfAborted(); diff --git a/tests/types/pool.test-d.ts b/tests/types/pool.test-d.ts index 28861fb..7820b7c 100644 --- a/tests/types/pool.test-d.ts +++ b/tests/types/pool.test-d.ts @@ -3,6 +3,7 @@ import { createMessage, type Receipt } from '../../index.js'; import { MockTransport } from '../../src/mock/index.js'; import { PoolTransport, + type PoolVerification, PriorityStrategy, RoundRobinStrategy, SelectorStrategy, @@ -51,3 +52,7 @@ expectError( continueOnSuccess: true, }), ); + +const verified = await pool.verifyAll(); +expectType(verified); +expectType<'verified' | 'failed' | 'unverifiable'>(verified[0].status); diff --git a/tests/unit/pool.test.js b/tests/unit/pool.test.js index 3b73618..ca2f83f 100644 --- a/tests/unit/pool.test.js +++ b/tests/unit/pool.test.js @@ -258,3 +258,49 @@ test('Pool: continueOnSuccess, which would deliver a message twice, is refused', ); assert.ok(new PoolTransport({ strategy: 'priority', transports, continueOnSuccess: false })); }); + +test('Pool: verifyAll tells how each transport did, a revoked key among them', async () => { + // Before: verify() passed when any transport verified, so a key a provider + // had revoked did not show while another still sent. + const ok = { + id: 'ok', + send: async () => ({ successful: true, messageId: 'm' }), + verify: async () => {}, + }; + const revoked = { + id: 'revoked', + send: async () => ({ successful: true, messageId: 'm' }), + verify: async () => { + throw new Error('401 Unauthorized'); + }, + }; + const plain = { id: 'plain', send: async () => ({ successful: true, messageId: 'm' }) }; + const off = { + id: 'off', + send: async () => ({ successful: true, messageId: 'm' }), + verify: async () => {}, + }; + const pool = new PoolTransport({ + strategy: 'priority', + transports: [ + { transport: ok }, + { transport: revoked }, + { transport: plain }, + { transport: off, enabled: false }, + ], + }); + await pool.verify(); + const results = await pool.verifyAll(); + assert.deepStrictEqual( + results.map((result) => [result.id, result.index, result.status]), + [ + ['ok', 0, 'verified'], + ['revoked', 1, 'failed'], + ['plain', 2, 'unverifiable'], + ], + ); + assert.match(results[1].error.message, /401/); + const controller = new AbortController(); + controller.abort(new Error('stop')); + await assert.rejects(pool.verifyAll({ signal: controller.signal }), /stop/); +}); From 705411a92c6d953b8b57c7591e5bd474a364e926 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:55:23 +0300 Subject: [PATCH 075/143] feat(webhooks): onVerify, each request a verifier takes or refuses A verifier told nothing of what it refused: a forged flood, a secret that no longer matches or a store that fails showed only in the answers of the endpoint. onVerify hears each request, with its provider, whether it was taken, refused for a reason, or failed otherwise, the events taken and the time it took, for a metric; one that throws is a process warning. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 1 + scripts/size-budgets.json | 2 +- src/webhooks/azure.js | 5 +-- src/webhooks/brevo.js | 5 +-- src/webhooks/index.d.ts | 35 +++++++++++++++++++++ src/webhooks/lettermint.js | 5 +-- src/webhooks/maileroo.js | 5 +-- src/webhooks/mailersend.js | 7 +++-- src/webhooks/mailgun.js | 5 +-- src/webhooks/mailjet.js | 5 +-- src/webhooks/mailtrap.js | 5 +-- src/webhooks/postmark.js | 5 +-- src/webhooks/request.js | 48 +++++++++++++++++++++++++++++ src/webhooks/resend.js | 5 +-- src/webhooks/sendgrid.js | 5 +-- src/webhooks/smtp2go.js | 7 +++-- src/webhooks/sns.js | 5 +-- src/webhooks/sparkpost.js | 5 +-- tests/types/webhooks.test-d.ts | 8 +++++ tests/unit/webhooks-request.test.js | 41 ++++++++++++++++++++++++ 20 files changed, 178 insertions(+), 31 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index d51eef0..06f8700 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -143,6 +143,7 @@ const replayStore = { | `toleranceMs` | 5 minutes (an hour for SES) | How far from now a request may be signed. | | `maxBodyBytes` | 5 MiB | The longest body taken. | | `now` | `Date.now` | The clock. | +| `onVerify` | | Hears each request: `{ provider, status, reason, events, durationMs }`, `status` being `taken`, `refused` or `error`, for a metric of refusals by `reason`. | ## Amazon SES diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 74b8ae9..4030f6b 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4.5, "./render": 3, - "./webhooks": 13.5, + "./webhooks": 14.5, "./smtp": 23, "./sendmail": 11.5, "./jmap": 13, diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index ac15de5..908aef6 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -12,6 +12,7 @@ const { deliveryOf, headerOf, isRecord, + observed, parseJsonOf, resolveCommonOptions, stringOf, @@ -116,7 +117,7 @@ const createAzureWebhookVerifier = (options = {}) => { const tokenHeader = options.header ?? 'authorization'; const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { - async verify(request) { + verify: observed(common, 'azure', async (request) => { const bytes = bodyBytesOf(request, common); const kind = headerOf(request, 'aeg-event-type'); // A validation carries no event, and answering it gives nothing away; @@ -139,7 +140,7 @@ const createAzureWebhookVerifier = (options = {}) => { entries.push({ key: `azure:${record.id}`, events: [eventOf(record)] }); } return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); - }, + }), }; }; diff --git a/src/webhooks/brevo.js b/src/webhooks/brevo.js index bcbb2ee..3b021db 100644 --- a/src/webhooks/brevo.js +++ b/src/webhooks/brevo.js @@ -16,6 +16,7 @@ const { deliveryOf, isRecord, jsonRecordOf, + observed, parseJsonOf, resolveCredentialOptions, stringOf, @@ -92,7 +93,7 @@ const createBrevoWebhookVerifier = (options = {}) => { const checkToken = tokenCheckOf(options, 'Brevo'); const common = resolveCredentialOptions(options); return { - async verify(request) { + verify: observed(common, 'brevo', async (request) => { const bytes = bodyBytesOf(request, common); checkToken(request); const payload = parseJsonOf(bytes); @@ -105,7 +106,7 @@ const createBrevoWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not a Brevo event.'); } return deliveryOf(events); - }, + }), }; }; diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index af30446..1567a77 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -102,10 +102,37 @@ export interface MemoryReplayStoreOptions { */ export declare function createMemoryReplayStore(options?: MemoryReplayStoreOptions): ReplayStore; +/** + * What {@link WebhookVerifierOptions.onVerify} hears of one request. + */ +export interface WebhookVerifyReport { + /** The provider of the verifier. */ + readonly provider: string; + /** + * `taken`, `refused` (a {@link WebhookVerificationError}), or `error` for + * what is not a refusal: a body that is not raw, a replay store that fails. + */ + readonly status: 'taken' | 'refused' | 'error'; + /** Why it was refused. */ + readonly reason?: WebhookVerificationReason; + /** The events taken. */ + readonly events: number; + /** The time the verification took, in milliseconds. */ + readonly durationMs: number; +} + +/** + * Hears each request a verifier takes or refuses, for a metric or a log; one + * that throws is a process warning (`MAILER_WEBHOOK_LISTENER_FAILED`). + */ +export type WebhookVerifyListener = (report: WebhookVerifyReport) => void; + /** * The options every verifier takes. */ export interface WebhookVerifierOptions { + /** Hears each request taken or refused. */ + readonly onVerify?: WebhookVerifyListener; /** The clock, in milliseconds. */ readonly now?: () => number; /** How far from now a request may be signed, in milliseconds. */ @@ -304,6 +331,8 @@ export declare function createMailgunWebhookVerifier( * id is kept: committing its delivery does nothing. */ export interface BasicWebhookVerifierOptions { + /** Hears each request taken or refused. */ + readonly onVerify?: WebhookVerifyListener; readonly username: string; readonly password: string; /** @@ -330,6 +359,8 @@ export declare function createPostmarkWebhookVerifier( * no time: an event is taken once by its id, kept a day and an hour. */ export interface SignedBodyWebhookVerifierOptions { + /** Hears each request taken or refused. */ + readonly onVerify?: WebhookVerifyListener; /** The signing secret of the webhook. */ readonly secret: string; /** The ids taken, as {@link WebhookVerifierOptions.replayStore}. */ @@ -373,6 +404,8 @@ export declare function createMailtrapWebhookVerifier( * The options of {@link createBrevoWebhookVerifier}. */ export interface BrevoWebhookVerifierOptions { + /** Hears each request taken or refused. */ + readonly onVerify?: WebhookVerifyListener; /** The token the webhook is configured with. */ readonly token: string; /** @@ -506,6 +539,8 @@ export declare function createSmtp2goWebhookVerifier( * The options of {@link createAzureWebhookVerifier}. */ export interface AzureWebhookVerifierOptions { + /** Hears each request taken or refused. */ + readonly onVerify?: WebhookVerifyListener; /** The token the Event Grid subscription sends as a secret delivery property. */ readonly token: string; /** diff --git a/src/webhooks/lettermint.js b/src/webhooks/lettermint.js index b4c9ec7..a9a00c7 100644 --- a/src/webhooks/lettermint.js +++ b/src/webhooks/lettermint.js @@ -14,6 +14,7 @@ const { headerOf, hmacOf, isRecord, + observed, parseJsonOf, requireSecret, resolveCommonOptions, @@ -96,7 +97,7 @@ const createLettermintWebhookVerifier = (options = {}) => { ); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { - async verify(request) { + verify: observed(common, 'lettermint', async (request) => { const bytes = bodyBytesOf(request, common); checkSignature(request, bytes, secret, common); const payload = parseJsonOf(bytes); @@ -106,7 +107,7 @@ const createLettermintWebhookVerifier = (options = {}) => { return claimDelivery(common, [ { key: `lettermint:${payload.id}`, events: [eventOf(payload)] }, ]); - }, + }), }; }; diff --git a/src/webhooks/maileroo.js b/src/webhooks/maileroo.js index 2359803..0d19c3e 100644 --- a/src/webhooks/maileroo.js +++ b/src/webhooks/maileroo.js @@ -11,6 +11,7 @@ const { createEvent, dateOf, isRecord, + observed, parseJsonOf, requireSecret, resolveCommonOptions, @@ -82,7 +83,7 @@ const createMailerooWebhookVerifier = (options = {}) => { ); const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { - async verify(request) { + verify: observed(common, 'maileroo', async (request) => { const bytes = bodyBytesOf(request, common); checkBodyHmac(request, bytes, 'x-maileroo-signature', secret, 'Maileroo'); const payload = parseJsonOf(bytes); @@ -91,7 +92,7 @@ const createMailerooWebhookVerifier = (options = {}) => { } const entries = [{ key: `maileroo:${payload.event_id}`, events: eventsOf(payload) }]; return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); - }, + }), }; }; diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js index 697c057..d48af47 100644 --- a/src/webhooks/mailersend.js +++ b/src/webhooks/mailersend.js @@ -10,9 +10,10 @@ const { claimDelivery, createEvent, dateOf, - isRecord, headerValueOf, + isRecord, metadataFromHeaders, + observed, parseJsonOf, requireSecret, resolveCommonOptions, @@ -93,7 +94,7 @@ const createMailerSendWebhookVerifier = (options = {}) => { ); const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { - async verify(request) { + verify: observed(common, 'mailersend', async (request) => { const bytes = bodyBytesOf(request, common); checkBodyHmac(request, bytes, 'signature', secret, 'MailerSend'); const payload = parseJsonOf(bytes); @@ -103,7 +104,7 @@ const createMailerSendWebhookVerifier = (options = {}) => { const event = eventOf(payload); const entries = [{ key: `mailersend:${event.id}`, events: [event] }]; return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); - }, + }), }; }; diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js index b63c92f..a35e893 100644 --- a/src/webhooks/mailgun.js +++ b/src/webhooks/mailgun.js @@ -19,6 +19,7 @@ const { dateOf, hmacOf, isRecord, + observed, parseJsonOf, requireSecret, resolveCommonOptions, @@ -92,7 +93,7 @@ const createMailgunWebhookVerifier = (options = {}) => { ); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { - async verify(request) { + verify: observed(common, 'mailgun', async (request) => { // The signature is a field of the body, read to find it and trusted // only once it matches. const payload = parseJsonOf(bodyBytesOf(request, common)); @@ -113,7 +114,7 @@ const createMailgunWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body has no Mailgun event.'); } return claimDelivery(common, [{ key: `mailgun:${token}`, events: [eventOf(data, token)] }]); - }, + }), }; }; diff --git a/src/webhooks/mailjet.js b/src/webhooks/mailjet.js index 8f7c2a2..6401b0e 100644 --- a/src/webhooks/mailjet.js +++ b/src/webhooks/mailjet.js @@ -12,6 +12,7 @@ const { deliveryOf, isRecord, jsonRecordOf, + observed, parseJsonOf, resolveCredentialOptions, stringOf, @@ -71,7 +72,7 @@ const createMailjetWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'Mailjet'); const common = resolveCredentialOptions(options); return { - async verify(request) { + verify: observed(common, 'mailjet', async (request) => { const bytes = bodyBytesOf(request, common); checkBasicCredentials(request, expected); const payload = parseJsonOf(bytes); @@ -84,7 +85,7 @@ const createMailjetWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not a Mailjet event.'); } return deliveryOf(events); - }, + }), }; }; diff --git a/src/webhooks/mailtrap.js b/src/webhooks/mailtrap.js index 5aa215e..1b04258 100644 --- a/src/webhooks/mailtrap.js +++ b/src/webhooks/mailtrap.js @@ -11,6 +11,7 @@ const { createEvent, dateOf, isRecord, + observed, parseJsonOf, requireSecret, resolveCommonOptions, @@ -110,7 +111,7 @@ const createMailtrapWebhookVerifier = (options = {}) => { ); const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); return { - async verify(request) { + verify: observed(common, 'mailtrap', async (request) => { const bytes = bodyBytesOf(request, common); checkBodyHmac(request, bytes, 'mailtrap-signature', secret, 'Mailtrap'); const entries = []; @@ -120,7 +121,7 @@ const createMailtrapWebhookVerifier = (options = {}) => { entries.push({ key: `mailtrap:${event.id}`, events: [event] }); } return claimDelivery(common, entries, {}, DAY_REPLAY_TTL_MS); - }, + }), }; }; diff --git a/src/webhooks/postmark.js b/src/webhooks/postmark.js index 5986f1d..e999b3e 100644 --- a/src/webhooks/postmark.js +++ b/src/webhooks/postmark.js @@ -17,6 +17,7 @@ const { dateOf, deliveryOf, isRecord, + observed, parseJsonOf, resolveCredentialOptions, stringOf, @@ -117,7 +118,7 @@ const createPostmarkWebhookVerifier = (options = {}) => { // No time or id to hold a request to: the credentials alone stand for it. const common = resolveCredentialOptions(options); return { - async verify(request) { + verify: observed(common, 'postmark', async (request) => { const bytes = bodyBytesOf(request, common); checkBasicCredentials(request, expected); const record = parseJsonOf(bytes); @@ -125,7 +126,7 @@ const createPostmarkWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not a Postmark event.'); } return deliveryOf([eventOf(record)]); - }, + }), }; }; diff --git a/src/webhooks/request.js b/src/webhooks/request.js index fa8fc23..4dbede7 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -7,6 +7,7 @@ // its events. const { createHmac, timingSafeEqual } = require('node:crypto'); +const { warnThrown } = require('../internal/callback.js'); const { checkOptionalFunction, checkPositiveInteger, @@ -144,9 +145,55 @@ const resolveCommonOptions = (options, defaultToleranceMs) => ({ toleranceMs: checkPositiveNumber(options.toleranceMs ?? defaultToleranceMs, 'toleranceMs'), replayStore: checkReplayStore(options.replayStore), leaseMs: checkPositiveNumber(options.leaseMs ?? DEFAULT_LEASE_MS, 'leaseMs'), + onVerify: checkOptionalFunction(options.onVerify, 'onVerify'), maxBodyBytes: checkPositiveNumber(options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES, 'maxBodyBytes'), }); +const tell = (listener, report) => { + try { + listener(report); + } catch (error) { + warnThrown('The onVerify of a webhook verifier', error, 'MAILER_WEBHOOK_LISTENER_FAILED'); + } +}; + +/** + * Tells `onVerify` of each request a verifier takes or refuses: the + * provider, `taken`, `refused` with its reason, or `error` for what is not a + * refusal (a body that is not raw, a store that fails), the events taken and + * the time it took. A listener that throws is a process warning. + * + * @internal + */ +const observed = (common, provider, verify) => { + const listener = common.onVerify; + if (listener === undefined) return verify; + return async (request) => { + const started = performance.now(); + let delivery; + try { + delivery = await verify(request); + } catch (error) { + const refused = error instanceof WebhookVerificationError; + tell(listener, { + provider, + status: refused ? 'refused' : 'error', + reason: refused ? error.reason : undefined, + events: 0, + durationMs: performance.now() - started, + }); + throw error; + } + tell(listener, { + provider, + status: 'taken', + events: delivery.events.length, + durationMs: performance.now() - started, + }); + return delivery; + }; +}; + /** * Resolves the options of a verifier of a provider that signs neither a time * nor an id: its credentials stand for a request, and nothing is kept of it. @@ -711,6 +758,7 @@ module.exports = { isRecord, jsonRecordOf, metadataFromHeaders, + observed, parseJsonOf, requireSecret, resolveCommonOptions, diff --git a/src/webhooks/resend.js b/src/webhooks/resend.js index 5813aea..d4422ad 100644 --- a/src/webhooks/resend.js +++ b/src/webhooks/resend.js @@ -23,6 +23,7 @@ const { headerOf, hmacOf, isRecord, + observed, parseJsonOf, resolveCommonOptions, splitNamedTags, @@ -135,7 +136,7 @@ const createResendWebhookVerifier = (options = {}) => { const key = svixKeyOf(options.secret, 'secret'); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { - async verify(request) { + verify: observed(common, 'resend', async (request) => { const bytes = bodyBytesOf(request, common); const id = checkSvixSignature(request, bytes, key, common); const payload = parseJsonOf(bytes); @@ -143,7 +144,7 @@ const createResendWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not a Resend event.'); } return claimDelivery(common, [{ key: `resend:${id}`, events: eventsOf(payload, id) }]); - }, + }), }; }; diff --git a/src/webhooks/sendgrid.js b/src/webhooks/sendgrid.js index eaa8fb8..b0785b5 100644 --- a/src/webhooks/sendgrid.js +++ b/src/webhooks/sendgrid.js @@ -19,6 +19,7 @@ const { dateOf, headerOf, isRecord, + observed, parseJsonOf, resolveCommonOptions, stringListOf, @@ -153,7 +154,7 @@ const createSendGridWebhookVerifier = (options = {}) => { const key = publicKeyOf(options.publicKey); const common = resolveCommonOptions(options, DEFAULT_TOLERANCE_MS); return { - async verify(request) { + verify: observed(common, 'sendgrid', async (request) => { const bytes = bodyBytesOf(request, common); const signature = headerOf(request, SIGNATURE); const timestamp = headerOf(request, TIMESTAMP); @@ -178,7 +179,7 @@ const createSendGridWebhookVerifier = (options = {}) => { entries.push({ key: `sendgrid:${event.id}`, events: [event] }); } return claimDelivery(common, entries); - }, + }), }; }; diff --git a/src/webhooks/smtp2go.js b/src/webhooks/smtp2go.js index 5731447..0b8d97e 100644 --- a/src/webhooks/smtp2go.js +++ b/src/webhooks/smtp2go.js @@ -10,9 +10,10 @@ const { createEvent, dateOf, deliveryOf, - isRecord, headerValueOf, + isRecord, metadataFromHeaders, + observed, resolveCredentialOptions, stringOf, } = require('./request.js'); @@ -92,7 +93,7 @@ const createSmtp2goWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'SMTP2GO'); const common = resolveCredentialOptions(options); return { - async verify(request) { + verify: observed(common, 'smtp2go', async (request) => { const bytes = bodyBytesOf(request, common); checkBasicCredentials(request, expected); const record = recordOf(bytes); @@ -100,7 +101,7 @@ const createSmtp2goWebhookVerifier = (options = {}) => { throw new WebhookVerificationError('payload', 'The body is not an SMTP2GO event.'); } return deliveryOf([eventOf(record)]); - }, + }), }; }; diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index 3b5fd79..a43690a 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -13,6 +13,7 @@ const { checkTime, claimDelivery, isRecord, + observed, parseJsonOf, resolveCommonOptions, } = require('./request.js'); @@ -229,7 +230,7 @@ const createSnsVerifier = (options, service) => { 'certificateTimeoutMs', ), ); - return async (request) => { + return observed(common, service.provider, async (request) => { const message = parseJsonOf(bodyBytesOf(request, common)); if (!isRecord(message) || typeof message.TopicArn !== 'string') { throw new WebhookVerificationError('payload', 'The body is not an SNS message.'); @@ -265,7 +266,7 @@ const createSnsVerifier = (options, service) => { } const events = service.eventsOf(message, isRecord(record) ? record : undefined); return claimDelivery(common, [{ key, events }]); - }; + }); }; module.exports = { certificateError, createSnsVerifier, isHttpsUrlOf }; diff --git a/src/webhooks/sparkpost.js b/src/webhooks/sparkpost.js index b7a99a2..44206fd 100644 --- a/src/webhooks/sparkpost.js +++ b/src/webhooks/sparkpost.js @@ -11,6 +11,7 @@ const { dateOf, deliveryOf, isRecord, + observed, parseJsonOf, resolveCredentialOptions, stringOf, @@ -98,7 +99,7 @@ const createSparkPostWebhookVerifier = (options = {}) => { const expected = basicCredentialsOf(options, 'SparkPost'); const common = resolveCredentialOptions(options); return { - async verify(request) { + verify: observed(common, 'sparkpost', async (request) => { const bytes = bodyBytesOf(request, common); checkBasicCredentials(request, expected); const payload = parseJsonOf(bytes); @@ -114,7 +115,7 @@ const createSparkPostWebhookVerifier = (options = {}) => { if (inner !== undefined) events.push(eventOf(inner, index)); } return deliveryOf(events); - }, + }), }; }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index ba128f6..fc81d98 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -116,3 +116,11 @@ createSesWebhookVerifier({ expectError( createSesWebhookVerifier({ topicArns: ['arn'], replayStore: null, signatureVersions: ['3'] }), ); +createPostmarkWebhookVerifier({ + username: 'u', + password: 'p', + onVerify: (report) => { + expectType<'taken' | 'refused' | 'error'>(report.status); + expectType(report.durationMs); + }, +}); diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 857a3d5..641c40c 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -336,3 +336,44 @@ test('Webhooks: an event carries the Message-ID it was sent with, and its recipi assert.strictEqual(events[0].rfcMessageId, ''); assert.strictEqual(events[0].recipient, 'Carol@example.org'); }); + +test('Webhooks: onVerify hears each request taken or refused, and why', async (t) => { + // Before: a verifier said nothing of what it refused, so a forged flood or + // a wrong secret did not show. + const reports = []; + const mailtrap = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: createMemoryReplayStore(), + onVerify: (report) => reports.push(report), + }); + const request = signMailtrap(mailtrapBatch('ev-1', 'ev-2')); + await (await mailtrap.verify(request)).commit(); + await assert.rejects(mailtrap.verify(request)); + await assert.rejects(mailtrap.verify({ body: '{}', headers: {} })); + await assert.rejects(mailtrap.verify({ body: {} }), TypeError); + assert.deepStrictEqual( + reports.map((report) => [report.provider, report.status, report.reason, report.events]), + [ + ['mailtrap', 'taken', undefined, 2], + ['mailtrap', 'refused', 'replay', 0], + ['mailtrap', 'refused', 'signature', 0], + ['mailtrap', 'error', undefined, 0], + ], + ); + for (const report of reports) assert.ok(report.durationMs >= 0); + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push(options.code)); + const postmark = createPostmarkWebhookVerifier({ + username: 'u', + password: 'p', + onVerify: () => { + throw new Error('metrics down'); + }, + }); + await assert.rejects(postmark.verify({ body: '{}' }), WebhookVerificationError); + assert.deepStrictEqual(warnings, ['MAILER_WEBHOOK_LISTENER_FAILED']); + assert.throws( + () => createMailtrapWebhookVerifier({ secret: 's', replayStore: null, onVerify: 1 }), + /onVerify must be a function/, + ); +}); From 9af06b75d34c6496c532b0b652e13b10df0aca5d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 02:57:27 +0300 Subject: [PATCH 076/143] feat: a logger for the breaker, the rate limit, idempotency and the tee The decorators of 1.1 took no logger: a circuit that opened, a limit that refused or waited, a key answered as before, in flight or used again, or a mirror that failed told nothing but a receipt or a process warning. Each takes the pino-like logger of the other transports, and writes those with a code; with a logger, the failure of an idempotency store after a send and of a mirror go to it in place of a process warning. Co-Authored-By: Claude Opus 5.5 --- docs/transports/breaker.md | 1 + docs/transports/idempotency.md | 1 + docs/transports/ratelimit.md | 1 + docs/transports/tee.md | 1 + scripts/size-budgets.json | 8 ++-- src/breaker/breaker-transport.js | 13 ++++++ src/breaker/config.js | 1 + src/breaker/index.d.ts | 6 +++ src/idempotency/config.js | 1 + src/idempotency/idempotency-transport.js | 31 +++++++++++++-- src/idempotency/index.d.ts | 8 ++++ src/ratelimit/config.js | 1 + src/ratelimit/index.d.ts | 6 +++ src/ratelimit/ratelimit-transport.js | 5 +++ src/tee/config.js | 1 + src/tee/index.d.ts | 6 +++ src/tee/tee-transport.js | 11 +++++- tests/unit/decorators.test.js | 50 ++++++++++++++++++++++++ 18 files changed, 144 insertions(+), 8 deletions(-) diff --git a/docs/transports/breaker.md b/docs/transports/breaker.md index 9ee3760..f3a5201 100644 --- a/docs/transports/breaker.md +++ b/docs/transports/breaker.md @@ -66,6 +66,7 @@ const pool = new PoolTransport({ | `isFailure` | the categories above | Whether a receipt counts as a failure. | | `onStateChange` | | Hears each change of state; one that throws becomes a process warning (`MAILER_BREAKER_LISTENER_FAILED`). | | `now` | `Date.now` | The clock. | +| `logger` | | A pino-like logger: a change of state (`breaker.open` at `warn`), a refused send at `debug`. | `sendMany()` goes through the `sendMany()` of the wrapped transport while the circuit is closed, so its batching is kept; a message read while it is open is diff --git a/docs/transports/idempotency.md b/docs/transports/idempotency.md index 4d16a95..ca20a92 100644 --- a/docs/transports/idempotency.md +++ b/docs/transports/idempotency.md @@ -181,6 +181,7 @@ const mongoStore = { | `ttlMs` | `86400000` | How long the receipt of a send is kept under its key. | | `key` | `message.idempotencyKey` | The key of a message. | | `namespace` | the wrapped transport's id | The prefix of the keys in the store. | +| `logger` | | A pino-like logger: a send answered as before at `debug`, one in flight at `info`, a key used again and a store that throws at `warn`. | `createMemoryIdempotencyStore({ maxEntries, now })` makes the in-process store, whose tokens come from `crypto.randomUUID()`. It keeps at most `maxEntries` diff --git a/docs/transports/ratelimit.md b/docs/transports/ratelimit.md index 449c42e..3fd5cdd 100644 --- a/docs/transports/ratelimit.md +++ b/docs/transports/ratelimit.md @@ -60,6 +60,7 @@ const perTenant = new RateLimitTransport(transport, { | `cost` | `1` | The tokens a message takes. | | `maxKeys` | `10000` | The most buckets kept, the least used dropped first. | | `now` | `Date.now` | The clock. | +| `logger` | | A pino-like logger: a refused send (`ratelimit.limited`) at `info`, a send that waits at `debug`. | `sendMany()` goes through the `sendMany()` of the wrapped transport, each message handed to it once it has its tokens, so a batch is throttled like as diff --git a/docs/transports/tee.md b/docs/transports/tee.md index dd9643b..d583222 100644 --- a/docs/transports/tee.md +++ b/docs/transports/tee.md @@ -45,6 +45,7 @@ were left out. | `awaitMirrors` | `false` | Waits for the copies before returning. | | `maxPending` | `100` | The most copies in flight while not awaited. | | `onMirror` | | Hears every outcome of a mirror. | +| `logger` | | A pino-like logger: without `onMirror`, a failed copy at `warn`, in place of a process warning. | `sendMany()` goes through the `sendMany()` of the primary, each message mirrored as it is handed to it. `sendRaw()` mirrors to the mirrors that send raw diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 4030f6b..380cf31 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -30,9 +30,9 @@ "./mock": 2, "./otel": 5, "./logger": 2.5, - "./breaker": 2.5, - "./ratelimit": 2.5, - "./idempotency": 3.5, + "./breaker": 3, + "./ratelimit": 3, + "./idempotency": 4, "./hooks": 2.5, - "./tee": 2.5 + "./tee": 3 } diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index a1dbb21..bfce1fd 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -16,6 +16,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createBreakerConfig } = require('./config.js'); @@ -51,6 +52,7 @@ class CircuitBreakerTransport { // Grows at each change of state: a result counts only in the period it // was sent in, not in one the circuit came to since. #period = 0; + #log; /** * @param transport The transport to guard. @@ -62,6 +64,7 @@ class CircuitBreakerTransport { this.wrappedTransport = transport; this.id = transport.id; this.config = createBreakerConfig(config); + this.#log = createLog(this.config.logger, { transport: transport.id, decorator: 'breaker' }); } /** @@ -205,6 +208,11 @@ class CircuitBreakerTransport { } #refusal(remaining) { + this.#log.write( + 'debug', + () => ({ code: 'breaker.open', remaining }), + 'The circuit refused a send.', + ); const receipt = createFailedReceipt( `The circuit of ${this.id} is open after ${this.config.failureThreshold} failures in a ` + 'row; the message was not sent.', @@ -280,6 +288,11 @@ class CircuitBreakerTransport { const previous = this.#state; this.#state = state; this.#period++; + this.#log.write( + state === 'open' ? 'warn' : 'info', + () => ({ code: `breaker.${state}`, previous }), + `The circuit of ${this.id} is ${state}.`, + ); try { this.config.onStateChange?.(state, previous); } catch (error) { diff --git a/src/breaker/config.js b/src/breaker/config.js index 8c75e7c..bef7375 100644 --- a/src/breaker/config.js +++ b/src/breaker/config.js @@ -28,6 +28,7 @@ const createBreakerConfig = (config = {}) => ({ isFailure: checkOptionalFunction(config.isFailure, 'isFailure'), onStateChange: checkOptionalFunction(config.onStateChange, 'onStateChange'), now: checkOptionalFunction(config.now, 'now') ?? Date.now, + logger: config.logger, }); module.exports = { createBreakerConfig }; diff --git a/src/breaker/index.d.ts b/src/breaker/index.d.ts index ed44de9..967347a 100644 --- a/src/breaker/index.d.ts +++ b/src/breaker/index.d.ts @@ -4,6 +4,7 @@ /// import type { + Logger, Message, RawMessage, RawTransport, @@ -57,6 +58,11 @@ export interface CircuitBreakerConfig { readonly onStateChange?: (state: CircuitState, previous: CircuitState) => void; /** The clock, in milliseconds. */ readonly now?: () => number; + /** + * A pino-like logger: it hears a change of state, at `warn` when the circuit opens, `info` + * otherwise, and a refused send at `debug`, each with its `code`. + */ + readonly logger?: Logger; } /** diff --git a/src/idempotency/config.js b/src/idempotency/config.js index df8b175..294b99b 100644 --- a/src/idempotency/config.js +++ b/src/idempotency/config.js @@ -38,6 +38,7 @@ const createIdempotencyConfig = (config = {}, transportId) => { ttlMs: checkPositiveNumber(config.ttlMs ?? DEFAULT_TTL_MS, 'ttlMs'), key: checkOptionalFunction(config.key, 'key'), namespace, + logger: config.logger, }; }; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index 5de8c1b..f419254 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -14,8 +14,9 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); -const { warnThrown } = require('../internal/callback.js'); +const { describeThrown, warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createIdempotencyConfig } = require('./config.js'); const { fingerprintOf } = require('./fingerprint.js'); @@ -43,6 +44,7 @@ class IdempotencyTransport { id; config; wrappedTransport; + #log; /** * @param transport The transport to guard. @@ -53,6 +55,10 @@ class IdempotencyTransport { this.wrappedTransport = transport; this.id = transport.id; this.config = createIdempotencyConfig(config, transport.id); + this.#log = createLog(this.config.logger, { + transport: transport.id, + decorator: 'idempotency', + }); } /** @@ -159,6 +165,7 @@ class IdempotencyTransport { #replyOf(record, fingerprint, key) { if (record.fingerprint !== fingerprint) { + this.#log.write('warn', { code: 'idempotency.key_reused', key }, 'A key was used again.'); return createFailedReceipt( `The idempotency key ${key} was used for another message; this one was not sent.`, { @@ -169,7 +176,11 @@ class IdempotencyTransport { }, ); } - if (record.state === 'done' && record.receipt !== undefined) return record.receipt; + if (record.state === 'done' && record.receipt !== undefined) { + this.#log.write('debug', { code: 'idempotency.hit', key }, 'A send was answered as before.'); + return record.receipt; + } + this.#log.write('info', { code: 'idempotency.in_flight', key }, 'A send of the key is out.'); // The category is given, as the words of the text, which hold the key, // would otherwise choose one (a key `order-timeout-1` made it a timeout, // which a circuit breaker counts against the provider). @@ -215,7 +226,7 @@ class IdempotencyTransport { } } catch (error) { // The receipt stands; the key stays reserved until its lease ends. - warnThrown('The idempotency store', error, STORE_FAILED); + this.#storeThrew(error); } } @@ -262,8 +273,22 @@ class IdempotencyTransport { try { await this.config.store.release(entry.key, entry.token); } catch (error) { + this.#storeThrew(error); + } + } + + // A store that failed after the send: the logger hears of it when there is + // one, the process otherwise. + #storeThrew(error) { + if (!this.#log.hasLogger) { warnThrown('The idempotency store', error, STORE_FAILED); + return; } + this.#log.write( + 'warn', + () => ({ code: STORE_FAILED, error: describeThrown(error) }), + 'The idempotency store threw.', + ); } } diff --git a/src/idempotency/index.d.ts b/src/idempotency/index.d.ts index 2f713f1..7666b01 100644 --- a/src/idempotency/index.d.ts +++ b/src/idempotency/index.d.ts @@ -4,6 +4,7 @@ /// import type { + Logger, Message, RawMessage, RawTransport, @@ -109,6 +110,13 @@ export interface IdempotencyConfig { * @default the id of the wrapped transport */ readonly namespace?: string; + /** + * A pino-like logger: it hears a send answered as before at `debug` (`idempotency.hit`), one + * out at `info` (`idempotency.in_flight`), a key used again and a store + * that throws after a send at `warn`, the latter in place of a process + * warning. + */ + readonly logger?: Logger; } /** diff --git a/src/ratelimit/config.js b/src/ratelimit/config.js index f8c6de2..358d5f4 100644 --- a/src/ratelimit/config.js +++ b/src/ratelimit/config.js @@ -36,6 +36,7 @@ const createRateLimitConfig = (config = {}) => { cost: checkOptionalFunction(config.cost, 'cost'), maxKeys: checkPositiveInteger(config.maxKeys ?? 10000, 'maxKeys'), now: checkOptionalFunction(config.now, 'now') ?? Date.now, + logger: config.logger, }; }; diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts index 56a5067..886bbc0 100644 --- a/src/ratelimit/index.d.ts +++ b/src/ratelimit/index.d.ts @@ -4,6 +4,7 @@ /// import type { + Logger, Message, RawMessage, RawTransport, @@ -59,6 +60,11 @@ export interface RateLimitConfig { readonly maxKeys?: number; /** The clock, in milliseconds. */ readonly now?: () => number; + /** + * A pino-like logger: it hears a refused send at `info` (`ratelimit.limited`) and a send that + * waits at `debug` (`ratelimit.waiting`), with `waitMs`. + */ + readonly logger?: Logger; } /** diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index c9c6fde..7f333d4 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -14,6 +14,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); const { refuseRaw } = require('../internal/capabilities.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); +const { createLog } = require('../internal/log.js'); const { abortReason } = require('../internal/sleep.js'); const { createRateLimitConfig } = require('./config.js'); @@ -35,6 +36,7 @@ class RateLimitTransport { #buckets = new Map(); #refillPerMs; #disposed = false; + #log; /** * @param transport The transport to throttle. @@ -47,6 +49,7 @@ class RateLimitTransport { this.id = transport.id; this.config = createRateLimitConfig(config); this.#refillPerMs = this.config.limit / this.config.intervalMs; + this.#log = createLog(this.config.logger, { transport: transport.id, decorator: 'ratelimit' }); } /** @@ -142,12 +145,14 @@ class RateLimitTransport { Math.ceil((bucket.queuedCost + cost - bucket.tokens) / this.#refillPerMs), ); if (this.config.mode === 'reject' || waitMs > this.config.maxWaitMs) { + this.#log.write('info', () => ({ code: 'ratelimit.limited', waitMs }), 'A send was refused.'); return this.#refused( `The local rate limit of ${this.id} (${this.config.limit} per ${this.config.intervalMs} ms) ` + `would hold the message ${waitMs} ms.`, { category: 'rate-limit', retryable: true, retryAfterMilliseconds: waitMs }, ); } + this.#log.write('debug', () => ({ code: 'ratelimit.waiting', waitMs }), 'A send waits.'); return this.#wait(bucket, cost, signal); } diff --git a/src/tee/config.js b/src/tee/config.js index 7d9c086..2e7c4ec 100644 --- a/src/tee/config.js +++ b/src/tee/config.js @@ -35,6 +35,7 @@ const createTeeConfig = (config = {}) => { awaitMirrors, maxPending: checkPositiveInteger(config.maxPending ?? 100, 'maxPending'), onMirror: checkOptionalFunction(config.onMirror, 'onMirror'), + logger: config.logger, }; }; diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts index d8b0d5a..f203a6a 100644 --- a/src/tee/index.d.ts +++ b/src/tee/index.d.ts @@ -4,6 +4,7 @@ /// import type { + Logger, Message, RawMessage, RawTransport, @@ -60,6 +61,11 @@ export interface TeeConfig { * throws becomes one too (`MAILER_TEE_LISTENER_FAILED`). */ readonly onMirror?: (outcome: MirrorOutcome) => void; + /** + * A pino-like logger: without `onMirror`, it hears a failed copy at `warn` + * (`MAILER_TEE_MIRROR_FAILED`) in place of a process warning. + */ + readonly logger?: Logger; } /** diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index c41b45d..836ff82 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -15,6 +15,7 @@ const { isRawTransport, isVerifiableTransport } = require('../core/transport.js' const { describeThrown, warnLimited, warnThrown } = require('../internal/callback.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createTeeConfig } = require('./config.js'); @@ -55,6 +56,7 @@ class TeeTransport { primary; #pending = 0; #detached = new Set(); + #log; /** * @param primary The transport whose receipt is returned. @@ -66,6 +68,7 @@ class TeeTransport { this.primary = primary; this.id = primary.id; this.config = createTeeConfig(config); + this.#log = createLog(this.config.logger, { transport: primary.id, decorator: 'tee' }); } /** @@ -202,7 +205,13 @@ class TeeTransport { #report(outcome) { const listener = this.config.onMirror; if (listener === undefined) { - if (outcome.receipt?.successful !== true) { + if (outcome.receipt?.successful !== true && this.#log.hasLogger) { + this.#log.write( + 'warn', + () => ({ code: 'MAILER_TEE_MIRROR_FAILED', mirror: outcome.mirror }), + `The mirror ${outcome.mirror} failed: ${describe(outcome)}`, + ); + } else if (outcome.receipt?.successful !== true) { warnLimited( 'MAILER_TEE_MIRROR_FAILED', `The mirror ${outcome.mirror} failed: ${describe(outcome)}`, diff --git a/tests/unit/decorators.test.js b/tests/unit/decorators.test.js index 25254f3..bfe45fd 100644 --- a/tests/unit/decorators.test.js +++ b/tests/unit/decorators.test.js @@ -136,3 +136,53 @@ test('Decorators: retry does not retry an unsupported message, nor does a breake assert.strictEqual((await breaker.send(message())).successful, true); assert.strictEqual(requests.length, 1); }); + +test('Decorators: a logger hears the breaker, the rate limit, idempotency and the tee', async () => { + // Before: they took no logger; a circuit that opened or a limit that + // refused told nothing but a receipt. + const { RateLimitTransport } = require('../../src/ratelimit/index.js'); + const { TeeTransport } = require('../../src/tee/index.js'); + const records = []; + const logger = {}; + for (const level of ['debug', 'info', 'warn', 'error']) { + logger[level] = (fields) => records.push([level, fields.code, fields.decorator]); + } + const mail = (fields = {}) => + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 'x' }, + ...fields, + }); + const breaker = new CircuitBreakerTransport( + fakeTransport(() => failed('network')), + { + failureThreshold: 1, + logger, + }, + ); + await breaker.send(mail()); + await breaker.send(mail()); + const limit = new RateLimitTransport(fakeTransport(), { limit: 1, mode: 'reject', logger }); + await limit.send(mail()); + await limit.send(mail()); + const idempotent = new IdempotencyTransport(fakeTransport(), { logger }); + await idempotent.send(mail({ idempotencyKey: 'k' })); + await idempotent.send(mail({ idempotencyKey: 'k' })); + await idempotent.send(mail({ idempotencyKey: 'k', subject: 'other' })); + const tee = new TeeTransport(fakeTransport(), { + mirrors: [fakeTransport(() => failed('server-error'))], + awaitMirrors: true, + logger, + }); + await tee.send(mail()); + assert.deepStrictEqual(records, [ + ['warn', 'breaker.open', 'breaker'], + ['debug', 'breaker.open', 'breaker'], + ['info', 'ratelimit.limited', 'ratelimit'], + ['debug', 'idempotency.hit', 'idempotency'], + ['warn', 'idempotency.key_reused', 'idempotency'], + ['warn', 'MAILER_TEE_MIRROR_FAILED', 'tee'], + ]); +}); From 5778caf41e959651b948d3d3e170a8a221c5ae36 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:01:04 +0300 Subject: [PATCH 077/143] feat(ratelimit): a store of buckets that processes share The buckets were those of the process, so N replicas sent N times the quota of a provider. A `store` with `take(key, cost, bucket)`, which on Redis refills and takes in one script, holds the buckets for them all; a send waits for the time the store says while maxWaitMs allows, and is refused past it, and disposal stops the wait. The guide gives the script. Co-Authored-By: Claude Opus 5.5 --- docs/transports/ratelimit.md | 36 ++++++++++++++++++++ src/ratelimit/config.js | 8 +++++ src/ratelimit/index.d.ts | 26 ++++++++++++++ src/ratelimit/ratelimit-transport.js | 38 +++++++++++++++++++-- tests/types/ratelimit.test-d.ts | 4 +++ tests/unit/ratelimit.test.js | 51 ++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 2 deletions(-) diff --git a/docs/transports/ratelimit.md b/docs/transports/ratelimit.md index 3fd5cdd..d289c7b 100644 --- a/docs/transports/ratelimit.md +++ b/docs/transports/ratelimit.md @@ -47,6 +47,41 @@ const perTenant = new RateLimitTransport(transport, { }); ``` +## Several processes + +The buckets are those of the process: N replicas each send `limit` a second, +N times the quota of the provider. Give them a `store` they share, whose `take` +refills and takes in one step, and the limit is one. A send then waits for the +time the store says, as long as `maxWaitMs` allows, and is refused past it. + +```js +// Redis: a token bucket in a script, on the clock of Redis. +const TAKE = ` +local key, cost, limit, interval, burst = KEYS[1], tonumber(ARGV[1]), tonumber(ARGV[2]), tonumber(ARGV[3]), tonumber(ARGV[4]) +local now = redis.call('TIME'); now = now[1] * 1000 + math.floor(now[2] / 1000) +local state = redis.call('HMGET', key, 'tokens', 'at') +local tokens = tonumber(state[1]) or burst +local at = tonumber(state[2]) or now +tokens = math.min(burst, tokens + (now - at) * limit / interval) +local taken = tokens >= cost +if taken then tokens = tokens - cost end +redis.call('HSET', key, 'tokens', tokens, 'at', now) +redis.call('PEXPIRE', key, math.ceil(interval * burst / limit)) +if taken then return 0 end +return math.ceil((cost - tokens) * interval / limit)`; + +const store = { + async take(key, cost, { limit, intervalMs, burst }) { + const waitMs = await redis.eval(TAKE, { + keys: [`ratelimit:${key}`], + arguments: [String(cost), String(limit), String(intervalMs), String(burst)], + }); + return waitMs === 0 ? { taken: true } : { taken: false, waitMs }; + }, +}; +const limited = new RateLimitTransport(transport, { limit: 14, store }); +``` + ## Options | Option | Default | Description | @@ -59,6 +94,7 @@ const perTenant = new RateLimitTransport(transport, { | `key` | one bucket | The bucket of a message. | | `cost` | `1` | The tokens a message takes. | | `maxKeys` | `10000` | The most buckets kept, the least used dropped first. | +| `store` | | The buckets the processes share, in place of this process's. | | `now` | `Date.now` | The clock. | | `logger` | | A pino-like logger: a refused send (`ratelimit.limited`) at `info`, a send that waits at `debug`. | diff --git a/src/ratelimit/config.js b/src/ratelimit/config.js index 358d5f4..bf92565 100644 --- a/src/ratelimit/config.js +++ b/src/ratelimit/config.js @@ -8,6 +8,13 @@ const { checkPositiveNumber, } = require('../internal/options.js'); +// A store of the buckets the processes of an application share. +const checkStore = (store) => { + if (store === undefined) return undefined; + if (typeof store?.take !== 'function') throw new TypeError('store must have a take method.'); + return store; +}; + /** * Resolves the options of the rate limiter. * @@ -37,6 +44,7 @@ const createRateLimitConfig = (config = {}) => { maxKeys: checkPositiveInteger(config.maxKeys ?? 10000, 'maxKeys'), now: checkOptionalFunction(config.now, 'now') ?? Date.now, logger: config.logger, + store: checkStore(config.store), }; }; diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts index 886bbc0..d81d774 100644 --- a/src/ratelimit/index.d.ts +++ b/src/ratelimit/index.d.ts @@ -17,6 +17,26 @@ import type { /** * The options of {@link RateLimitTransport}. */ +/** + * A store of token buckets that the processes of an application share, so + * that N of them send no more than the one limit of the provider: on Redis, + * a script that refills and takes in one step. + */ +export interface RateLimitStore { + /** + * Takes `cost` tokens of the bucket of `key`, refilled at `limit` tokens + * every `intervalMs` up to `burst`, when it holds them: `{ taken: true }`; + * or `{ taken: false, waitMs }`, the time until it would. + */ + take( + key: string, + cost: number, + bucket: { readonly limit: number; readonly intervalMs: number; readonly burst: number }, + ): + | { readonly taken: boolean; readonly waitMs?: number } + | Promise<{ readonly taken: boolean; readonly waitMs?: number }>; +} + export interface RateLimitConfig { /** The tokens a bucket gains every `intervalMs`: the sustained rate. */ readonly limit: number; @@ -65,6 +85,12 @@ export interface RateLimitConfig { * waits at `debug` (`ratelimit.waiting`), with `waitMs`. */ readonly logger?: Logger; + /** + * The buckets the processes share, in place of the ones of this process: + * a send waits, while `maxWaitMs` allows, for the time the store says, and + * the store, not this process, keeps the order. + */ + readonly store?: RateLimitStore; } /** diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index 7f333d4..1aa8c56 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -15,7 +15,7 @@ const { isRawTransport, isVerifiableTransport } = require('../core/transport.js' const { refuseRaw } = require('../internal/capabilities.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); const { createLog } = require('../internal/log.js'); -const { abortReason } = require('../internal/sleep.js'); +const { abortReason, sleep } = require('../internal/sleep.js'); const { createRateLimitConfig } = require('./config.js'); const disposeOf = async (transport) => { @@ -36,6 +36,8 @@ class RateLimitTransport { #buckets = new Map(); #refillPerMs; #disposed = false; + // Aborts the waits on a shared store when the transport is disposed of. + #closing = new AbortController(); #log; /** @@ -101,6 +103,7 @@ class RateLimitTransport { */ async [Symbol.asyncDispose]() { this.#disposed = true; + this.#closing.abort(disposedError()); const reason = disposedError(); for (const bucket of this.#buckets.values()) { clearTimeout(bucket.timer); @@ -133,7 +136,9 @@ class RateLimitTransport { { category: 'configuration', code: 'ratelimit.cost_over_capacity' }, ); } - const bucket = this.#bucketOf(keyOf === undefined ? '' : String(keyOf(message))); + const key = keyOf === undefined ? '' : String(keyOf(message)); + if (this.config.store !== undefined) return this.#takeShared(key, cost, signal); + const bucket = this.#bucketOf(key); this.#refill(bucket); if (bucket.waiters.size === 0 && bucket.tokens >= cost) { bucket.tokens -= cost; @@ -156,6 +161,35 @@ class RateLimitTransport { return this.#wait(bucket, cost, signal); } + // Takes the tokens from the store the processes share, waiting while it + // says when they will be there, within `maxWaitMs` of the arrival. + async #takeShared(key, cost, signal) { + const { limit, intervalMs, burst } = this.config; + const waits = + signal === undefined ? this.#closing.signal : AbortSignal.any([signal, this.#closing.signal]); + let waited = 0; + for (;;) { + const answer = await this.config.store.take(key, cost, { limit, intervalMs, burst }); + if (answer?.taken === true) return undefined; + const waitMs = Math.max(1, Math.ceil(Number(answer?.waitMs) || 1)); + if (this.config.mode === 'reject' || waited + waitMs > this.config.maxWaitMs) { + this.#log.write( + 'info', + () => ({ code: 'ratelimit.limited', waitMs }), + 'A send was refused.', + ); + return this.#refused( + `The shared rate limit of ${this.id} (${limit} per ${intervalMs} ms) would hold the ` + + `message ${waitMs} ms.`, + { category: 'rate-limit', retryable: true, retryAfterMilliseconds: waitMs }, + ); + } + this.#log.write('debug', () => ({ code: 'ratelimit.waiting', waitMs }), 'A send waits.'); + await sleep(waitMs, waits); + waited += waitMs; + } + } + #refused(text, options) { return Promise.resolve( createFailedReceipt(text, { diff --git a/tests/types/ratelimit.test-d.ts b/tests/types/ratelimit.test-d.ts index 9b6259c..8c9ca92 100644 --- a/tests/types/ratelimit.test-d.ts +++ b/tests/types/ratelimit.test-d.ts @@ -33,3 +33,7 @@ expectType>>(limiter.send(message)); expectType>>(limiter.sendMany([message])); expectError(new RateLimitTransport(inner, {})); expectError(new RateLimitTransport(inner, { limit: 1, mode: 'drop' })); +new RateLimitTransport(inner, { + limit: 14, + store: { take: async () => ({ taken: false, waitMs: 70 }) }, +}); diff --git a/tests/unit/ratelimit.test.js b/tests/unit/ratelimit.test.js index 8ea937c..760b9b8 100644 --- a/tests/unit/ratelimit.test.js +++ b/tests/unit/ratelimit.test.js @@ -264,3 +264,54 @@ test('RateLimit: a signal that aborted between two messages of a batch stops the await assert.rejects(iterator.next(), /stop/); assert.strictEqual(inner.calls.length, 1); }); + +test('RateLimit: processes that share a store share one limit', async (t) => { + // Before: each process had its buckets, so N of them sent N times the + // quota of the provider. + const clock = timeline(t); + const buckets = new Map(); + const calls = []; + const store = { + // A token bucket as a Redis script would keep it, on the clock of the test. + take: async (key, cost, { limit, intervalMs, burst }) => { + calls.push([key, cost, limit, intervalMs, burst]); + const now = clock.now(); + const bucket = buckets.get(key) ?? { tokens: burst, at: now }; + bucket.tokens = Math.min(burst, bucket.tokens + ((now - bucket.at) * limit) / intervalMs); + bucket.at = now; + buckets.set(key, bucket); + if (bucket.tokens >= cost) { + bucket.tokens -= cost; + return { taken: true }; + } + return { taken: false, waitMs: ((cost - bucket.tokens) * intervalMs) / limit }; + }, + }; + const first = new RateLimitTransport(fakeTransport(), { limit: 1, intervalMs: 1000, store }); + const second = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 1000, + mode: 'reject', + store, + }); + assert.strictEqual((await first.send(message())).successful, true); + const refused = await second.send(message()); + assert.strictEqual(refused.errors[0].category, 'rate-limit'); + assert.strictEqual(refused.errors[0].retryAfterMilliseconds, 1000); + // The first waits for the store's tokens, on the store's word. + const waiting = first.send(message()); + // Lets the answer of the store come, and its wait start, before the clock moves. + await clock.advance(0); + await clock.advance(1000); + assert.strictEqual((await waiting).successful, true); + assert.deepStrictEqual(calls[0], ['', 1, 1, 1000, 1]); + // Disposal stops a wait on the store. + const held = first.send(message()); + await clock.advance(0); + await first[Symbol.asyncDispose](); + await assert.rejects(held, { name: 'AbortError' }); + assert.throws( + () => new RateLimitTransport(fakeTransport(), { limit: 1, store: {} }), + /take method/, + ); +}); From afb6d81787e518b4ff545ab0ae1970253822a0ef Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:03:13 +0300 Subject: [PATCH 078/143] perf: answers in place no longer split the batches of the wrapped sendMany The decorators' sendMany ended a run of the wrapped transport at each message they answered in place (a refusal, a stored receipt) and started another: with every tenth message refused, SES, Resend, Postmark or Mailjet got batches of nine. One run of the wrapped sendMany carries the whole input; the answers in place take their turn among its receipts, in the order of the messages. A run ends past 64 answers in place in a row, so that a circuit open over an endless input is not read ahead without bound. A key repeated in a batch is now read while its first send is in flight, and answered as in flight, where the split used to wait for the first. Co-Authored-By: Claude Opus 5.5 --- src/internal/gate.js | 79 +++++++++++++++++++++------------- tests/unit/gate.test.js | 38 ++++++++++++++++ tests/unit/idempotency.test.js | 5 ++- 3 files changed, 91 insertions(+), 31 deletions(-) diff --git a/src/internal/gate.js b/src/internal/gate.js index 51fafff..e55a3d6 100644 --- a/src/internal/gate.js +++ b/src/internal/gate.js @@ -42,11 +42,19 @@ const iteratorOf = (messages) => ? messages[Symbol.asyncIterator]() : messages[Symbol.iterator](); +// The most answers in place a run holds while a message handed to the +// wrapped transport waits for its receipt: past it, the run ends, so that a +// long streak of them, a circuit open over an endless input, is not read +// ahead without bound. +const MAX_IN_PLACE = 64; + /** * Sends messages through a gate. `gate.admit(message)` decides each one: `{ * receipt }` answers it in place; `{ message, …state }` hands `message` to the - * wrapped transport. The admitted ones go through `transport.sendMany()` in - * runs, a run ending at a message answered in place, and each receipt goes + * wrapped transport. The admitted ones go through one `transport.sendMany()`, + * the answers in place taking their turn among its receipts, so that a + * transport that batches keeps its batches; a run ends only at the end of the + * input or past `MAX_IN_PLACE` answers in place in a row. Each receipt goes * through `gate.settle(receipt, admitted)` before it is yielded. When the * wrapped `sendMany()` throws, `gate.fail(error, admitted)` hears of each * message still waiting for its receipt; when the caller stops reading, or @@ -63,23 +71,21 @@ const iteratorOf = (messages) => const gatedSendMany = async function* (transport, messages, options, gate) { options?.signal?.throwIfAborted(); const iterator = iteratorOf(messages); - let first; try { for (;;) { - if (first === undefined) { - const step = await iterator.next(); - if (step.done) return; - const decision = await gate.admit(step.value); - if (decision.receipt !== undefined) { - yield decision.receipt; - continue; - } - first = decision; + // Until a message goes to the transport, the answers in place go at once. + const step = await iterator.next(); + if (step.done) return; + const decision = await gate.admit(step.value); + if (decision.receipt !== undefined) { + yield decision.receipt; + continue; } - const run = { iterator, first, fed: new Fifo(), refusal: undefined, exhausted: false }; - first = undefined; + // In the order of the messages: `{ admitted }` waiting for its receipt, + // or `{ receipt }` answered in place. + const run = { iterator, first: decision, slots: new Fifo(), inPlace: 0, exhausted: false }; yield* sendRun(transport, run, options, gate); - if (run.refusal !== undefined) yield run.refusal; + while (run.slots.size > 0) yield run.slots.shift().receipt; if (run.exhausted) return; } } finally { @@ -87,12 +93,13 @@ const gatedSendMany = async function* (transport, messages, options, gate) { } }; -// The admitted messages of a run, read until one is answered in place or the -// input ends. +// The admitted messages of a run, read until the input ends or too many +// answers in place wait. const feedRun = async function* (run, gate) { - run.fed.push(run.first); + run.slots.push({ admitted: run.first }); yield run.first.message; for (;;) { + if (run.inPlace >= MAX_IN_PLACE) return; const step = await run.iterator.next(); if (step.done) { run.exhausted = true; @@ -100,33 +107,47 @@ const feedRun = async function* (run, gate) { } const decision = await gate.admit(step.value); if (decision.receipt !== undefined) { - run.refusal = decision.receipt; - return; + run.slots.push({ receipt: decision.receipt }); + run.inPlace++; + continue; } - run.fed.push(decision); + run.slots.push({ admitted: decision }); yield decision.message; } }; +// The messages of a run still waiting for their receipts, to be failed or +// abandoned; the answers in place among them stay for the caller. +const drainAdmitted = async (run, each) => { + const kept = new Fifo(); + while (run.slots.size > 0) { + const slot = run.slots.shift(); + if (slot.admitted === undefined) kept.push(slot); + else await each(slot.admitted); + } + run.slots = kept; +}; + const sendRun = async function* (transport, run, options, gate) { let failed = false; try { for await (const receipt of transport.sendMany(feedRun(run, gate), options)) { - const admitted = run.fed.shift(); - yield gate.settle === undefined ? receipt : await gate.settle(receipt, admitted); + // The answers in place before the message this receipt is for go first. + while (run.slots.peek()?.receipt !== undefined) { + run.inPlace--; + yield run.slots.shift().receipt; + } + const slot = run.slots.shift(); + yield gate.settle === undefined ? receipt : await gate.settle(receipt, slot?.admitted); } } catch (error) { failed = true; - if (gate.fail !== undefined) { - while (run.fed.size > 0) await gate.fail(error, run.fed.shift()); - } + if (gate.fail !== undefined) await drainAdmitted(run, (admitted) => gate.fail(error, admitted)); throw error; } finally { // The caller stopped reading, or the transport answered fewer messages // than it was handed: those it was handed are neither settled nor failed. - if (!failed && gate.abandon !== undefined) { - while (run.fed.size > 0) await gate.abandon(run.fed.shift()); - } + if (!failed && gate.abandon !== undefined) await drainAdmitted(run, gate.abandon); } }; diff --git a/tests/unit/gate.test.js b/tests/unit/gate.test.js index edebdc8..83d7de2 100644 --- a/tests/unit/gate.test.js +++ b/tests/unit/gate.test.js @@ -94,3 +94,41 @@ test('Gate: what a transport reading ahead was handed is abandoned on a break', assert.deepStrictEqual(await collect(gatedSendMany(short, [1, 2], undefined, gate)), []); assert.deepStrictEqual(abandoned, [1, 2]); }); + +test('Gate: answers in place do not split the batch of the wrapped transport', async () => { + // Before: each answer in place ended the run, so a batching transport got + // batches of the messages between two of them. + let calls = 0; + const batching = { + async *sendMany(messages) { + calls++; + for await (const message of messages) yield sent(String(message)); + }, + }; + const many = []; + for (let i = 1; i <= 100; i++) many.push(i); + const receipts = await collect( + gatedSendMany(batching, many, undefined, { + admit: (n) => (n % 10 === 0 ? { receipt: sent(`own-${n}`) } : { message: n }), + }), + ); + assert.strictEqual(calls, 1); + assert.strictEqual(receipts.length, 100); + assert.strictEqual(receipts[9].messageId, 'own-10'); + assert.strictEqual(receipts[10].messageId, '11'); + // A streak of answers in place past the bound ends the run, and the input + // is not read ahead without end. + calls = 0; + let read = 0; + const endless = { + [Symbol.asyncIterator]() { + return { next: async () => ({ done: false, value: read++ }) }; + }, + }; + for await (const receipt of gatedSendMany(batching, endless, undefined, { + admit: (n) => (n === 0 ? { message: n } : { receipt: sent(`own-${n}`) }), + })) { + if (receipt.messageId === 'own-70') break; + } + assert.ok(read < 200, String(read)); +}); diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js index 3897f42..ca43c19 100644 --- a/tests/unit/idempotency.test.js +++ b/tests/unit/idempotency.test.js @@ -332,10 +332,11 @@ test('Idempotency: sendMany answers repeated keys in place, the batch too', asyn message(), ]), ); - // The second x comes once the first is done, and gets its receipt. + // An answer in place no longer ends the batch, so the second x is read + // while the first is in flight, and is not sent. assert.deepStrictEqual( receipts.map((receipt) => receipt.messageId ?? receipt.errors[0].code), - ['m1', 'm0', 'm1', 'm2'], + ['m1', 'm0', 'idempotency.in_flight', 'm2'], ); // Read while the first is in flight, a second y is not sent. const twice = await collect( From 7bc002c07cf0ae077d10cf333f9e35b017a730bc Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:04:42 +0300 Subject: [PATCH 079/143] chore(bench): the hot paths of 1.1, and a baseline taken after them The harness measured seven paths of 1.0, against a baseline taken before the bodies were encoded by their content. It measures the encodings of 100 KB of Cyrillic and Latin text, htmlToText of a megabyte, the fingerprint of idempotency, the verification of a Svix, a SendGrid ECDSA and a Mailtrap HMAC request, and the gate under five decorators and with every tenth message answered in place; baseline.json is taken anew. Co-Authored-By: Claude Opus 5.5 --- bench/baseline.json | 52 ++++++++++++++--- bench/bench.js | 134 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 8 deletions(-) diff --git a/bench/baseline.json b/bench/baseline.json index cb4819a..25edd70 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -1,34 +1,70 @@ { "node": "v24.14.1", - "date": "2026-10-16T14:05:15.910Z", + "date": "2026-10-18T00:04:42.115Z", "results": [ { "name": "MIME — compose a text and HTML message", - "opsPerSec": 2796 + "opsPerSec": 9401 }, { "name": "MIME — compose a message with a 1 MiB attachment", - "opsPerSec": 40 + "opsPerSec": 406 }, { "name": "DKIM — compose and sign a text message (RSA-2048)", - "opsPerSec": 755 + "opsPerSec": 1312 }, { "name": "SMTP — dot-stuff and write 1 MiB of message data", - "opsPerSec": 301 + "opsPerSec": 745 }, { "name": "Address — isEmailAddress", - "opsPerSec": 1687799 + "opsPerSec": 3559030 }, { "name": "Receipt — classifyReceiptError of a reset", - "opsPerSec": 31384578 + "opsPerSec": 32283909 }, { "name": "Pool — select by priority among 10 transports", - "opsPerSec": 2263836 + "opsPerSec": 10120402 + }, + { + "name": "MIME — compose 100 KB of Cyrillic (base64)", + "opsPerSec": 594 + }, + { + "name": "MIME — compose 100 KB of Latin (quoted-printable)", + "opsPerSec": 750 + }, + { + "name": "Render — htmlToText of 1 MB", + "opsPerSec": 38 + }, + { + "name": "Idempotency — fingerprintOf a message", + "opsPerSec": 74060 + }, + { + "name": "Webhooks — verify a Svix (Resend) request", + "opsPerSec": 181926 + }, + { + "name": "Webhooks — verify a SendGrid ECDSA request", + "opsPerSec": 12227 + }, + { + "name": "Webhooks — verify a Mailtrap HMAC request", + "opsPerSec": 329750 + }, + { + "name": "Gate — 100 messages through five decorators", + "opsPerSec": 1847 + }, + { + "name": "Gate — 100 messages, every tenth answered in place", + "opsPerSec": 15761 } ] } diff --git a/bench/bench.js b/bench/bench.js index 81955b9..9b2baa8 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -28,6 +28,19 @@ const { readAttachmentContent, } = require('../index.js'); const { composeMessage } = require('../src/mime/index.js'); +const { htmlToText } = require('../src/render/index.js'); +const { fingerprintOf } = require('../src/idempotency/fingerprint.js'); +const { gatedSendMany } = require('../src/internal/gate.js'); +const { + createMailtrapWebhookVerifier, + createResendWebhookVerifier, + createSendGridWebhookVerifier, +} = require('../src/webhooks/index.js'); +const { CircuitBreakerTransport } = require('../src/breaker/index.js'); +const { HooksTransport } = require('../src/hooks/index.js'); +const { IdempotencyTransport } = require('../src/idempotency/index.js'); +const { RateLimitTransport } = require('../src/ratelimit/index.js'); +const { TeeTransport } = require('../src/tee/index.js'); const { writeMessageData } = require('../src/smtp/data-stream.js'); const { PriorityStrategy } = require('../src/pool/strategies/priority-strategy.js'); @@ -115,6 +128,88 @@ const dottedBody = Buffer.from( '\r\n', ); +// 100 KB of Cyrillic text, which goes base64, and of Latin text, which goes +// quoted-printable. +const cyrillicMessage = createMessage({ + from: ADDRESS, + to: ADDRESS, + subject: 'Рахунок', + content: { text: 'Шановний клієнте, ваш рахунок за жовтень готовий. '.repeat(2000) }, +}); +const latinMessage = createMessage({ + from: ADDRESS, + to: ADDRESS, + subject: 'Invoice', + content: { text: 'Dear customer, your invoice for October is ready, à bientôt. '.repeat(1700) }, +}); +const megabyteHtml = '

Hello world, & more

'.repeat( + 16000, +); + +// Requests signed as each provider signs them, verified without a replay store. +const SVIX_SECRET = 'whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'; +const svixRequest = () => { + const body = JSON.stringify({ type: 'email.delivered', data: { email_id: 'e', to: [ADDRESS] } }); + const timestamp = String(Math.floor(Date.now() / 1000)); + const key = Buffer.from(SVIX_SECRET.slice(6), 'base64'); + const signature = crypto + .createHmac('sha256', key) + .update(`msg_1.${timestamp}.${body}`) + .digest('base64'); + return { + body, + headers: { + 'svix-id': 'msg_1', + 'svix-timestamp': timestamp, + 'svix-signature': `v1,${signature}`, + }, + }; +}; +const ecdsa = crypto.generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); +const sendgridRequest = () => { + const body = JSON.stringify([ + { event: 'delivered', email: ADDRESS, sg_event_id: 'e1', timestamp: 1 }, + ]); + const timestamp = String(Math.floor(Date.now() / 1000)); + return { + body, + headers: { + 'x-twilio-email-event-webhook-signature': crypto + .sign('sha256', Buffer.from(timestamp + body), ecdsa.privateKey) + .toString('base64'), + 'x-twilio-email-event-webhook-timestamp': timestamp, + }, + }; +}; +const mailtrapBody = JSON.stringify({ + events: [{ event: 'delivery', event_id: 'e1', email: ADDRESS }], +}); +const mailtrapRequest = { + body: mailtrapBody, + headers: { + 'mailtrap-signature': crypto.createHmac('sha256', 'mt').update(mailtrapBody).digest('hex'), + }, +}; + +// A transport that answers at once, under the five decorators of 1.1. +const instant = { + id: 'instant', + send: async () => ({ successful: true, messageId: 'm' }), + async *sendMany(messages) { + for await (const message of messages) yield { successful: true, messageId: 'm', message }; + }, +}; +const decorated = new HooksTransport( + new TeeTransport( + new IdempotencyTransport( + new RateLimitTransport(new CircuitBreakerTransport(instant), { limit: 1e9, burst: 1e9 }), + ), + { mirrors: [instant], awaitMirrors: true }, + ), + { afterSend: () => {} }, +); +const hundred = Array.from({ length: 100 }, () => textMessage); + const main = async () => { if (!json) console.log(`Node ${process.version} | ${new Date().toISOString()}\n`); const { privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 }); @@ -138,6 +233,15 @@ const main = async () => { cause: Object.assign(new Error('connect ECONNRESET'), { code: 'ECONNRESET' }), }); const slow = { warmup: 20 }; + const resend = createResendWebhookVerifier({ secret: SVIX_SECRET, replayStore: null }); + const sendgrid = createSendGridWebhookVerifier({ + publicKey: ecdsa.publicKey.export({ format: 'pem', type: 'spki' }), + replayStore: null, + }); + const mailtrap = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: null, + }); const scenarios = [ ['MIME — compose a text and HTML message', () => composeAndRead(textMessage), slow], @@ -170,6 +274,36 @@ const main = async () => { 'Pool — select by priority among 10 transports', () => strategy.select(textMessage, transports, new Set()), ], + ['MIME — compose 100 KB of Cyrillic (base64)', () => composeAndRead(cyrillicMessage), slow], + ['MIME — compose 100 KB of Latin (quoted-printable)', () => composeAndRead(latinMessage), slow], + ['Render — htmlToText of 1 MB', () => htmlToText(megabyteHtml), slow], + ['Idempotency — fingerprintOf a message', () => fingerprintOf(textMessage)], + ['Webhooks — verify a Svix (Resend) request', () => resend.verify(svixRequest())], + ['Webhooks — verify a SendGrid ECDSA request', () => sendgrid.verify(sendgridRequest())], + ['Webhooks — verify a Mailtrap HMAC request', () => mailtrap.verify(mailtrapRequest)], + [ + 'Gate — 100 messages through five decorators', + async () => { + let sent = 0; + for await (const receipt of decorated.sendMany(hundred)) if (receipt.successful) sent++; + return sent; + }, + slow, + ], + [ + 'Gate — 100 messages, every tenth answered in place', + async () => { + let n = 0; + for await (const receipt of gatedSendMany(instant, hundred, undefined, { + admit: (message) => + ++n % 10 === 0 ? { receipt: { successful: true, messageId: 'own' } } : { message }, + })) { + if (receipt.successful) n++; + } + return n; + }, + slow, + ], ]; const results = []; From 683bc88766807f8ecc01c3e235f9da6647edc364 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:05:19 +0300 Subject: [PATCH 080/143] refactor: one place for what the decorators forward to their transport disposeOf was copied into four decorators and written out in a fifth, the forwarding of verify into five, isAbortOf into two, and the text of what was thrown into several. They live in src/internal/decorator.js. Co-Authored-By: Claude Opus 5.5 --- src/breaker/breaker-transport.js | 18 +++------- src/hooks/hooks-transport.js | 18 +++------- src/idempotency/idempotency-transport.js | 32 ++++++------------ src/internal/decorator.js | 43 ++++++++++++++++++++++++ src/ratelimit/ratelimit-transport.js | 14 +++----- src/tee/tee-transport.js | 14 +++----- 6 files changed, 71 insertions(+), 68 deletions(-) create mode 100644 src/internal/decorator.js diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index bfce1fd..6e834c9 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -13,9 +13,10 @@ // batch that throws counts once. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, isAbortOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createBreakerConfig } = require('./config.js'); @@ -31,9 +32,6 @@ const isProviderFailure = (receipt) => { return false; }; -const isAbortOf = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); - /** * A transport that stops sending through the one it wraps after it failed * `failureThreshold` times in a row, answers at once while open, and after @@ -146,9 +144,8 @@ class CircuitBreakerTransport { * Verifies the wrapped transport, whatever the state of the circuit; it has * nothing to check when that transport cannot verify. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** @@ -156,12 +153,7 @@ class CircuitBreakerTransport { * management. */ async [Symbol.asyncDispose]() { - const transport = Object(this.wrappedTransport); - if (typeof transport[Symbol.asyncDispose] === 'function') { - await transport[Symbol.asyncDispose](); - } else if (typeof transport[Symbol.dispose] === 'function') { - transport[Symbol.dispose](); - } + await disposeOf(this.wrappedTransport); } async #guard(options, send) { diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js index 7702b07..1255e56 100644 --- a/src/hooks/hooks-transport.js +++ b/src/hooks/hooks-transport.js @@ -13,21 +13,16 @@ // message or refuse it. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, messageOf, verifyWrapped } = require('../internal/decorator.js'); const { gatedSendMany } = require('../internal/gate.js'); const { checkOptionalFunction } = require('../internal/options.js'); const HOOK_FAILED = 'MAILER_HOOK_FAILED'; const HOOKS = ['beforeSend', 'beforeSendRaw', 'afterSend', 'onRetry', 'onError']; -const disposeOf = async (transport) => { - const target = Object(transport); - if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); - else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); -}; - /** * A transport that calls the hooks of the application around each send of * the one it wraps. @@ -111,9 +106,8 @@ class HooksTransport { } /** Verifies the wrapped transport. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** Disposes of the wrapped transport when it supports disposal. */ @@ -152,9 +146,7 @@ class HooksTransport { replaced = await hook(message, context); } catch (error) { return { - receipt: this.#refusal( - `The ${name} hook refused the message: ${error instanceof Error ? error.message : String(error)}`, - ), + receipt: this.#refusal(`The ${name} hook refused the message: ${messageOf(error)}`), }; } if (replaced === undefined) return { message }; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index f419254..56468a7 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -13,9 +13,10 @@ // was asked about alone. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { describeThrown, warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, isAbortOf, messageOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createIdempotencyConfig } = require('./config.js'); @@ -26,15 +27,6 @@ const STORE_FAILED = 'MAILER_IDEMPOTENCY_STORE_FAILED'; const isToken = (value) => typeof value === 'string' && value !== ''; const isRecord = (value) => typeof value === 'object' && value !== null; -const isAbortOf = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); - -const disposeOf = async (transport) => { - const target = Object(transport); - if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); - else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); -}; - /** * A transport that sends a message with an idempotency key once: a send under * a key already used answers with the receipt of the first, or as of unknown @@ -111,9 +103,8 @@ class IdempotencyTransport { } /** Verifies the wrapped transport. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** Disposes of the wrapped transport when it supports disposal. */ @@ -198,15 +189,12 @@ class IdempotencyTransport { // Retryable, but not a failure of the provider, which a circuit breaker // would count. #storeFailure(error) { - return createFailedReceipt( - `The idempotency store failed: ${error instanceof Error ? error.message : String(error)}`, - { - provider: this.id, - category: 'unknown', - retryable: true, - code: 'idempotency.store_failed', - }, - ); + return createFailedReceipt(`The idempotency store failed: ${messageOf(error)}`, { + provider: this.id, + category: 'unknown', + retryable: true, + code: 'idempotency.store_failed', + }); } // A sent or possibly sent message keeps its key with its receipt; one that diff --git a/src/internal/decorator.js b/src/internal/decorator.js new file mode 100644 index 0000000..1cdc871 --- /dev/null +++ b/src/internal/decorator.js @@ -0,0 +1,43 @@ +// Not ported: what the decorators share in forwarding to the transport they +// wrap: its disposal, its verification, and the reading of a caller's abort +// and of what was thrown. + +const { isVerifiableTransport } = require('../core/transport.js'); + +/** + * Disposes of a transport that supports disposal, asynchronous or not. + * + * @internal + */ +const disposeOf = async (transport) => { + const target = Object(transport); + if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); + else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); +}; + +/** + * Verifies a transport that can verify; one that cannot has nothing to check. + * + * @internal + */ +const verifyWrapped = async (transport, options) => { + options?.signal?.throwIfAborted(); + if (isVerifiableTransport(transport)) await transport.verify(options); +}; + +/** + * Whether an error is the abort of the caller's signal. + * + * @internal + */ +const isAbortOf = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +/** + * The text of what was thrown. + * + * @internal + */ +const messageOf = (error) => (error instanceof Error ? error.message : String(error)); + +module.exports = { disposeOf, isAbortOf, messageOf, verifyWrapped }; diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index 1aa8c56..66d1b6b 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -11,19 +11,14 @@ // least used dropped first; and disposal stops the waits. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); const { createLog } = require('../internal/log.js'); const { abortReason, sleep } = require('../internal/sleep.js'); const { createRateLimitConfig } = require('./config.js'); -const disposeOf = async (transport) => { - const target = Object(transport); - if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); - else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); -}; - /** * A transport that sends through the one it wraps at most `limit` messages * every `intervalMs`, with bursts up to `burst`, a bucket per `key`. @@ -92,9 +87,8 @@ class RateLimitTransport { } /** Verifies the wrapped transport, without taking tokens. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 836ff82..4d7aded 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -11,20 +11,15 @@ // address in it. const { pinMessageId } = require('../core/message-id.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { describeThrown, warnLimited, warnThrown } = require('../internal/callback.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createTeeConfig } = require('./config.js'); -const disposeOf = async (transport) => { - const target = Object(transport); - if (typeof target[Symbol.asyncDispose] === 'function') await target[Symbol.asyncDispose](); - else if (typeof target[Symbol.dispose] === 'function') target[Symbol.dispose](); -}; - const describe = (outcome) => { if (outcome.skipped !== undefined) return outcome.skipped; if (outcome.error !== undefined) return describeThrown(outcome.error); @@ -119,9 +114,8 @@ class TeeTransport { } /** Verifies the primary; the mirrors are not checked. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.primary)) await this.primary.verify(options); + verify(options) { + return verifyWrapped(this.primary, options); } /** From 3d9f346aaffaa8494ff88f90074501a3a30f107d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:08:03 +0300 Subject: [PATCH 081/143] refactor: the send, sendMany and verify the providers of 1.1 share The ten providers of 1.1 each wrote out the same send (convert, request, the receipt of the answer or of unknown delivery without an id, the caller's abort thrown and any other failure the provider's), the same sendMany through sendConcurrently and the same verify of one GET. They are sendConverted, sentWith, sendManyConcurrently and verifyByGet of src/internal/api.js; what differs stays in each provider. Co-Authored-By: Claude Opus 5.5 --- src/brevo/brevo-transport.js | 65 ++++++++------------- src/cloudflare/cloudflare-transport.js | 33 +++++------ src/internal/api.js | 80 ++++++++++++++++++++++++++ src/mailersend/mailersend-transport.js | 64 +++++++-------------- src/mailjet/mailjet-transport.js | 19 +----- src/postmark/postmark-transport.js | 45 ++++++--------- src/scaleway/scaleway-transport.js | 65 +++++++-------------- src/smtp2go/smtp2go-transport.js | 53 ++++++----------- src/sparkpost/sparkpost-transport.js | 51 ++++++---------- src/zeptomail/zeptomail-transport.js | 37 ++++-------- 10 files changed, 223 insertions(+), 289 deletions(-) diff --git a/src/brevo/brevo-transport.js b/src/brevo/brevo-transport.js index 81837c0..681a258 100644 --- a/src/brevo/brevo-transport.js +++ b/src/brevo/brevo-transport.js @@ -11,10 +11,14 @@ // delivery already made, not a failure to retry. const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { + providerFailure, + sendConverted, + sendManyConcurrently, + sentWith, + verifyByGet, +} = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createBrevoConfig } = require('./config.js'); const { createBrevoClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); @@ -58,36 +62,26 @@ class BrevoTransport { * @returns The receipt, with Brevo's messageId. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - const answer = await this.httpClient.send(payload, options); - const messageId = answer.json?.messageId ?? answer.json?.messageIds?.[0]; - if (typeof messageId !== 'string') { - return createFailedReceipt('Brevo answered without a messageId.', { - provider: 'brevo', - deliveryUnknown: true, - }); - } - return { successful: true, messageId, provider: 'brevo' }; - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return failureOf(error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'brevo', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => + sentWith( + 'brevo', + answer.json?.messageId ?? answer.json?.messageIds?.[0], + 'Brevo answered without a messageId.', + ), + failureOf, + }); } /** * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** @@ -96,21 +90,8 @@ class BrevoTransport { * @param options Optional abort signal. * @throws {Error} If Brevo refuses the key or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'Brevo', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/v3/account`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + verify(options) { + return verifyByGet(this, 'Brevo', `${this.config.baseUrl}/v3/account`, options); } } diff --git a/src/cloudflare/cloudflare-transport.js b/src/cloudflare/cloudflare-transport.js index d5c7f36..afbcf0c 100644 --- a/src/cloudflare/cloudflare-transport.js +++ b/src/cloudflare/cloudflare-transport.js @@ -12,8 +12,13 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure, ProviderApiError } = require('../internal/api.js'); +const { + ProviderApiError, + isCallerAbort, + providerFailure, + sendConverted, + sendManyConcurrently, +} = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createCloudflareConfig } = require('./config.js'); @@ -96,16 +101,13 @@ class CloudflareTransport { * @returns The receipt, with the `message_id` of the answer. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, options?.signal); - options?.signal?.throwIfAborted(); - return receiptOf((await this.httpClient.send(payload, options)).json, this.config); - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('cloudflare', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'cloudflare', + convert: (sent, signal) => convertMessage(sent, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => receiptOf(answer.json, this.config), + }); } /** @@ -133,12 +135,7 @@ class CloudflareTransport { * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** diff --git a/src/internal/api.js b/src/internal/api.js index 25c7957..02ac708 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -5,6 +5,8 @@ const { readAttachmentContent } = require('../core/attachment.js'); const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); +const { sendConcurrently } = require('./batch.js'); +const { verifyByRequest } = require('./verify.js'); const { toBase64 } = require('./base64.js'); const { readText } = require('./body.js'); const { canonicalHeaderName } = require('./headers.js'); @@ -319,6 +321,80 @@ const headerFieldsOf = (message) => { return fields; }; +/** + * Sends one message of a provider: `convert(message, signal)`, then + * `request(payload, options)`, then `receiptOf(answer)`. The caller's abort + * is thrown; any other failure is the receipt of `failureOf(error)`, the + * provider's by default. + * + * @internal + */ +const sendConverted = async (message, options, steps) => { + const signal = options?.signal; + signal?.throwIfAborted(); + try { + const payload = await steps.convert(message, signal); + signal?.throwIfAborted(); + return steps.receiptOf(await steps.request(payload, options)); + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + return steps.failureOf === undefined + ? providerFailure(steps.provider, error) + : steps.failureOf(error); + } +}; + +/** + * The receipt of an answer that should name the message: a success with its + * id, or, without one, a failure of unknown delivery, as the provider took + * the request. + * + * @internal + */ +const sentWith = (provider, messageId, missing) => + typeof messageId === 'string' && messageId !== '' + ? { successful: true, messageId, provider } + : createFailedReceipt(missing, { provider, deliveryUnknown: true }); + +/** + * Sends the messages of an iterable `concurrency` at a time through the + * `send` of a transport, the receipts in their order. + * + * @internal + */ +const sendManyConcurrently = (transport, messages, options) => + sendConcurrently( + messages, + transport.config.concurrency, + (message) => transport.send(message, options), + options?.signal, + ); + +/** + * Verifies the credentials of a transport with one authenticated read of + * `url`, a GET by default. + * + * @param transport The transport, with its `config` and `httpClient`. + * @param label The provider's name, for the error. + * @param url The URL read. + * @param options The caller's options. + * @param request `init` of the request in place of a GET, and `check`. + * @internal + */ +const verifyByGet = async (transport, label, url, options, request = {}) => { + options?.signal?.throwIfAborted(); + const init = request.init ?? { method: 'GET', headers: { Accept: 'application/json' } }; + return verifyByRequest({ + provider: label, + send: (signal) => transport.httpClient.fetchWithAuth(url, init, signal), + check: request.check, + retries: transport.config.retries, + timeoutMs: transport.config.timeout, + signal: options?.signal, + log: transport.httpClient.log, + }); +}; + module.exports = { ProviderApiError, base64Attachments, @@ -329,4 +405,8 @@ module.exports = { isCallerAbort, mergeBatch, providerFailure, + sendConverted, + sendManyConcurrently, + sentWith, + verifyByGet, }; diff --git a/src/mailersend/mailersend-transport.js b/src/mailersend/mailersend-transport.js index 9ca38b3..058a1a9 100644 --- a/src/mailersend/mailersend-transport.js +++ b/src/mailersend/mailersend-transport.js @@ -7,11 +7,13 @@ // through the bulk API, which answers with one id for the whole batch and // accepts what may fail later, unknown to the receipts. -const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { + sendConverted, + sendManyConcurrently, + sentWith, + verifyByGet, +} = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createMailerSendConfig } = require('./config.js'); const { createMailerSendClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); @@ -43,36 +45,25 @@ class MailerSendTransport { * @returns The receipt, with the `X-Message-Id` of the answer. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - const answer = await this.httpClient.send(payload, options); - const messageId = answer.headers.get('x-message-id') ?? answer.json?.message_id; - if (typeof messageId !== 'string' || messageId === '') { - return createFailedReceipt('MailerSend answered without an X-Message-Id.', { - provider: 'mailersend', - deliveryUnknown: true, - }); - } - return { successful: true, messageId, provider: 'mailersend' }; - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('mailersend', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'mailersend', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => + sentWith( + 'mailersend', + answer.headers.get('x-message-id') ?? answer.json?.message_id, + 'MailerSend answered without an X-Message-Id.', + ), + }); } /** * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** @@ -82,21 +73,8 @@ class MailerSendTransport { * @param options Optional abort signal. * @throws {Error} If MailerSend refuses the token or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'MailerSend', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/v1/domains`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + verify(options) { + return verifyByGet(this, 'MailerSend', `${this.config.baseUrl}/v1/domains`, options); } } diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index c9acb23..38e5739 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -15,9 +15,9 @@ const { isCallerAbort, mergeBatch, providerFailure, + verifyByGet, } = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createMailjetConfig } = require('./config.js'); const { createMailjetClient } = require('./http-client.js'); const { MAX_RECIPIENTS, convertMessage, recipientCount } = require('./message-converter.js'); @@ -187,21 +187,8 @@ class MailjetTransport { * @param options Optional abort signal. * @throws {Error} If Mailjet refuses the keys or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'Mailjet', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/v3/REST/myprofile`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + verify(options) { + return verifyByGet(this, 'Mailjet', `${this.config.baseUrl}/v3/REST/myprofile`, options); } } diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index d455b8e..61e769f 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -10,9 +10,15 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { chunks, jsonBytes } = require('../internal/batch.js'); -const { convertBatch, isCallerAbort, mergeBatch, providerFailure } = require('../internal/api.js'); +const { + convertBatch, + isCallerAbort, + mergeBatch, + providerFailure, + sendConverted, + verifyByGet, +} = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createPostmarkConfig } = require('./config.js'); const { classifyErrorCode, createPostmarkClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); @@ -71,17 +77,13 @@ class PostmarkTransport { * @returns The receipt, with Postmark's MessageID. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - const answer = await this.httpClient.send(payload, options); - return receiptOf(answer.json); - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('postmark', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'postmark', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => receiptOf(answer.json), + }); } /** @@ -146,21 +148,8 @@ class PostmarkTransport { * @param options Optional abort signal. * @throws {Error} If Postmark refuses the token or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'Postmark', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/server`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + verify(options) { + return verifyByGet(this, 'Postmark', `${this.config.baseUrl}/server`, options); } } diff --git a/src/scaleway/scaleway-transport.js b/src/scaleway/scaleway-transport.js index 4aeffcc..b487375 100644 --- a/src/scaleway/scaleway-transport.js +++ b/src/scaleway/scaleway-transport.js @@ -3,11 +3,13 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind // Changed in the port: `verify` lists the domains of the project. -const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { + sendConverted, + sendManyConcurrently, + sentWith, + verifyByGet, +} = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createScalewayConfig } = require('./config.js'); const { createScalewayClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); @@ -40,37 +42,25 @@ class ScalewayTransport { * @returns The receipt. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - const answer = await this.httpClient.send(payload, options); - const emails = answer.json?.emails; - const messageId = Array.isArray(emails) ? emails[0]?.id : undefined; - if (typeof messageId !== 'string' || messageId === '') { - return createFailedReceipt('Scaleway answered without an email id.', { - provider: 'scaleway', - deliveryUnknown: true, - }); - } - return { successful: true, messageId, provider: 'scaleway' }; - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('scaleway', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'scaleway', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => + sentWith( + 'scaleway', + answer.json?.emails?.[0]?.id, + 'Scaleway answered without an email id.', + ), + }); } /** * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** @@ -80,22 +70,9 @@ class ScalewayTransport { * @param options Optional abort signal. * @throws {Error} If Scaleway refuses the key or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); + verify(options) { const query = `project_id=${encodeURIComponent(this.config.projectId)}&page_size=1`; - await verifyByRequest({ - provider: 'Scaleway', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.httpClient.base}/domains?${query}`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + return verifyByGet(this, 'Scaleway', `${this.httpClient.base}/domains?${query}`, options); } } diff --git a/src/smtp2go/smtp2go-transport.js b/src/smtp2go/smtp2go-transport.js index db4fb4a..4882c70 100644 --- a/src/smtp2go/smtp2go-transport.js +++ b/src/smtp2go/smtp2go-transport.js @@ -6,10 +6,8 @@ // the account. const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { sendConverted, sendManyConcurrently, verifyByGet } = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createSmtp2goConfig } = require('./config.js'); const { createSmtp2goClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); @@ -75,28 +73,20 @@ class Smtp2goTransport { * @returns The receipt, with the `email_id` of the answer. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - return receiptOf((await this.httpClient.send(payload, options)).json); - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('smtp2go', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'smtp2go', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => receiptOf(answer.json), + }); } /** * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** @@ -107,26 +97,15 @@ class Smtp2goTransport { * @param options Optional abort signal. * @throws {Error} If SMTP2GO refuses the key or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'SMTP2GO', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/v3/stats/email_summary`, - { - method: 'POST', - headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, - body: '{}', - }, - signal, - ), + verify(options) { + return verifyByGet(this, 'SMTP2GO', `${this.config.baseUrl}/v3/stats/email_summary`, options, { + init: { + method: 'POST', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: '{}', + }, // SMTP2GO answers a key that may not read the summary with a 400. check: (status, text) => status === 400 && text.includes('ENDPOINT_PERMISSION_DENIED'), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, }); } } diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js index 68faa32..740d4ba 100644 --- a/src/sparkpost/sparkpost-transport.js +++ b/src/sparkpost/sparkpost-transport.js @@ -6,11 +6,15 @@ // are counted on the receipt, or fail it when it took none. const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { + isCallerAbort, + providerFailure, + sendConverted, + sendManyConcurrently, + verifyByGet, +} = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { verifyByRequest } = require('../internal/verify.js'); const { createSparkPostConfig } = require('./config.js'); const { createSparkPostClient } = require('./http-client.js'); const { convertMessage, convertRawMessage } = require('./message-converter.js'); @@ -69,16 +73,13 @@ class SparkPostTransport { * @returns The receipt, with the id of the transmission. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const transmission = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - return receiptOf((await this.httpClient.send(transmission, options)).json); - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('sparkpost', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'sparkpost', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => receiptOf(answer.json), + }); } /** @@ -107,12 +108,7 @@ class SparkPostTransport { * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } /** @@ -122,21 +118,8 @@ class SparkPostTransport { * @param options Optional abort signal. * @throws {Error} If SparkPost refuses the key or cannot be reached. */ - async verify(options) { - options?.signal?.throwIfAborted(); - await verifyByRequest({ - provider: 'SparkPost', - send: (signal) => - this.httpClient.fetchWithAuth( - `${this.config.baseUrl}/api/v1/account`, - { method: 'GET', headers: { Accept: 'application/json' } }, - signal, - ), - retries: this.config.retries, - timeoutMs: this.config.timeout, - signal: options?.signal, - log: this.httpClient.log, - }); + verify(options) { + return verifyByGet(this, 'SparkPost', `${this.config.baseUrl}/api/v1/account`, options); } } diff --git a/src/zeptomail/zeptomail-transport.js b/src/zeptomail/zeptomail-transport.js index b8529d0..ef76c2f 100644 --- a/src/zeptomail/zeptomail-transport.js +++ b/src/zeptomail/zeptomail-transport.js @@ -5,9 +5,7 @@ // `/email/batch` of one message to many recipients, whose one answer every // message shared. -const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConcurrently } = require('../internal/batch.js'); -const { isCallerAbort, providerFailure } = require('../internal/api.js'); +const { sendConverted, sendManyConcurrently, sentWith } = require('../internal/api.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createZeptoMailConfig } = require('./config.js'); const { createZeptoMailClient } = require('./http-client.js'); @@ -40,36 +38,21 @@ class ZeptoMailTransport { * @returns The receipt, with the `request_id` of the answer. * @throws The reason of the signal if it aborts. */ - async send(message, options) { - options?.signal?.throwIfAborted(); - try { - const payload = await convertMessage(message, this.config, options?.signal); - options?.signal?.throwIfAborted(); - const answer = await this.httpClient.send(payload, options); - const messageId = answer.json?.request_id; - if (typeof messageId !== 'string' || messageId === '') { - return createFailedReceipt('ZeptoMail answered without a request_id.', { - provider: 'zeptomail', - deliveryUnknown: true, - }); - } - return { successful: true, messageId, provider: 'zeptomail' }; - } catch (error) { - if (isCallerAbort(error, options?.signal)) throw error; - return providerFailure('zeptomail', error); - } + send(message, options) { + return sendConverted(message, options, { + provider: 'zeptomail', + convert: (sent, signal) => convertMessage(sent, this.config, signal), + request: (payload, sendOptions) => this.httpClient.send(payload, sendOptions), + receiptOf: (answer) => + sentWith('zeptomail', answer.json?.request_id, 'ZeptoMail answered without a request_id.'), + }); } /** * Sends messages `concurrency` at a time, the receipts in their order. */ sendMany(messages, options) { - return sendConcurrently( - messages, - this.config.concurrency, - (message) => this.send(message, options), - options?.signal, - ); + return sendManyConcurrently(this, messages, options); } } From 45ebc5ab057c97c14425647f9e718572f004f2c7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:09:08 +0300 Subject: [PATCH 082/143] refactor: keep verifyByGet and sendManyConcurrently with what they use In src/internal/api.js they made every provider bundle the verification and the batch helpers, ZeptoMail's ./zeptomail, which verifies nothing, included, past the budgets. verifyByGet goes with verifyByRequest in verify.js and sendManyConcurrently with sendConcurrently in batch.js. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/brevo/brevo-transport.js | 10 ++---- src/cloudflare/cloudflare-transport.js | 2 +- src/internal/api.js | 43 -------------------------- src/internal/batch.js | 16 +++++++++- src/internal/verify.js | 27 +++++++++++++++- src/mailersend/mailersend-transport.js | 9 ++---- src/mailjet/mailjet-transport.js | 2 +- src/postmark/postmark-transport.js | 2 +- src/scaleway/scaleway-transport.js | 9 ++---- src/smtp2go/smtp2go-transport.js | 4 ++- src/sparkpost/sparkpost-transport.js | 10 ++---- src/zeptomail/zeptomail-transport.js | 3 +- 13 files changed, 62 insertions(+), 77 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 380cf31..8a43d48 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -16,7 +16,7 @@ "./lettermint": 6.5, "./plunk": 6, "./postmark": 7.5, - "./brevo": 7, + "./brevo": 7.5, "./mailjet": 8, "./mailersend": 7.5, "./sparkpost": 8, diff --git a/src/brevo/brevo-transport.js b/src/brevo/brevo-transport.js index 681a258..75a676b 100644 --- a/src/brevo/brevo-transport.js +++ b/src/brevo/brevo-transport.js @@ -11,13 +11,9 @@ // delivery already made, not a failure to retry. const { createFailedReceipt } = require('../core/receipt.js'); -const { - providerFailure, - sendConverted, - sendManyConcurrently, - sentWith, - verifyByGet, -} = require('../internal/api.js'); +const { providerFailure, sendConverted, sentWith } = require('../internal/api.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createBrevoConfig } = require('./config.js'); const { createBrevoClient } = require('./http-client.js'); diff --git a/src/cloudflare/cloudflare-transport.js b/src/cloudflare/cloudflare-transport.js index afbcf0c..c37434f 100644 --- a/src/cloudflare/cloudflare-transport.js +++ b/src/cloudflare/cloudflare-transport.js @@ -17,9 +17,9 @@ const { isCallerAbort, providerFailure, sendConverted, - sendManyConcurrently, } = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createCloudflareConfig } = require('./config.js'); const { createCloudflareClient, errorCodeOf } = require('./http-client.js'); diff --git a/src/internal/api.js b/src/internal/api.js index 02ac708..9bab791 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -5,8 +5,6 @@ const { readAttachmentContent } = require('../core/attachment.js'); const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); -const { sendConcurrently } = require('./batch.js'); -const { verifyByRequest } = require('./verify.js'); const { toBase64 } = require('./base64.js'); const { readText } = require('./body.js'); const { canonicalHeaderName } = require('./headers.js'); @@ -356,45 +354,6 @@ const sentWith = (provider, messageId, missing) => ? { successful: true, messageId, provider } : createFailedReceipt(missing, { provider, deliveryUnknown: true }); -/** - * Sends the messages of an iterable `concurrency` at a time through the - * `send` of a transport, the receipts in their order. - * - * @internal - */ -const sendManyConcurrently = (transport, messages, options) => - sendConcurrently( - messages, - transport.config.concurrency, - (message) => transport.send(message, options), - options?.signal, - ); - -/** - * Verifies the credentials of a transport with one authenticated read of - * `url`, a GET by default. - * - * @param transport The transport, with its `config` and `httpClient`. - * @param label The provider's name, for the error. - * @param url The URL read. - * @param options The caller's options. - * @param request `init` of the request in place of a GET, and `check`. - * @internal - */ -const verifyByGet = async (transport, label, url, options, request = {}) => { - options?.signal?.throwIfAborted(); - const init = request.init ?? { method: 'GET', headers: { Accept: 'application/json' } }; - return verifyByRequest({ - provider: label, - send: (signal) => transport.httpClient.fetchWithAuth(url, init, signal), - check: request.check, - retries: transport.config.retries, - timeoutMs: transport.config.timeout, - signal: options?.signal, - log: transport.httpClient.log, - }); -}; - module.exports = { ProviderApiError, base64Attachments, @@ -406,7 +365,5 @@ module.exports = { mergeBatch, providerFailure, sendConverted, - sendManyConcurrently, sentWith, - verifyByGet, }; diff --git a/src/internal/batch.js b/src/internal/batch.js index f2d4846..b7c36be 100644 --- a/src/internal/batch.js +++ b/src/internal/batch.js @@ -91,4 +91,18 @@ const sendConcurrently = async function* (messages, limit, send, signal) { */ const jsonBytes = (value) => Buffer.byteLength(JSON.stringify(value)) + 1; -module.exports = { chunks, jsonBytes, readItems, sendConcurrently }; +/** + * Sends the messages of an iterable `concurrency` at a time through the + * `send` of a transport, the receipts in their order. + * + * @internal + */ +const sendManyConcurrently = (transport, messages, options) => + sendConcurrently( + messages, + transport.config.concurrency, + (message) => transport.send(message, options), + options?.signal, + ); + +module.exports = { chunks, jsonBytes, readItems, sendConcurrently, sendManyConcurrently }; diff --git a/src/internal/verify.js b/src/internal/verify.js index f1c1476..484dd28 100644 --- a/src/internal/verify.js +++ b/src/internal/verify.js @@ -60,4 +60,29 @@ const verifyByRequest = async (request) => { request.check?.(answer.status, answer.body); }; -module.exports = { verifyByRequest }; +/** + * Verifies the credentials of a transport with one authenticated read of + * `url`, a GET by default. + * + * @param transport The transport, with its `config` and `httpClient`. + * @param label The provider's name, for the error. + * @param url The URL read. + * @param options The caller's options. + * @param request `init` of the request in place of a GET, and `check`. + * @internal + */ +const verifyByGet = async (transport, label, url, options, request = {}) => { + options?.signal?.throwIfAborted(); + const init = request.init ?? { method: 'GET', headers: { Accept: 'application/json' } }; + return verifyByRequest({ + provider: label, + send: (signal) => transport.httpClient.fetchWithAuth(url, init, signal), + check: request.check, + retries: transport.config.retries, + timeoutMs: transport.config.timeout, + signal: options?.signal, + log: transport.httpClient.log, + }); +}; + +module.exports = { verifyByGet, verifyByRequest }; diff --git a/src/mailersend/mailersend-transport.js b/src/mailersend/mailersend-transport.js index 058a1a9..5eb27ce 100644 --- a/src/mailersend/mailersend-transport.js +++ b/src/mailersend/mailersend-transport.js @@ -7,12 +7,9 @@ // through the bulk API, which answers with one id for the whole batch and // accepts what may fail later, unknown to the receipts. -const { - sendConverted, - sendManyConcurrently, - sentWith, - verifyByGet, -} = require('../internal/api.js'); +const { sendConverted, sentWith } = require('../internal/api.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createMailerSendConfig } = require('./config.js'); const { createMailerSendClient } = require('./http-client.js'); diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index 38e5739..e43c9c3 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -15,8 +15,8 @@ const { isCallerAbort, mergeBatch, providerFailure, - verifyByGet, } = require('../internal/api.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createMailjetConfig } = require('./config.js'); const { createMailjetClient } = require('./http-client.js'); diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index 61e769f..cacba07 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -16,8 +16,8 @@ const { mergeBatch, providerFailure, sendConverted, - verifyByGet, } = require('../internal/api.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createPostmarkConfig } = require('./config.js'); const { classifyErrorCode, createPostmarkClient } = require('./http-client.js'); diff --git a/src/scaleway/scaleway-transport.js b/src/scaleway/scaleway-transport.js index b487375..7c03978 100644 --- a/src/scaleway/scaleway-transport.js +++ b/src/scaleway/scaleway-transport.js @@ -3,12 +3,9 @@ // https://github.com/productdevbook/unemail — MIT, Copyright 2025 Wind // Changed in the port: `verify` lists the domains of the project. -const { - sendConverted, - sendManyConcurrently, - sentWith, - verifyByGet, -} = require('../internal/api.js'); +const { sendConverted, sentWith } = require('../internal/api.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createScalewayConfig } = require('./config.js'); const { createScalewayClient } = require('./http-client.js'); diff --git a/src/smtp2go/smtp2go-transport.js b/src/smtp2go/smtp2go-transport.js index 4882c70..d92e242 100644 --- a/src/smtp2go/smtp2go-transport.js +++ b/src/smtp2go/smtp2go-transport.js @@ -6,7 +6,9 @@ // the account. const { createFailedReceipt } = require('../core/receipt.js'); -const { sendConverted, sendManyConcurrently, verifyByGet } = require('../internal/api.js'); +const { sendConverted } = require('../internal/api.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createSmtp2goConfig } = require('./config.js'); const { createSmtp2goClient } = require('./http-client.js'); diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js index 740d4ba..e7af1fb 100644 --- a/src/sparkpost/sparkpost-transport.js +++ b/src/sparkpost/sparkpost-transport.js @@ -6,14 +6,10 @@ // are counted on the receipt, or fail it when it took none. const { createFailedReceipt } = require('../core/receipt.js'); -const { - isCallerAbort, - providerFailure, - sendConverted, - sendManyConcurrently, - verifyByGet, -} = require('../internal/api.js'); +const { isCallerAbort, providerFailure, sendConverted } = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); +const { verifyByGet } = require('../internal/verify.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createSparkPostConfig } = require('./config.js'); const { createSparkPostClient } = require('./http-client.js'); diff --git a/src/zeptomail/zeptomail-transport.js b/src/zeptomail/zeptomail-transport.js index ef76c2f..94bad39 100644 --- a/src/zeptomail/zeptomail-transport.js +++ b/src/zeptomail/zeptomail-transport.js @@ -5,7 +5,8 @@ // `/email/batch` of one message to many recipients, whose one answer every // message shared. -const { sendConverted, sendManyConcurrently, sentWith } = require('../internal/api.js'); +const { sendConverted, sentWith } = require('../internal/api.js'); +const { sendManyConcurrently } = require('../internal/batch.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createZeptoMailConfig } = require('./config.js'); const { createZeptoMailClient } = require('./http-client.js'); From 8d337bae8d7c23419b9da1eed68a448d480e1c51 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:11:37 +0300 Subject: [PATCH 083/143] fix(hooks): a signal for the hooks, and what beforeSend may return The context of a hook had no signal, so a hook that waited on a store could not heed the caller's abort, and an abort while beforeSend ran still sent the message. The context carries the signal, and an abort during beforeSend sends nothing. A beforeSend could return any object, which went on as the message past the checks of createMessage; one that is not a message refuses it. The rejection of an async onRetry hook, which is not awaited, went unhandled; it is a process warning. Co-Authored-By: Claude Opus 5.5 --- docs/transports/hooks.md | 9 ++++++-- src/hooks/hooks-transport.js | 38 ++++++++++++++++++++++++++-------- src/hooks/index.d.ts | 2 ++ tests/unit/hooks.test.js | 40 ++++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/docs/transports/hooks.md b/docs/transports/hooks.md index a826fd4..13ba4b7 100644 --- a/docs/transports/hooks.md +++ b/docs/transports/hooks.md @@ -24,12 +24,17 @@ const transport = new HooksTransport(ses, { ## The hooks +The `context` of a hook holds the `transportId`, the `operation`, when the send +began (`startedAt`) and the caller's `signal`, for a hook that waits on something +to heed it; an abort while `beforeSend` runs sends nothing. + + | Hook | Called | When it throws | | --- | --- | --- | -| `beforeSend(message, context)` | before a message is sent; it may return a message to send in its place | the message is not sent; its receipt is `rejected` (code `hooks.refused`), not retryable | +| `beforeSend(message, context)` | before a message is sent; it may return a message of `createMessage` to send in its place (anything else refuses it) | the message is not sent; its receipt is `rejected` (code `hooks.refused`), not retryable | | `beforeSendRaw(raw, context)` | before a raw message is sent, the same way | the same | | `afterSend(receipt, message, context)` | after every send, its receipt successful or not | a process warning (`MAILER_HOOK_FAILED`); the receipt stands | -| `onRetry(event, context)` | before each retry of the wrapped transport, or of a `./retry` inside it | a process warning | +| `onRetry(event, context)` | before each retry of the wrapped transport, or of a `./retry` inside it, not awaited | a process warning, a promise that rejects too | | `onError(error, message, context)` | when the wrapped transport throws, before the error goes on | a process warning; the error goes on | A hook may be async; it is awaited. The context, `{ transportId, operation, diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js index 1255e56..e7126d5 100644 --- a/src/hooks/hooks-transport.js +++ b/src/hooks/hooks-transport.js @@ -56,9 +56,11 @@ class HooksTransport { */ async send(message, options) { options?.signal?.throwIfAborted(); - const context = this.#contextOf('send'); + const context = this.#contextOf('send', options?.signal); const before = await this.#before('beforeSend', message, context); if (before.receipt !== undefined) return before.receipt; + // A hook that took its time may outlast the caller's wish to send. + options?.signal?.throwIfAborted(); return this.#around(before.message, context, options, (sendOptions) => this.wrappedTransport.send(before.message, sendOptions), ); @@ -72,9 +74,10 @@ class HooksTransport { async sendRaw(message, options) { options?.signal?.throwIfAborted(); if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); - const context = this.#contextOf('sendRaw'); + const context = this.#contextOf('sendRaw', options?.signal); const before = await this.#before('beforeSendRaw', message, context); if (before.receipt !== undefined) return before.receipt; + options?.signal?.throwIfAborted(); return this.#around(before.message, context, options, (sendOptions) => this.wrappedTransport.sendRaw(before.message, sendOptions), ); @@ -85,10 +88,10 @@ class HooksTransport { * between its hooks; one `beforeSend` refuses is answered in place. */ sendMany(messages, options) { - const batch = this.#contextOf('sendMany'); + const batch = this.#contextOf('sendMany', options?.signal); return gatedSendMany(this.wrappedTransport, messages, this.#optionsOf(options, batch), { admit: async (message) => { - const context = this.#contextOf('sendMany'); + const context = this.#contextOf('sendMany', options?.signal); const before = await this.#before('beforeSend', message, context); return before.receipt === undefined ? { message: before.message, context } @@ -115,8 +118,8 @@ class HooksTransport { await disposeOf(this.wrappedTransport); } - #contextOf(operation) { - return { transportId: this.id, operation, startedAt: this.config.now() }; + #contextOf(operation, signal) { + return { transportId: this.id, operation, startedAt: this.config.now(), signal }; } // The options of one send, its retries told to `onRetry` with its context. @@ -128,9 +131,12 @@ class HooksTransport { ...options, onRetry: (event) => { try { - hook(event, context); + // Not awaited, as a retry does not wait for it; a promise that + // rejects is told all the same. + const result = hook(event, context); + if (typeof result?.then === 'function') result.then(undefined, warnRetryHook); } catch (error) { - warnThrown('The onRetry hook of HooksTransport', error, HOOK_FAILED); + warnRetryHook(error); } onRetry?.(event); }, @@ -150,7 +156,7 @@ class HooksTransport { }; } if (replaced === undefined) return { message }; - if (replaced === null || typeof replaced !== 'object') { + if (!isShaped(name, replaced)) { return { receipt: this.#refusal(`The ${name} hook returned neither a message nor nothing.`), }; @@ -198,6 +204,20 @@ class HooksTransport { } } +// What a hook may hand on in place of the message: one of `createMessage` +// for beforeSend, which the converters of the providers rely on to have +// checked its fields, and a raw message with its envelope for beforeSendRaw. +const isShaped = (name, value) => { + if (value === null || typeof value !== 'object') return false; + if (name === 'beforeSendRaw') { + return typeof value.envelope === 'object' && value.envelope !== null; + } + return typeof value.sender?.address === 'string' && Array.isArray(value.recipients); +}; + +const warnRetryHook = (error) => + warnThrown('The onRetry hook of HooksTransport', error, HOOK_FAILED); + const abandonedError = () => new DOMException( 'The batch was left before the message was answered; it may have been sent.', diff --git a/src/hooks/index.d.ts b/src/hooks/index.d.ts index f8c9960..c909c31 100644 --- a/src/hooks/index.d.ts +++ b/src/hooks/index.d.ts @@ -24,6 +24,8 @@ export interface HookContext { readonly operation: 'send' | 'sendMany' | 'sendRaw'; /** When the send began, by the clock `now`, in milliseconds. */ readonly startedAt: number; + /** The caller's signal, for a hook that waits on something to heed. */ + readonly signal?: AbortSignal; } /** diff --git a/tests/unit/hooks.test.js b/tests/unit/hooks.test.js index b33b348..b2c21cc 100644 --- a/tests/unit/hooks.test.js +++ b/tests/unit/hooks.test.js @@ -235,3 +235,43 @@ test('Hooks: a message a batch left before its answer ends with onError', async ['AbortError', 'c'], ]); }); + +test('Hooks: the context carries the signal, an abort during beforeSend sends nothing', async () => { + // Before: a hook had no signal to heed, and an abort while it ran still + // sent the message. + const inner = fakeTransport(); + const controller = new AbortController(); + let seen; + const hooks = new HooksTransport(inner, { + beforeSend: async (sentMessage, context) => { + seen = context.signal; + controller.abort(new Error('gone')); + }, + }); + await assert.rejects(hooks.send(message(), { signal: controller.signal }), /gone/); + assert.strictEqual(seen, controller.signal); + assert.strictEqual(inner.calls.length, 0); +}); + +test('Hooks: a beforeSend that returns something else than a message refuses it', async () => { + // Before: any object went on as the message, past the checks of createMessage. + const inner = fakeTransport(); + const hooks = new HooksTransport(inner, { beforeSend: () => ({ subject: 'not a message' }) }); + const receipt = await hooks.send(message()); + assert.strictEqual(receipt.errors[0].code, 'hooks.refused'); + assert.strictEqual(inner.calls.length, 0); +}); + +test('Hooks: an onRetry hook whose promise rejects is a warning', async (t) => { + // Before: the rejection of an async onRetry hook went unhandled. + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push(options.code)); + const inner = fakeTransport(async (_message, _index, options) => { + options.onRetry({ attempt: 1, delayMilliseconds: 0, error: new Error('busy') }); + return sent('m'); + }); + const hooks = new HooksTransport(inner, { onRetry: async () => Promise.reject(new Error('no')) }); + await hooks.send(message()); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepStrictEqual(warnings, ['MAILER_HOOK_FAILED']); +}); From d58c203e1b3c714c48fd99f46b7d1b9dd17e14b7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:12:43 +0300 Subject: [PATCH 084/143] fix: measure the intervals of breaker, rate limit and stores on a monotonic clock The circuit breaker, the rate limit and the memory stores of idempotency and replays measured their intervals with Date.now, so a correction of the system time half-opened a circuit early or held it open, refilled a bucket at once, or ended leases and times. Their default clock is performance.now(); the verifiers of webhooks, which compare the time a provider signs, keep Date.now. Co-Authored-By: Claude Opus 5.5 --- src/breaker/config.js | 5 ++++- src/breaker/index.d.ts | 5 ++++- src/idempotency/index.d.ts | 5 ++++- src/idempotency/memory-store.js | 5 ++++- src/ratelimit/config.js | 5 ++++- src/ratelimit/index.d.ts | 5 ++++- src/webhooks/index.d.ts | 5 ++++- src/webhooks/request.js | 3 ++- tests/unit/breaker.test.js | 19 +++++++++++++++++++ 9 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/breaker/config.js b/src/breaker/config.js index bef7375..e4f3916 100644 --- a/src/breaker/config.js +++ b/src/breaker/config.js @@ -8,6 +8,9 @@ const { checkPositiveNumber, } = require('../internal/options.js'); +// The intervals are measured on a clock no change of the time of day moves. +const monotonic = () => performance.now(); + /** * Resolves the options of the circuit breaker. * @@ -27,7 +30,7 @@ const createBreakerConfig = (config = {}) => ({ ), isFailure: checkOptionalFunction(config.isFailure, 'isFailure'), onStateChange: checkOptionalFunction(config.onStateChange, 'onStateChange'), - now: checkOptionalFunction(config.now, 'now') ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? monotonic, logger: config.logger, }); diff --git a/src/breaker/index.d.ts b/src/breaker/index.d.ts index 967347a..bcdf882 100644 --- a/src/breaker/index.d.ts +++ b/src/breaker/index.d.ts @@ -56,7 +56,10 @@ export interface CircuitBreakerConfig { * (`MAILER_BREAKER_LISTENER_FAILED`). */ readonly onStateChange?: (state: CircuitState, previous: CircuitState) => void; - /** The clock, in milliseconds. */ + /** + * The clock of the intervals, in milliseconds: `performance.now()` by + * default, which no change of the time of day moves. + */ readonly now?: () => number; /** * A pino-like logger: it hears a change of state, at `warn` when the circuit opens, `info` diff --git a/src/idempotency/index.d.ts b/src/idempotency/index.d.ts index 7666b01..c7308b8 100644 --- a/src/idempotency/index.d.ts +++ b/src/idempotency/index.d.ts @@ -141,7 +141,10 @@ export interface MemoryIdempotencyStoreOptions { * @default 100000 */ readonly maxEntries?: number; - /** The clock, in milliseconds. */ + /** + * The clock of the intervals, in milliseconds: `performance.now()` by + * default, which no change of the time of day moves. + */ readonly now?: () => number; } diff --git a/src/idempotency/memory-store.js b/src/idempotency/memory-store.js index 40cb30e..71ec620 100644 --- a/src/idempotency/memory-store.js +++ b/src/idempotency/memory-store.js @@ -11,6 +11,9 @@ const { randomUUID } = require('node:crypto'); const { checkOptionalFunction, checkPositiveInteger } = require('../internal/options.js'); +// The intervals are measured on a clock no change of the time of day moves. +const monotonic = () => performance.now(); + /** * Makes a store of idempotency keys in this process: shared by the * transports given it, not by other processes, and lost on a restart. A key @@ -29,7 +32,7 @@ const { checkOptionalFunction, checkPositiveInteger } = require('../internal/opt */ const createMemoryIdempotencyStore = (options = {}) => { const maxEntries = checkPositiveInteger(options.maxEntries ?? 100000, 'maxEntries'); - const now = checkOptionalFunction(options.now, 'now') ?? Date.now; + const now = checkOptionalFunction(options.now, 'now') ?? monotonic; // A key is in one of the two. Each Map iterates in the order of insertion // and an entry is written anew when it changes; as the keys of one state // share a lease or a time, the first of each is also the first to expire. diff --git a/src/ratelimit/config.js b/src/ratelimit/config.js index bf92565..7acc45b 100644 --- a/src/ratelimit/config.js +++ b/src/ratelimit/config.js @@ -15,6 +15,9 @@ const checkStore = (store) => { return store; }; +// The intervals are measured on a clock no change of the time of day moves. +const monotonic = () => performance.now(); + /** * Resolves the options of the rate limiter. * @@ -42,7 +45,7 @@ const createRateLimitConfig = (config = {}) => { key: checkOptionalFunction(config.key, 'key'), cost: checkOptionalFunction(config.cost, 'cost'), maxKeys: checkPositiveInteger(config.maxKeys ?? 10000, 'maxKeys'), - now: checkOptionalFunction(config.now, 'now') ?? Date.now, + now: checkOptionalFunction(config.now, 'now') ?? monotonic, logger: config.logger, store: checkStore(config.store), }; diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts index d81d774..8e97d73 100644 --- a/src/ratelimit/index.d.ts +++ b/src/ratelimit/index.d.ts @@ -78,7 +78,10 @@ export interface RateLimitConfig { * @default 10000 */ readonly maxKeys?: number; - /** The clock, in milliseconds. */ + /** + * The clock of the intervals, in milliseconds: `performance.now()` by + * default, which no change of the time of day moves. + */ readonly now?: () => number; /** * A pino-like logger: it hears a refused send at `info` (`ratelimit.limited`) and a send that diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 1567a77..5649e92 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -89,7 +89,10 @@ export interface MemoryReplayStoreOptions { * @default 10000 */ readonly maxEntries?: number; - /** The clock, in milliseconds. */ + /** + * The clock of the intervals, in milliseconds: `performance.now()` by + * default, which no change of the time of day moves. + */ readonly now?: () => number; } diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 4dbede7..74cb89a 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -80,7 +80,8 @@ class WebhookVerificationError extends Error { */ const createMemoryReplayStore = (options = {}) => { const maxEntries = checkPositiveInteger(options.maxEntries ?? 10000, 'maxEntries'); - const now = checkOptionalFunction(options.now, 'now') ?? Date.now; + // Leases and times are intervals, on a clock no change of the time of day moves. + const now = checkOptionalFunction(options.now, 'now') ?? (() => performance.now()); const entries = new Map(); // A Map iterates in the order of insertion, and an entry is written anew diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js index 244ec05..2a05ba2 100644 --- a/tests/unit/breaker.test.js +++ b/tests/unit/breaker.test.js @@ -365,3 +365,22 @@ test('Breaker: a batch that throws is one failure, and an isFailure that throws assert.strictEqual(judged.state, 'open'); assert.deepStrictEqual(warnings, ['MAILER_BREAKER_LISTENER_FAILED']); }); + +test('Breaker: a jump of the time of day does not move the circuit', async (t) => { + // Before: the default clock was Date.now, so a correction of the system + // time half-opened a circuit early, or held it open long. + let shift = 0; + const realNow = Date.now.bind(Date); + t.mock.method(Date, 'now', () => realNow() + shift); + const breaker = new CircuitBreakerTransport( + fakeTransport(() => failed('network')), + { + failureThreshold: 1, + resetTimeoutMs: 60_000, + }, + ); + await breaker.send(message()); + assert.strictEqual(breaker.state, 'open'); + shift = 60 * 60 * 1000; + assert.strictEqual(breaker.state, 'open'); +}); From ff65ca71ac6609cde9f59ecd9ec8c890535e52d2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:18:35 +0300 Subject: [PATCH 085/143] fix(webhooks): compare credentials by digest, keep ids hashed, refuse an unread window Basic credentials and tokens went through equalBytes, which answers at once for a length apart, so the time of a refusal told the length of the secret. They are compared as HMACs under a key of the process. The replay store was given the provider's id as it was, and an event of Mailtrap without an event_id is its message, type and recipient, so a shared store held addresses. It is given the provider and a SHA-256 of the id. The verifiers of providers that sign no time took toleranceMs and now and never read them; they throw a TypeError. The guide says that Mailgun signs neither body nor event, so every process needs the same store. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- docs/guide/webhooks.md | 14 +++-- src/webhooks/azure.js | 4 +- src/webhooks/index.d.ts | 6 ++- src/webhooks/maileroo.js | 4 +- src/webhooks/mailersend.js | 4 +- src/webhooks/mailtrap.js | 4 +- src/webhooks/request.js | 63 +++++++++++++++++++---- tests/helpers/webhooks/store.js | 14 +++++ tests/types/webhooks.test-d.ts | 1 + tests/unit/webhooks-request.test.js | 79 +++++++++++++++++++++++++++-- tests/unit/webhooks-ses.test.js | 5 +- 12 files changed, 173 insertions(+), 30 deletions(-) create mode 100644 tests/helpers/webhooks/store.js diff --git a/CHANGELOG.md b/CHANGELOG.md index dc46def..70d89d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,7 +140,10 @@ additive but the defaults listed below. `DKIM-Signature` field (the fix of 1.0 missed it). - **The webhooks of a provider are verified before they are read,** where sently parsed SES, SendGrid, Postmark and Brevo events unverified and checked Resend and Mailgun with no window - of time or replay check. + of time or replay check. Basic credentials and tokens are compared by an HMAC of each, so the + time of a refusal tells nothing of their length; a replay store is given the provider and a + SHA-256 of each id, never an id that names a recipient; and a verifier whose provider signs no + time refuses `toleranceMs` and `now` rather than ignore them. ### Fixed diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 06f8700..0887b32 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -116,7 +116,9 @@ are committed. `createMemoryReplayStore()` in one process; for several processes, or a function that makes its verifier for each request, a store they share; `null` for an application that deduplicates the events by their `id` itself. A shared -store implements three methods: +store implements three methods. The ids it is given are the provider and a +SHA-256 of the provider's id (`mailtrap:3q2+7w…`), as some ids name a +recipient: ```js // Redis, with node-redis: an atomic reservation, a commit, and a release @@ -140,9 +142,9 @@ const replayStore = { | --- | --- | --- | | `replayStore` | required | The ids taken, or `null`. | | `leaseMs` | 30 seconds | How long an id stays reserved while it is handled: longer than a handler takes, shorter than the provider's first retry (20 seconds for SNS). | -| `toleranceMs` | 5 minutes (an hour for SES) | How far from now a request may be signed. | +| `toleranceMs` | 5 minutes (an hour for SES) | How far from now a request may be signed. A verifier whose provider signs no time (Postmark, Brevo, Mailjet, SparkPost, SMTP2GO, MailerSend, Mailtrap, Maileroo, Azure) refuses it, and `now`. | | `maxBodyBytes` | 5 MiB | The longest body taken. | -| `now` | `Date.now` | The clock. | +| `now` | `Date.now` | The clock of `toleranceMs`. | | `onVerify` | | Hears each request: `{ provider, status, reason, events, durationMs }`, `status` being `taken`, `refused` or `error`, for a metric of refusals by `reason`. | ## Amazon SES @@ -211,7 +213,11 @@ const mailgun = createMailgunWebhookVerifier({ ``` The signature is the HMAC-SHA256 of the timestamp and the token of the body, -with the HTTP webhook signing key; the token is taken once. A permanent +with the HTTP webhook signing key; the token is taken once. Mailgun signs +neither the body nor the event in it, so a timestamp, token and signature +seen once can carry any event until the token is taken: give every process +the same shared `replayStore`, as one store per process takes the same token +once in each. A permanent failure is a hard bounce, a temporary one a deferred soft bounce, and a failure for a suppressed address `rejected`. The user variables are the metadata, and `messageId` the id of the receipt, in angle brackets. diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index 908aef6..13262b9 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -14,7 +14,7 @@ const { isRecord, observed, parseJsonOf, - resolveCommonOptions, + resolveUntimedOptions, stringOf, tokenCheckOf, } = require('./request.js'); @@ -115,7 +115,7 @@ const validationOf = (payload) => { const createAzureWebhookVerifier = (options = {}) => { const checkToken = tokenCheckOf(options, 'Event Grid'); const tokenHeader = options.header ?? 'authorization'; - const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); + const common = resolveUntimedOptions(options); return { verify: observed(common, 'azure', async (request) => { const bytes = bodyBytesOf(request, common); diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 5649e92..65d3b50 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -131,7 +131,7 @@ export interface WebhookVerifyReport { export type WebhookVerifyListener = (report: WebhookVerifyReport) => void; /** - * The options every verifier takes. + * The options of a verifier whose provider signs the time of a request. */ export interface WebhookVerifierOptions { /** Hears each request taken or refused. */ @@ -331,7 +331,8 @@ export declare function createMailgunWebhookVerifier( /** * The options of a verifier of a webhook whose URL carries Basic credentials, * as its provider signs nothing (Postmark, Mailjet, SparkPost, SMTP2GO). No - * id is kept: committing its delivery does nothing. + * id is kept: committing its delivery does nothing. With no time signed, + * `toleranceMs` and `now` throw a `TypeError`. */ export interface BasicWebhookVerifierOptions { /** Hears each request taken or refused. */ @@ -360,6 +361,7 @@ export declare function createPostmarkWebhookVerifier( /** * The options of a verifier of a webhook signed with an HMAC of its body and * no time: an event is taken once by its id, kept a day and an hour. + * `toleranceMs` and `now` throw a `TypeError`. */ export interface SignedBodyWebhookVerifierOptions { /** Hears each request taken or refused. */ diff --git a/src/webhooks/maileroo.js b/src/webhooks/maileroo.js index 0d19c3e..5cb3157 100644 --- a/src/webhooks/maileroo.js +++ b/src/webhooks/maileroo.js @@ -14,7 +14,7 @@ const { observed, parseJsonOf, requireSecret, - resolveCommonOptions, + resolveUntimedOptions, splitNamedTags, stringListOf, stringOf, @@ -81,7 +81,7 @@ const createMailerooWebhookVerifier = (options = {}) => { options.secret, 'secret is the signing secret of the Maileroo webhook.', ); - const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); + const common = resolveUntimedOptions(options); return { verify: observed(common, 'maileroo', async (request) => { const bytes = bodyBytesOf(request, common); diff --git a/src/webhooks/mailersend.js b/src/webhooks/mailersend.js index d48af47..c0a719b 100644 --- a/src/webhooks/mailersend.js +++ b/src/webhooks/mailersend.js @@ -16,7 +16,7 @@ const { observed, parseJsonOf, requireSecret, - resolveCommonOptions, + resolveUntimedOptions, stringOf, stringsOf, } = require('./request.js'); @@ -92,7 +92,7 @@ const createMailerSendWebhookVerifier = (options = {}) => { options.secret, 'secret is the signing secret of the MailerSend webhook.', ); - const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); + const common = resolveUntimedOptions(options); return { verify: observed(common, 'mailersend', async (request) => { const bytes = bodyBytesOf(request, common); diff --git a/src/webhooks/mailtrap.js b/src/webhooks/mailtrap.js index 1b04258..8bafbfb 100644 --- a/src/webhooks/mailtrap.js +++ b/src/webhooks/mailtrap.js @@ -14,7 +14,7 @@ const { observed, parseJsonOf, requireSecret, - resolveCommonOptions, + resolveUntimedOptions, stringOf, stringRecordOf, } = require('./request.js'); @@ -109,7 +109,7 @@ const createMailtrapWebhookVerifier = (options = {}) => { options.secret, 'secret is the signing secret of the Mailtrap webhook.', ); - const common = resolveCommonOptions(options, DAY_REPLAY_TTL_MS); + const common = resolveUntimedOptions(options); return { verify: observed(common, 'mailtrap', async (request) => { const bytes = bodyBytesOf(request, common); diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 74cb89a..9c04af0 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -6,7 +6,7 @@ // verified, and an id is kept as taken only once the application has handled // its events. -const { createHmac, timingSafeEqual } = require('node:crypto'); +const { createHash, createHmac, randomBytes, timingSafeEqual } = require('node:crypto'); const { warnThrown } = require('../internal/callback.js'); const { checkOptionalFunction, @@ -201,8 +201,27 @@ const observed = (common, provider, verify) => { * * @internal */ +// A provider that signs no time leaves no window to hold a request to, so an +// option of one would be taken and never read. +const UNTIMED = ['toleranceMs', 'now']; + +/** + * Resolves the options of a verifier whose provider signs no time: an id is + * kept a day and an hour, and `toleranceMs` or `now` is refused. + * + * @internal + */ +const resolveUntimedOptions = (options) => { + for (const name of UNTIMED) { + if (options[name] !== undefined) { + throw new TypeError(`${name} is not read: the provider signs no time.`); + } + } + return resolveCommonOptions(options, DAY_REPLAY_TTL_MS); +}; + const resolveCredentialOptions = (options) => - resolveCommonOptions({ ...options, replayStore: null }, DEFAULT_TOLERANCE_MS); + resolveUntimedOptions({ ...options, replayStore: null }); /** * The events of a verified request, whose ids stay reserved in the replay @@ -298,6 +317,16 @@ class WebhookDelivery { * @returns The delivery. * @internal */ +// The key of an id in the replay store: its provider, and a digest of the id, +// which may name a recipient (an event of Mailtrap without an event_id). +const storeKeyOf = (key) => { + const colon = key.indexOf(':'); + const digest = createHash('sha256') + .update(key.slice(colon + 1)) + .digest('base64url'); + return `${key.slice(0, colon + 1)}${digest}`; +}; + const claimDelivery = async (common, entries, fields = {}, ttlMs = common.toleranceMs * 2) => { const store = common.replayStore; const delivery = new WebhookDelivery(fields, store, ttlMs); @@ -305,8 +334,9 @@ const claimDelivery = async (common, entries, fields = {}, ttlMs = common.tolera let pending = 0; try { for (const entry of entries) { - const state = store === null ? 'reserved' : await store.reserve(entry.key, common.leaseMs); - if (state === 'reserved') delivery.add(entry.key, entry.events); + const key = store === null ? undefined : storeKeyOf(entry.key); + const state = store === null ? 'reserved' : await store.reserve(key, common.leaseMs); + if (state === 'reserved') delivery.add(key, entry.events); else if (state === 'pending') pending++; else if (state === 'done') done++; else throw new TypeError(`replayStore.reserve answered ${String(state)}.`); @@ -321,14 +351,14 @@ const claimDelivery = async (common, entries, fields = {}, ttlMs = common.tolera throw new WebhookVerificationError( 'pending', single - ? `The request ${entries[0].key} is being handled by another delivery.` + ? `The request ${storeKeyOf(entries[0].key)} is being handled by another delivery.` : 'The events of the batch are being handled by another delivery.', ); } throw new WebhookVerificationError( 'replay', single - ? `The request ${entries[0].key} was taken before.` + ? `The request ${storeKeyOf(entries[0].key)} was taken before.` : 'Every event of the batch was taken before.', ); }; @@ -427,12 +457,25 @@ const checkSignedSeconds = (value, common, label) => { }; /** - * Compares two secrets in constant time. + * Compares two digests in constant time; a length apart is unequal. * * @internal */ const equalBytes = (a, b) => a.length === b.length && timingSafeEqual(a, b); +// A key of this process: the digests of two secrets compare in a time that +// tells nothing of their length, as the secrets themselves would. +const COMPARE_KEY = randomBytes(32); +const digestOf = (bytes) => createHmac('sha256', COMPARE_KEY).update(bytes).digest(); + +/** + * Compares a secret given with the one expected, in constant time whatever + * their lengths. + * + * @internal + */ +const equalSecrets = (given, expected) => timingSafeEqual(digestOf(given), digestOf(expected)); + /** * The HMAC-SHA256 of the parts given, in order. * @@ -570,7 +613,7 @@ const checkBasicCredentials = (request, expected) => { throw new WebhookVerificationError('credentials', 'The request has no Basic credentials.'); } const given = Buffer.from(header.slice(space + 1).trim(), 'base64'); - if (!equalBytes(given, expected)) { + if (!equalSecrets(given, expected)) { throw new WebhookVerificationError('credentials', 'The credentials do not match.'); } }; @@ -703,7 +746,7 @@ const tokenCheckOf = (options, label) => { ); return (request) => { const given = headerOf(request, header); - if (given === undefined || !equalBytes(Buffer.from(given.trim(), 'utf8'), expected)) { + if (given === undefined || !equalSecrets(Buffer.from(given.trim(), 'utf8'), expected)) { throw new WebhookVerificationError('credentials', 'The webhook token does not match.'); } }; @@ -753,6 +796,7 @@ module.exports = { dateOf, deliveryOf, equalBytes, + equalSecrets, headerOf, headerValueOf, hmacOf, @@ -764,6 +808,7 @@ module.exports = { requireSecret, resolveCommonOptions, resolveCredentialOptions, + resolveUntimedOptions, splitNamedTags, stringListOf, stringOf, diff --git a/tests/helpers/webhooks/store.js b/tests/helpers/webhooks/store.js new file mode 100644 index 0000000..5506980 --- /dev/null +++ b/tests/helpers/webhooks/store.js @@ -0,0 +1,14 @@ +// Not ported: the key under which ./webhooks keeps an id in its replay store, +// the provider and a SHA-256 of the id, for tests that watch a store. + +const { createHash } = require('node:crypto'); + +const storeKey = (key) => { + const colon = key.indexOf(':'); + const digest = createHash('sha256') + .update(key.slice(colon + 1)) + .digest('base64url'); + return `${key.slice(0, colon + 1)}${digest}`; +}; + +module.exports = { storeKey }; diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index fc81d98..8ca135a 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -73,6 +73,7 @@ expectAssignable[]>( (await postmark.verify({ body: '' })).events, ); expectError(createPostmarkWebhookVerifier({ username: 'u' })); +expectError(createPostmarkWebhookVerifier({ username: 'u', password: 'p', toleranceMs: 1 })); expectError(createResendWebhookVerifier({})); const mailersend = createMailerSendWebhookVerifier({ secret: 's', replayStore: null }); diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 641c40c..fc40af1 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -5,8 +5,10 @@ const { createMemoryReplayStore, dateOf, equalBytes, + equalSecrets, headerOf, } = require('../../src/webhooks/request.js'); +const { storeKey } = require('../helpers/webhooks/store.js'); test('Webhooks: a header field is read from Headers or from Node headers, in any case', () => { const headers = new Headers({ 'Svix-Id': 'msg_1' }); @@ -23,6 +25,35 @@ test('Webhooks: secrets compare in constant time, a length apart is unequal', () assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abc')), true); assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abd')), false); assert.strictEqual(equalBytes(Buffer.from('abc'), Buffer.from('abcd')), false); + assert.strictEqual(equalSecrets(Buffer.from('abc'), Buffer.from('abc')), true); + assert.strictEqual(equalSecrets(Buffer.from('abc'), Buffer.from('abd')), false); + assert.strictEqual(equalSecrets(Buffer.from('abc'), Buffer.from('abcd')), false); +}); + +test('Webhooks: credentials of another length take the time of the comparison', async (t) => { + // Before: Basic and Bearer credentials were compared with equalBytes, which + // answers at once for a length apart, so the time told the secret's length. + const crypto = require('node:crypto'); + const compared = []; + const original = crypto.timingSafeEqual; + t.mock.method(crypto, 'timingSafeEqual', (a, b) => { + compared.push([a.length, b.length]); + return original(a, b); + }); + const root = require('node:path').join(__dirname, '../../src/webhooks/'); + const fresh = () => { + for (const name of Object.keys(require.cache)) { + if (name.startsWith(root)) delete require.cache[name]; + } + }; + fresh(); + t.after(fresh); + const { createPostmarkWebhookVerifier: postmark } = require('../../src/webhooks/index.js'); + const verifier = postmark({ username: 'user', password: 'a long password of the hook' }); + const short = `Basic ${Buffer.from('user:x').toString('base64')}`; + const body = JSON.stringify({ RecordType: 'Delivery', MessageID: 'm', Recipient: 'b@e.com' }); + await assert.rejects(verifier.verify({ body, headers: { authorization: short } }), /credentials/); + assert.deepStrictEqual(compared, [[32, 32]]); }); test('Webhooks: dates come from seconds, milliseconds or ISO text', () => { @@ -187,7 +218,7 @@ test('Webhooks: a store that fails releases the ids reserved before it', async ( const store = { reserve: async (id) => { calls.push(['reserve', id]); - if (id === 'mailtrap:ev-2') throw new Error('store down'); + if (id === storeKey('mailtrap:ev-2')) throw new Error('store down'); return 'reserved'; }, commit: async (id) => { @@ -200,9 +231,9 @@ test('Webhooks: a store that fails releases the ids reserved before it', async ( const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', replayStore: store }); await assert.rejects(mailtrap.verify(signMailtrap(mailtrapBatch('ev-1', 'ev-2'))), /store down/); assert.deepStrictEqual(calls, [ - ['reserve', 'mailtrap:ev-1'], - ['reserve', 'mailtrap:ev-2'], - ['release', 'mailtrap:ev-1'], + ['reserve', storeKey('mailtrap:ev-1')], + ['reserve', storeKey('mailtrap:ev-2')], + ['release', storeKey('mailtrap:ev-1')], ]); const odd = createMailtrapWebhookVerifier({ secret: 'mt', @@ -211,6 +242,27 @@ test('Webhooks: a store that fails releases the ids reserved before it', async ( await assert.rejects(odd.verify(signMailtrap(mailtrapBatch('ev-9'))), /answered true/); }); +test('Webhooks: an id is kept in the store by its digest, never with an address', async () => { + // Before: an event of Mailtrap without an event_id went into the store as + // its message, type and recipient, so a shared store held the addresses. + const keys = []; + const mailtrap = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: { + reserve: async (id) => keys.push(id) && 'reserved', + commit: async () => {}, + release: async () => {}, + }, + }); + const body = JSON.stringify({ + events: [{ event: 'delivery', message_id: 'm-1', email: 'bob@example.com' }], + }); + const { events } = await mailtrap.verify(signMailtrap(body)); + assert.strictEqual(events[0].id, 'm-1:delivery:bob@example.com'); + assert.deepStrictEqual(keys, [storeKey('mailtrap:m-1:delivery:bob@example.com')]); + assert.match(keys[0], /^mailtrap:[\w-]{43}$/); +}); + test('Webhooks: a batch with nothing new waits for its other delivery, or is a replay', async () => { const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', @@ -266,6 +318,25 @@ test('Webhooks: every verifier that keeps ids needs a replay store, null to keep assert.ok(createSmtp2goWebhookVerifier({ username: 'u', password: 'p' })); }); +test('Webhooks: a verifier of a provider that signs no time refuses a window of time', () => { + // Before: toleranceMs and now were taken and never read, so a caller who + // set a window believed old requests were refused. + const makers = [ + (more) => createPostmarkWebhookVerifier({ username: 'u', password: 'p', ...more }), + (more) => createBrevoWebhookVerifier({ token: 't', ...more }), + (more) => createSmtp2goWebhookVerifier({ username: 'u', password: 'p', ...more }), + (more) => createMailerSendWebhookVerifier({ secret: 's', replayStore: null, ...more }), + (more) => createMailtrapWebhookVerifier({ secret: 's', replayStore: null, ...more }), + (more) => createMailerooWebhookVerifier({ secret: 's', replayStore: null, ...more }), + (more) => createAzureWebhookVerifier({ token: 't', replayStore: null, ...more }), + ]; + for (const make of makers) { + assert.ok(make({})); + assert.throws(() => make({ toleranceMs: 60_000 }), /toleranceMs is not read/); + assert.throws(() => make({ now: Date.now }), /now is not read/); + } +}); + test('Webhooks: a refusal says what to answer, a certificate not read asks for a retry', async () => { assert.strictEqual(new WebhookVerificationError('signature', 'x').status, 401); assert.strictEqual(new WebhookVerificationError('payload', 'x').status, 400); diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js index e1c1d53..2d43023 100644 --- a/tests/unit/webhooks-ses.test.js +++ b/tests/unit/webhooks-ses.test.js @@ -14,6 +14,7 @@ const { notification, signSns, } = require('../helpers/webhooks/sns.js'); +const { storeKey } = require('../helpers/webhooks/store.js'); const NOW = Date.parse('2027-01-01T12:00:30Z'); const verifier = (options = {}) => { @@ -352,8 +353,8 @@ test('Webhooks SES: the options are checked, and a replay store may be given or const good = notification({ eventType: 'Send', mail: mail() }); await (await verify(request(good))).commit(); assert.deepStrictEqual(seen, [ - ['reserve', 'ses:sns-1', 30 * 1000], - ['commit', 'ses:sns-1', 2 * 60 * 60 * 1000], + ['reserve', storeKey('ses:sns-1'), 30 * 1000], + ['commit', storeKey('ses:sns-1'), 2 * 60 * 60 * 1000], ]); const open = verifier({ replayStore: null }); await open.verify(request(good)); From 53a4f121f5cb7bdb1b248ed056874cedaaa87890 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:19:59 +0300 Subject: [PATCH 086/143] fix(core): bound the keys and the values of metadata metadata took any number of keys of any length, and each went into every request, and into the header fields of the providers that write it as X-Metadata-*. createMessage takes at most 64 keys and values of at most 1024 characters; each provider's tighter limits stay its own. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- docs/messages/compose.md | 5 +++-- index.d.ts | 4 +++- src/core/message.js | 12 ++++++++++++ tests/unit/message-fields.test.js | 21 +++++++++++++++++++++ 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d89d4..d29ecb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,7 +134,8 @@ additive but the defaults listed below. URI that does not parse is refused rather than read as a path. - **A List-Unsubscribe URL or a metadata entry cannot add a header field:** a line break, whitespace or angle bracket in a URL is refused, and a metadata key is held to - `[A-Za-z0-9_.-]`, its value to a string without control characters. + `[A-Za-z0-9_.-]`, its value to a string without control characters, with at most 64 keys and + 1024 characters to a value. - **A DKIM signing domain or selector with a tab, CR or LF is refused** before it reaches the host parser, which removed them and let the name, line break and all, into the `DKIM-Signature` field (the fix of 1.0 missed it). diff --git a/docs/messages/compose.md b/docs/messages/compose.md index 6bf1585..db37feb 100644 --- a/docs/messages/compose.md +++ b/docs/messages/compose.md @@ -265,8 +265,9 @@ const message = createMessage({ configuration of the transport, where the provider takes it per message. - `metadata` holds values the provider keeps with the message and returns in its events. A key is 1 to 128 letters, digits, `_`, `-` or `.`, since it may become - the name of a header or form field; a value is a string without control - characters. + the name of a header or form field; a value is a string of at most 1024 + characters without control characters; and there are at most 64 keys. Each + provider has tighter limits of its own (Postmark 10 fields of 80 characters). SMTP and JMAP ignore `tracking` and `metadata`, which have no meaning in the message itself. Each provider's page says which of the three it takes. diff --git a/index.d.ts b/index.d.ts index 60cbce3..1d9b6df 100644 --- a/index.d.ts +++ b/index.d.ts @@ -501,7 +501,9 @@ export interface Message { readonly tracking?: MessageTracking; /** * Values the provider keeps with the message and returns in its events - * (custom arguments, variables), where its provider takes them. + * (custom arguments, variables), where its provider takes them: at most 64 + * keys of 1 to 128 of `[A-Za-z0-9_.-]`, each value at most 1024 characters + * with no control character. */ readonly metadata?: Readonly>; } diff --git a/src/core/message.js b/src/core/message.js index 5f276c8..b6f0098 100644 --- a/src/core/message.js +++ b/src/core/message.js @@ -250,6 +250,11 @@ const checkTracking = (tracking) => { // form field (`v:`), so it holds no more than letters, digits, `_`, `-` // and `.`; a value no control character. const METADATA_KEY = /^[A-Za-z0-9_.-]{1,128}$/; +// Every provider bounds what it keeps (Postmark 10 fields of 80 characters, +// SendGrid 10,000 bytes); these bounds keep a value meant for an event from +// making a message of any size, and leave each provider's own to it. +const MAX_METADATA_KEYS = 64; +const MAX_METADATA_VALUE = 1024; const hasControl = (text) => { for (let i = 0; i < text.length; i++) { @@ -265,13 +270,20 @@ const checkMetadata = (metadata) => { throwTypeError('Invalid metadata: it must be an object of strings.'); } const checked = Object.create(null); + let count = 0; for (const key in metadata) { if (!Object.hasOwn(metadata, key)) continue; + if (++count > MAX_METADATA_KEYS) { + throwTypeError(`Invalid metadata: more than ${MAX_METADATA_KEYS} keys.`); + } if (!METADATA_KEY.test(key)) throwTypeError(`Invalid metadata key: ${JSON.stringify(key)}`); const value = metadata[key]; if (typeof value !== 'string' || hasControl(value)) { throwTypeError(`Invalid metadata value of ${key}: a string without control characters.`); } + if (value.length > MAX_METADATA_VALUE) { + throwTypeError(`Invalid metadata value of ${key}: over ${MAX_METADATA_VALUE} characters.`); + } checked[key] = value; } return Object.freeze(checked); diff --git a/tests/unit/message-fields.test.js b/tests/unit/message-fields.test.js index be71f7d..0788ea5 100644 --- a/tests/unit/message-fields.test.js +++ b/tests/unit/message-fields.test.js @@ -74,6 +74,27 @@ test('Fields: an invalid scheduledAt, tracking or metadata is refused', () => { } }); +test('Fields: metadata holds at most 64 keys, each value at most 1024 characters', () => { + // Before: any number of keys of any length went into every request, and + // into the header fields of the providers that write them as X-Metadata-*. + const many = (count) => { + const metadata = {}; + for (let i = 0; i < count; i++) metadata[`k${i}`] = 'v'; + return metadata; + }; + assert.strictEqual( + Object.keys(createMessage({ ...base, metadata: many(64) }).metadata).length, + 64, + ); + assert.throws(() => createMessage({ ...base, metadata: many(65) }), /more than 64 keys/); + const longest = createMessage({ ...base, metadata: { key: 'v'.repeat(1024) } }); + assert.strictEqual(longest.metadata.key.length, 1024); + assert.throws( + () => createMessage({ ...base, metadata: { key: 'v'.repeat(1025) } }), + /Invalid metadata value of key: over 1024 characters/, + ); +}); + const assertRefused = (receipt) => { assert.strictEqual(receipt.successful, false); assert.strictEqual(receipt.retryable, false); From 65772d4d167b476a8680a17ffa6229cf27e55924 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:20:44 +0300 Subject: [PATCH 087/143] docs: what the EHLO name tells recipients, and when tee starts its mirrors The default EHLO name is the machine's host name, which the server writes into the Received field of every message. A mirror of the tee receives a message as the primary does, whatever the primary answers. Co-Authored-By: Claude Opus 5.5 --- docs/transports/smtp.md | 8 ++++++++ docs/transports/tee.md | 5 +++++ src/smtp/index.d.ts | 2 ++ src/tee/index.d.ts | 5 ++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/transports/smtp.md b/docs/transports/smtp.md index d7bb0ef..49d4c12 100644 --- a/docs/transports/smtp.md +++ b/docs/transports/smtp.md @@ -146,6 +146,14 @@ qualified domain name, its address as an address literal, or `[127.0.0.1]`, as nodemailer does; a server that checks the name, such as Postfix with `reject_non_fqdn_helo_hostname`, refuses `localhost`. A name with whitespace or a control character is refused when the transport is made. + +The server writes the EHLO name into the `Received` field it adds to each +message, which every recipient can read, so the default tells them the host +name of the machine that sent it, an internal one such as +`ip-10-0-1-5.ec2.internal` included. Set `localName` to the public name of +the sending host (one whose address resolves back to it) wherever the machine's +own name should stay private. + Connection pooling improves performance by reusing connections across multiple messages. diff --git a/docs/transports/tee.md b/docs/transports/tee.md index d583222..a2d8416 100644 --- a/docs/transports/tee.md +++ b/docs/transports/tee.md @@ -30,6 +30,11 @@ timeout, and follow the caller's signal. Disposing of the tee waits for the copies in flight before it disposes of the mirrors. The primary and the mirrors send one Message-ID, so a copy reads as the message it is. +The copies start with the primary's send, not after it: a mirror receives the +message whether or not the primary takes it, so a copy may go out for a +message the primary refused. A shadow that should see only what the primary +sent belongs after the send, in `afterSend` of [`./hooks`](/transports/hooks). + `onMirror` hears the outcome of every copy: its `receipt`, the `error` its send threw, or why it was `skipped`. Without it, a failed or skipped copy becomes a process warning (`MAILER_TEE_MIRROR_FAILED`), with the addresses of its text diff --git a/src/smtp/index.d.ts b/src/smtp/index.d.ts index 0fdeebe..d1ed4ac 100644 --- a/src/smtp/index.d.ts +++ b/src/smtp/index.d.ts @@ -92,6 +92,8 @@ export interface SmtpConfig extends TransportDependencies { * default the host name of this machine when it is a fully qualified * domain name, its address as an address literal, or `[127.0.0.1]`, as * nodemailer gives it; a server that checks the name refuses `localhost`. + * The server writes it into the `Received` field of each message, which + * the recipients read: set it where the machine's own name is private. */ readonly localName?: string; /** diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts index f203a6a..1546809 100644 --- a/src/tee/index.d.ts +++ b/src/tee/index.d.ts @@ -106,7 +106,10 @@ export declare class TeeTransport * @throws {TypeError} If an option is invalid. */ constructor(primary: Transport, config: TeeConfig); - /** Sends a message through the primary and mirrors it. */ + /** + * Sends a message through the primary and mirrors it; the copies start + * with the primary's send, whatever the primary answers. + */ send(message: Message, options?: TransportOptions): Promise>; /** * Sends a raw message through the primary, mirrored to the mirrors that send From 068467d2c73cc8e5d416b1e9c29480fee05a3cdf Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:22:35 +0300 Subject: [PATCH 088/143] perf(pool): make the entries of sendRaw once Every sendRaw copied each entry of the configuration to mark the transports that send raw messages. The configuration does not change, so the copies are made at the first call and kept. The budget of ./pool goes from 3 to 3.5 KB, which it had reached with verifyAll and the private deliver. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/pool/pool-transport.js | 25 ++++++++++++++++++------- tests/unit/raw-verify.test.js | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 8a43d48..7aa0ad4 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -25,7 +25,7 @@ "./azure": 8.5, "./scaleway": 7.5, "./cloudflare": 8, - "./pool": 3, + "./pool": 3.5, "./retry": 3, "./mock": 2, "./otel": 5, diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 68996fd..9b8f332 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -75,6 +75,10 @@ class PoolTransport { strategy; + // The entries of sendRaw(), made at its first call: the configuration does + // not change, and the raw transports among it neither. + #raw; + /** * Creates a new PoolTransport instance. * @@ -120,6 +124,18 @@ class PoolTransport { */ async sendRaw(message, options) { options?.signal?.throwIfAborted(); + const raw = this.#rawEntries(); + if (raw.enabled === 0) return refuseRaw(this); + return this.#deliver( + message, + raw.entries, + (transport, sendOptions) => transport.sendRaw(message, sendOptions), + options, + ); + } + + #rawEntries() { + if (this.#raw !== undefined) return this.#raw; const entries = []; let enabled = 0; for (const entry of this.config.transports) { @@ -127,13 +143,8 @@ class PoolTransport { if (canSend) enabled++; entries.push({ ...entry, enabled: canSend, selector: undefined }); } - if (enabled === 0) return refuseRaw(this); - return this.#deliver( - message, - entries, - (transport, sendOptions) => transport.sendRaw(message, sendOptions), - options, - ); + this.#raw = { entries, enabled }; + return this.#raw; } /** diff --git a/tests/unit/raw-verify.test.js b/tests/unit/raw-verify.test.js index 6e4fa6f..90fee7e 100644 --- a/tests/unit/raw-verify.test.js +++ b/tests/unit/raw-verify.test.js @@ -123,6 +123,22 @@ test('Raw: the pool sends through the transports that can, failing over among th assertRefused(await disabled.sendRaw(raw()), 'pool'); }); +test('Raw: the pool makes the entries of sendRaw once, not for every message', async (t) => { + // Before: every sendRaw copied each entry of the configuration. + const pool = new PoolTransport({ + strategy: 'round-robin', + transports: [{ transport: plain() }, { transport: new MockTransport() }], + }); + const select = t.mock.method(pool.strategy, 'select'); + await pool.sendRaw(raw()); + await pool.sendRaw(raw()); + assert.strictEqual(select.mock.callCount(), 2); + const [first, second] = select.mock.calls; + assert.strictEqual(first.arguments[1], second.arguments[1]); + assert.strictEqual(first.arguments[1][0].enabled, false); + assert.strictEqual(first.arguments[1][1].enabled, true); +}); + test('Raw: the pool verifies when one transport verifies', async () => { const fails = (text) => ({ ...plain(), verify: async () => Promise.reject(new Error(text)) }); const passes = { ...plain(), verify: async () => {} }; From 391ffefec352900a2a3947549ab766d10b3580eb Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:24:26 +0300 Subject: [PATCH 089/143] perf(webhooks): send the ids of a batch to the replay store together Each id of a batch waited for the store's answer to the one before it, a round trip per event; a SendGrid batch of a thousand events made a thousand for its reservation and as many for its commit. The ids go to the store up to 64 at a time, the events keep the order of the request whatever order the store answers in, and an id whose commit or release failed stays to be tried again. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 3 +- src/webhooks/index.d.ts | 3 +- src/webhooks/request.js | 123 ++++++++++++++++++---------- tests/unit/webhooks-request.test.js | 49 +++++++++++ 4 files changed, 134 insertions(+), 44 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 0887b32..1778338 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -118,7 +118,8 @@ function that makes its verifier for each request, a store they share; `null` for an application that deduplicates the events by their `id` itself. A shared store implements three methods. The ids it is given are the provider and a SHA-256 of the provider's id (`mailtrap:3q2+7w…`), as some ids name a -recipient: +recipient. The ids of a batch go to it up to 64 at a time, so a store must +take calls that run at once, as a Redis client does by pipelining them: ```js // Redis, with node-redis: an atomic reservation, a commit, and a release diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 65d3b50..b0d1513 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -64,7 +64,8 @@ export type ReplayState = 'reserved' | 'pending' | 'done'; * application commits it, so that a request whose handler failed is taken * again on the provider's next delivery. A store shared by several * processes, on Redis or a database, makes `reserve` one atomic operation - * (`SET NX PX`, an insert under a unique index). + * (`SET NX PX`, an insert under a unique index). The ids of a batch are + * passed up to 64 at a time, so its methods are called while others run. */ export interface ReplayStore { /** diff --git a/src/webhooks/request.js b/src/webhooks/request.js index 9c04af0..fc05f6c 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -223,6 +223,43 @@ const resolveUntimedOptions = (options) => { const resolveCredentialOptions = (options) => resolveUntimedOptions({ ...options, replayStore: null }); +// The key of an id in the replay store: its provider, and a digest of the id, +// which may name a recipient (an event of Mailtrap without an event_id). +const storeKeyOf = (key) => { + const colon = key.indexOf(':'); + const digest = createHash('sha256') + .update(key.slice(colon + 1)) + .digest('base64url'); + return `${key.slice(0, colon + 1)}${digest}`; +}; + +// The calls to a store go out a window at a time, so that a batch of a +// thousand events waits for a few round trips rather than a thousand. +const STORE_WINDOW = 64; + +/** + * Calls `call` for each key and its index, a window of them at a time, and + * settles every call of a window before the next. + * + * @returns The keys whose call succeeded, and the first error, if any. + * @internal + */ +const callEach = async (keys, call) => { + const succeeded = []; + let failure; + for (let start = 0; start < keys.length && failure === undefined; start += STORE_WINDOW) { + const end = Math.min(start + STORE_WINDOW, keys.length); + const calls = []; + for (let i = start; i < end; i++) calls.push(call(keys[i], i)); + const settled = await Promise.allSettled(calls); + for (let i = 0; i < settled.length; i++) { + if (settled[i].status === 'fulfilled') succeeded.push(keys[start + i]); + else failure ??= { error: settled[i].reason }; + } + } + return { succeeded, failure }; +}; + /** * The events of a verified request, whose ids stay reserved in the replay * store until the application commits them, or releases them so that the @@ -266,25 +303,20 @@ class WebhookDelivery { */ async commit(events) { if (this.#store === null) return; + const keys = []; if (events === undefined) { - for (const key of this.#left.keys()) { - await this.#store.commit(key, this.#ttlMs); - this.#left.delete(key); + for (const key of this.#left.keys()) keys.push(key); + } else { + for (const event of events) { + const key = this.#keyOf.get(event); + const left = key === undefined ? undefined : this.#left.get(key); + if (left === undefined) continue; + this.#keyOf.delete(event); + if (left > 1) this.#left.set(key, left - 1); + else keys.push(key); } - return; - } - for (const event of events) { - const key = this.#keyOf.get(event); - const left = key === undefined ? undefined : this.#left.get(key); - if (left === undefined) continue; - this.#keyOf.delete(event); - if (left > 1) { - this.#left.set(key, left - 1); - continue; - } - await this.#store.commit(key, this.#ttlMs); - this.#left.delete(key); } + await this.#settle(keys, (key) => this.#store.commit(key, this.#ttlMs)); } /** @@ -293,10 +325,16 @@ class WebhookDelivery { */ async release() { if (this.#store === null) return; - for (const key of this.#left.keys()) { - await this.#store.release(key); - this.#left.delete(key); - } + const keys = []; + for (const key of this.#left.keys()) keys.push(key); + await this.#settle(keys, (key) => this.#store.release(key)); + } + + // A key whose call failed stays, to be committed or released again. + async #settle(keys, call) { + const { succeeded, failure } = await callEach(keys, call); + for (const key of succeeded) this.#left.delete(key); + if (failure !== undefined) throw failure.error; } } @@ -317,35 +355,36 @@ class WebhookDelivery { * @returns The delivery. * @internal */ -// The key of an id in the replay store: its provider, and a digest of the id, -// which may name a recipient (an event of Mailtrap without an event_id). -const storeKeyOf = (key) => { - const colon = key.indexOf(':'); - const digest = createHash('sha256') - .update(key.slice(colon + 1)) - .digest('base64url'); - return `${key.slice(0, colon + 1)}${digest}`; -}; - const claimDelivery = async (common, entries, fields = {}, ttlMs = common.toleranceMs * 2) => { const store = common.replayStore; const delivery = new WebhookDelivery(fields, store, ttlMs); let done = 0; let pending = 0; - try { - for (const entry of entries) { - const key = store === null ? undefined : storeKeyOf(entry.key); - const state = store === null ? 'reserved' : await store.reserve(key, common.leaseMs); - if (state === 'reserved') delivery.add(key, entry.events); - else if (state === 'pending') pending++; - else if (state === 'done') done++; - else throw new TypeError(`replayStore.reserve answered ${String(state)}.`); - } - } catch (error) { + if (store === null) { + for (const entry of entries) delivery.add(undefined, entry.events); + return delivery; + } + const keys = []; + for (const entry of entries) keys.push(storeKeyOf(entry.key)); + const states = []; + const { failure } = await callEach(keys, async (key, index) => { + states[index] = await store.reserve(key, common.leaseMs); + }); + // The events keep the order of the request, whatever order the store + // answered in. + let invalid; + for (let i = 0; i < keys.length; i++) { + const state = states[i]; + if (state === 'reserved') delivery.add(keys[i], entries[i].events); + else if (state === 'pending') pending++; + else if (state === 'done') done++; + else invalid ??= { state }; + } + if (failure !== undefined || invalid !== undefined) { await delivery.release(); - throw error; + throw failure?.error ?? new TypeError(`replayStore.reserve answered ${String(invalid.state)}.`); } - if (store === null || delivery.reserved > 0 || done + pending === 0) return delivery; + if (delivery.reserved > 0 || done + pending === 0) return delivery; const single = entries.length === 1; if (pending > 0) { throw new WebhookVerificationError( diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index fc40af1..35d67f3 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -240,6 +240,55 @@ test('Webhooks: a store that fails releases the ids reserved before it', async ( replayStore: { ...store, reserve: () => true }, }); await assert.rejects(odd.verify(signMailtrap(mailtrapBatch('ev-9'))), /answered true/); + const silent = createMailtrapWebhookVerifier({ + secret: 'mt', + replayStore: { ...store, reserve: async () => {} }, + }); + await assert.rejects(silent.verify(signMailtrap(mailtrapBatch('ev-8'))), /answered undefined/); +}); + +test('Webhooks: the ids of a batch go to the store together, the events in their order', async () => { + // Before: each id of a batch waited for the store's answer to the one + // before it, a round trip per event of a batch of a thousand. + let inFlight = 0; + let most = 0; + const answers = []; + const call = () => + new Promise((resolve) => { + inFlight++; + most = Math.max(most, inFlight); + answers.push(() => { + inFlight--; + resolve(); + }); + }); + const store = { + reserve: async (id) => { + await call(); + return 'reserved'; + }, + commit: call, + release: call, + }; + // The store answers the last id first. + const answer = async () => { + while (answers.length === 0) await new Promise((resolve) => setImmediate(resolve)); + while (answers.length > 0) answers.pop()(); + }; + const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', replayStore: store }); + const verified = mailtrap.verify(signMailtrap(mailtrapBatch('ev-1', 'ev-2', 'ev-3'))); + await answer(); + const delivery = await verified; + assert.strictEqual(most, 3); + assert.deepStrictEqual( + delivery.events.map((event) => event.id), + ['ev-1', 'ev-2', 'ev-3'], + ); + most = 0; + const committed = delivery.commit(); + await answer(); + await committed; + assert.strictEqual(most, 3); }); test('Webhooks: an id is kept in the store by its digest, never with an address', async () => { From 711755bce38b5c5adb363fa76d8049e90777558b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:25:16 +0300 Subject: [PATCH 090/143] docs(contributing): list the gates prepublishOnly runs, kept by a test CONTRIBUTING said prepublishOnly ran lint, format check, coverage, tsd and check:dts, with the size gate in CI alone; it has run the size gate, test:runtime and check:pack too. The list before a PR is now the gates of prepublishOnly in its order, and a test fails when the two differ. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 18 ++++++++++++------ tests/unit/scripts.test.js | 10 ++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4dc6b6..5479178 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,9 @@ the runtime exports and the declarations drift apart. Node's built-in `node:test`, no external framework. - `tests/unit/*.test.js`: one file per area. Run one with `node --test tests/unit/mailer.test.js`. -- `tests/types/*.test-d.ts`: `tsd` assertions against `index.d.ts`. +- `tests/runtime/*.test.js`: the package by its name, with no mock (sockets, TLS, `fetch`, a child + process), on Node, Bun (`pnpm test:bun`) and Deno (`pnpm test:deno`). +- `tests/types/*.test-d.ts`: `tsd` assertions, one file per entry of the exports map. `pnpm run test:coverage` enforces 98% lines and statements, 90% branches and 100% functions. @@ -47,17 +49,21 @@ the bug, not the implementation. ```bash pnpm run lint -pnpm run format +pnpm run format:check pnpm run test:coverage +pnpm run test:runtime pnpm run test:types pnpm run check:dts pnpm size --check +pnpm run check:pack ``` -`prepublishOnly` runs lint, format check, coverage, tsd and `check:dts`; CI adds the bundle-size -gate (a min+gzip budget per entry of the exports map, in `scripts/size-budgets.json`), the -comparison drift check and the docs build. Treat that as the merge gate. A new subpath needs a -budget there, or `pnpm size --check` fails. +These are the gates of `prepublishOnly`, in its order (`tests/unit/scripts.test.js` keeps the two +lists equal); `pnpm run format` fixes what `format:check` finds. `pnpm size --check` holds every +entry of the exports map to its min+gzip budget in `scripts/size-budgets.json`, so a new subpath +needs one; `check:pack` holds the tarball to the exports map and the `files` allowlist. CI adds +the comparison drift check, the docs build, and the unit and runtime suites on Bun and Deno. +Treat that as the merge gate. The comparison table in `docs/guide/comparison.md` is generated: edit `docs/comparison.json` and run `pnpm docs:comparison`. diff --git a/tests/unit/scripts.test.js b/tests/unit/scripts.test.js index 94116b6..01c1a90 100644 --- a/tests/unit/scripts.test.js +++ b/tests/unit/scripts.test.js @@ -145,6 +145,16 @@ test('Scripts: check-pack wants every file of the exports map and nothing else', ]); }); +test('Scripts: CONTRIBUTING lists the gates of prepublishOnly, in its order', () => { + // Before: CONTRIBUTING said prepublishOnly ran no size gate, which it had + // run since 1.0, with test:runtime and check:pack it did not name. + const text = fs.readFileSync(path.join(ROOT, 'CONTRIBUTING.md'), 'utf8'); + const section = text.slice(text.indexOf('## Before opening a PR')); + const start = section.indexOf('```bash\n') + '```bash\n'.length; + const block = section.slice(start, section.indexOf('\n```', start)); + assert.deepStrictEqual(block.split('\n'), pkg.scripts.prepublishOnly.split(' && ')); +}); + test('Scripts: check-dts compiles the types of every entry of the exports map', () => { const { typesOf } = require('../../scripts/check-dts.js'); assert.deepStrictEqual( From ff568e1204d31d0ba753859febff6301af077cb7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:28:24 +0300 Subject: [PATCH 091/143] docs(claude): the policy of the size budgets A budget is raised in the commit that needs the room, by the smallest half-KB step that passes, with what took it in the message; lowered when its entry falls a whole KB under it; and the root grows only for the contract every subpath shares. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3eceb7d..cf7be92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -884,6 +884,11 @@ change. **This file wins** if they ever disagree. - `tests/unit/scripts.test.js` checks that every entry of the exports map has a budget in `scripts/size-budgets.json` and nothing else does, and runs `scripts/size.js --check` with budgets it always meets, budgets it never can and a missing one. +- **Size budgets** (`scripts/size-budgets.json`, min+gzip KB, the root alone and each subpath on + top of it) are raised in the commit whose change needs the room, never in one of their own, by + the smallest half-KB step that passes, and that commit's message says what took it. A budget is + lowered when its entry falls a whole KB under it. The root is in the bundle of every subpath, so + it grows only for the contract they share; what one subpath needs stays in that subpath. - New behavior needs a test that fails without the change. Network transports are tested against local fake servers (`tests/helpers/smtp/mock-smtp-server.js` on `node:net`, servers on `node:http`) or a stubbed `fetch` (`t.mock.method(globalThis, 'fetch', …)`), never against a From c2f7211c959d999c8bd28b1b92ad647fd47a3f62 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:30:09 +0300 Subject: [PATCH 092/143] test(e2e): the providers of 1.1 against their real APIs One table of Postmark, Brevo, Mailjet, MailerSend, SparkPost, ZeptoMail, SMTP2GO, Azure, Scaleway and Cloudflare, each skipped until its credentials, sender and recipient are set: a text email, HTML with an attachment, sendMany, verify(), and refused credentials as a non-retryable auth failure. The release skill asks for the opt-in suites before a release. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/npm-publish/SKILL.md | 7 + CLAUDE.md | 4 +- tests/e2e/providers.e2e.test.js | 200 ++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/providers.e2e.test.js diff --git a/.claude/skills/npm-publish/SKILL.md b/.claude/skills/npm-publish/SKILL.md index 9b14f98..2b09346 100644 --- a/.claude/skills/npm-publish/SKILL.md +++ b/.claude/skills/npm-publish/SKILL.md @@ -193,6 +193,13 @@ pnpm run lint && pnpm run format:check && pnpm run test:coverage && pnpm run tes Must exit 0. If it fails, fix and re-run. +Then ask the user whether to run the opt-in suites against the real services, which only they +can: `pnpm test:e2e` with the sandbox credentials of the providers (each suite is skipped until +its variables are set; `tests/e2e/providers.e2e.test.js` names those of the providers of 1.1), +and `pnpm test:mailpit` against the local Mailpit. A release that changed a provider's mapping or +its error handling should have run that provider's suite. Report which suites ran and which were +skipped. + ## Step 8: Summary & Next Steps Present a summary to the user: diff --git a/CLAUDE.md b/CLAUDE.md index cf7be92..dcc71b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -897,7 +897,9 @@ change. **This file wins** if they ever disagree. inbox is shared; `docker compose -f tests/mailpit/docker-compose.yml up -d` and `MAILPIT_SMTP_HOST`/`MAILPIT_API_URL`, plus `MAILPIT_SENDMAIL="docker exec -i mailer-mailpit /mailpit sendmail"` for the sendmail case) and `tests/e2e/` (`pnpm test:e2e`, each suite skipped - until its provider's variables are set; Stalwart for JMAP in `tests/e2e/jmap/`). + until its provider's variables are set; Stalwart for JMAP in `tests/e2e/jmap/`; the providers of + 1.1 in one table, `providers.e2e.test.js`, which also checks that refused credentials are a + non-retryable `auth` failure). The release skill asks for them before a release. - **Platform-dependent tests:** the mock SMTP server takes a client that has gone as no failure: a reset (`ECONNRESET`; macOS and Windows send an RST where Linux sends a FIN) and a reply written after the client closed (`EPIPE` on Linux, `ECONNABORTED` on Windows), which the 221 to diff --git a/tests/e2e/providers.e2e.test.js b/tests/e2e/providers.e2e.test.js new file mode 100644 index 0000000..51be267 --- /dev/null +++ b/tests/e2e/providers.e2e.test.js @@ -0,0 +1,200 @@ +// Not ported: the providers of 1.1 against their real APIs, each skipped until +// its variables are set: _FROM and _TO, a verified sender +// and a recipient, and the credentials named in its row below. +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { createMessage } = require('../../src/index.js'); + +const env = process.env; + +// Each provider: its transport, the variables it needs, its configuration +// from them, and one whose credentials the provider refuses. +const PROVIDERS = [ + { + name: 'Postmark', + prefix: 'POSTMARK', + Transport: require('../../src/postmark/index.js').PostmarkTransport, + needs: ['POSTMARK_SERVER_TOKEN'], + config: () => ({ + serverToken: env.POSTMARK_SERVER_TOKEN, + ...(env.POSTMARK_MESSAGE_STREAM && { messageStream: env.POSTMARK_MESSAGE_STREAM }), + }), + refused: () => ({ serverToken: '00000000-0000-0000-0000-000000000000' }), + }, + { + name: 'Brevo', + prefix: 'BREVO', + Transport: require('../../src/brevo/index.js').BrevoTransport, + needs: ['BREVO_API_KEY'], + config: () => ({ apiKey: env.BREVO_API_KEY }), + refused: () => ({ apiKey: 'xkeysib-invalid' }), + }, + { + name: 'Mailjet', + prefix: 'MAILJET', + Transport: require('../../src/mailjet/index.js').MailjetTransport, + needs: ['MAILJET_API_KEY', 'MAILJET_API_SECRET'], + config: () => ({ apiKey: env.MAILJET_API_KEY, apiSecret: env.MAILJET_API_SECRET }), + refused: () => ({ apiKey: env.MAILJET_API_KEY, apiSecret: 'invalid' }), + }, + { + name: 'MailerSend', + prefix: 'MAILERSEND', + Transport: require('../../src/mailersend/index.js').MailerSendTransport, + needs: ['MAILERSEND_API_KEY'], + config: () => ({ apiKey: env.MAILERSEND_API_KEY }), + refused: () => ({ apiKey: 'mlsn.invalid' }), + }, + { + name: 'SparkPost', + prefix: 'SPARKPOST', + Transport: require('../../src/sparkpost/index.js').SparkPostTransport, + needs: ['SPARKPOST_API_KEY'], + config: () => ({ + apiKey: env.SPARKPOST_API_KEY, + ...(env.SPARKPOST_REGION && { region: env.SPARKPOST_REGION }), + }), + refused: () => ({ + apiKey: 'invalid', + ...(env.SPARKPOST_REGION && { region: env.SPARKPOST_REGION }), + }), + }, + { + name: 'ZeptoMail', + prefix: 'ZEPTOMAIL', + Transport: require('../../src/zeptomail/index.js').ZeptoMailTransport, + needs: ['ZEPTOMAIL_TOKEN'], + config: () => ({ + token: env.ZEPTOMAIL_TOKEN, + ...(env.ZEPTOMAIL_BOUNCE_ADDRESS && { bounceAddress: env.ZEPTOMAIL_BOUNCE_ADDRESS }), + }), + refused: () => ({ token: 'Zoho-enczapikey invalid' }), + }, + { + name: 'SMTP2GO', + prefix: 'SMTP2GO', + Transport: require('../../src/smtp2go/index.js').Smtp2goTransport, + needs: ['SMTP2GO_API_KEY'], + config: () => ({ + apiKey: env.SMTP2GO_API_KEY, + ...(env.SMTP2GO_REGION && { region: env.SMTP2GO_REGION }), + }), + refused: () => ({ apiKey: 'api-00000000000000000000000000000000' }), + }, + { + name: 'Azure', + prefix: 'AZURE', + Transport: require('../../src/azure/index.js').AzureTransport, + needs: ['AZURE_CONNECTION_STRING'], + config: () => ({ connectionString: env.AZURE_CONNECTION_STRING }), + refused: () => ({ + connectionString: env.AZURE_CONNECTION_STRING.replace( + /accesskey=[^;]*/i, + `accesskey=${Buffer.from('invalid').toString('base64')}`, + ), + }), + }, + { + name: 'Scaleway', + prefix: 'SCALEWAY', + Transport: require('../../src/scaleway/index.js').ScalewayTransport, + needs: ['SCALEWAY_SECRET_KEY', 'SCALEWAY_PROJECT_ID'], + config: () => ({ + secretKey: env.SCALEWAY_SECRET_KEY, + projectId: env.SCALEWAY_PROJECT_ID, + ...(env.SCALEWAY_REGION && { region: env.SCALEWAY_REGION }), + }), + refused: () => ({ + secretKey: '00000000-0000-0000-0000-000000000000', + projectId: env.SCALEWAY_PROJECT_ID, + ...(env.SCALEWAY_REGION && { region: env.SCALEWAY_REGION }), + }), + }, + { + name: 'Cloudflare', + prefix: 'CLOUDFLARE', + Transport: require('../../src/cloudflare/index.js').CloudflareTransport, + needs: ['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN'], + config: () => ({ accountId: env.CLOUDFLARE_ACCOUNT_ID, apiToken: env.CLOUDFLARE_API_TOKEN }), + refused: () => ({ accountId: env.CLOUDFLARE_ACCOUNT_ID, apiToken: 'invalid' }), + }, +]; + +const pause = () => new Promise((resolve) => setTimeout(resolve, 1000)); + +const assertSent = (receipt) => { + assert.equal( + receipt.successful, + true, + receipt.successful ? undefined : receipt.errorMessages.join(', '), + ); + assert.ok(receipt.messageId); +}; + +for (const provider of PROVIDERS) { + const needs = [...provider.needs, `${provider.prefix}_FROM`, `${provider.prefix}_TO`]; + const missing = needs.filter((name) => !env[name]); + const skip = missing.length > 0 ? `set ${missing.join(', ')}` : false; + const message = (fields) => + createMessage({ + from: { name: 'mailer E2E', address: env[`${provider.prefix}_FROM`] }, + to: env[`${provider.prefix}_TO`], + subject: `[E2E] ${provider.name}`, + content: { text: `This is a ${provider.name} E2E test email of @alexify/mailer.` }, + tags: ['mailer-e2e'], + ...fields, + }); + + test(`E2E ${provider.name}: a text email`, { skip }, async () => { + await pause(); + const receipt = await new provider.Transport(provider.config()).send(message({})); + assertSent(receipt); + console.log(`${provider.name} text email: ${receipt.messageId}`); + }); + + test(`E2E ${provider.name}: an HTML email with an attachment`, { skip }, async () => { + await pause(); + const receipt = await new provider.Transport(provider.config()).send( + message({ + subject: `[E2E] ${provider.name} HTML and attachment`, + content: { + html: `

${provider.name}

An E2E test with a file.

`, + text: `${provider.name}\n\nAn E2E test with a file.`, + }, + attachments: [ + { + filename: 'mailer-e2e.txt', + contentType: 'text/plain', + content: new TextEncoder().encode('Attached by the mailer E2E test.'), + }, + ], + }), + ); + assertSent(receipt); + console.log(`${provider.name} HTML email: ${receipt.messageId}`); + }); + + test(`E2E ${provider.name}: two emails with sendMany`, { skip }, async () => { + await pause(); + const transport = new provider.Transport(provider.config()); + const receipts = []; + const batch = [message({ subject: '[E2E] batch 1' }), message({ subject: '[E2E] batch 2' })]; + for await (const receipt of transport.sendMany(batch)) receipts.push(receipt); + assert.equal(receipts.length, 2); + for (const receipt of receipts) assertSent(receipt); + }); + + test(`E2E ${provider.name}: verify() accepts the credentials`, { skip }, async () => { + const transport = new provider.Transport(provider.config()); + if (typeof transport.verify !== 'function') return; + await transport.verify(); + }); + + test(`E2E ${provider.name}: refused credentials are an auth failure`, { skip }, async () => { + await pause(); + const receipt = await new provider.Transport(provider.refused()).send(message({})); + assert.equal(receipt.successful, false); + assert.equal(receipt.retryable, false, receipt.errorMessages.join(', ')); + assert.equal(receipt.errors[0].category, 'auth', receipt.errorMessages.join(', ')); + }); +} From 5cac9b31652628e92ca542751860d9b3d4dbb15d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:33:29 +0300 Subject: [PATCH 093/143] docs: the changelog of 1.1 with what its review added, verifyAll in the pool page The 1.1.0 entry names verifyAll, onVerify and rfcMessageId, the logger of the decorators, the shared store of the rate limit, the probe deadline of the breaker, the timeout and bound of sendmail, the batches cut by bytes and the delivery-unknown signals of otel and logger. The pool page documents verifyAll; the README says that a webhook is taken once handled. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 35 ++++++++++++++++++++++++----------- README.md | 5 +++-- docs/transports/pool.md | 11 +++++++++++ src/pool/index.d.ts | 5 ++++- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d29ecb2..73a3303 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,9 +60,9 @@ additive but the defaults listed below. - **`scheduledAt`, `tracking` and `metadata` on the message,** from unemail's model: when the provider should deliver it, open and click tracking for this message, and values the provider returns in its events. A transport that cannot schedule refuses the message with an - `unsupported` receipt rather than sending it at once, and `./pool` fails over with it. Resend, SendGrid, Mailgun, Maileroo, - Brevo, MailerSend, SparkPost and SMTP2GO schedule; ten providers take `tracking`, seventeen - `metadata` (each page says how). + `unsupported` receipt rather than sending it at once, and `./pool` fails over with it. Resend, + SendGrid, Mailgun, Maileroo, Brevo, MailerSend, SparkPost and SMTP2GO schedule; ten providers + take `tracking`, seventeen `metadata` (each page says how). - **`./attachments`:** `fromPath`, `fromUrl`, `fromDataUri`, `fromStream` and `inlineDataUrls`, for what nodemailer read through `path`, `href` and data URIs. Nothing is read implicitly: `fromUrl` goes through an injected `fetch`, checks each redirect with `allowUrl` (by default the @@ -76,11 +76,13 @@ additive but the defaults listed below. - **`./smtp` connects through what you give it:** `connect` opens each connection (a proxy, a tunnel, a socket of another module) with implicit TLS or STARTTLS over it; `createHttpProxyConnector` makes one for an HTTP or HTTPS proxy, whose credentials go in clear - to an `http://` proxy only on this machine or with `allowPlaintextCredentials`; `localAddress` picks the address - to connect from. + to an `http://` proxy only on this machine or with `allowPlaintextCredentials`, under one + deadline for the CONNECT; `localAddress` picks the address to connect from. - **`./sendmail`:** the sendmail command of the system (Postfix, Exim, msmtp), one process per - message, composed by `./mime` with DKIM; `sendRaw` and `verify`. Only an exit with 0 succeeds, a - signal after the whole message is `deliveryUnknown`, and the sysexits codes are classified. + message, composed by `./mime` with DKIM; `sendRaw` and `verify`. Only an exit with 0 after the + whole message succeeds, a signal after it is `deliveryUnknown`, and the sysexits codes are + classified; a process has a `timeout` (60 seconds, then SIGTERM and SIGKILL) and at most + `maxConcurrent` (10) run at once. - **Raw messages and `verify()` through every decorator:** `./pool`, `./retry`, `./otel`, `./logger` and `./mock` pass `sendRaw` and `verify` on. `./ses` and `./mailgun` send raw MIME, and `./ses`, `./sendgrid`, `./resend` and `./mailgun` verify their key, one that may only send @@ -91,13 +93,15 @@ additive but the defaults listed below. signed with the access key, repeatable requests), `./scaleway` and `./cloudflare` (the REST API of Cloudflare Email Service, raw MIME). Each goes through the request layer of the others: no redirect followed, a bounded body, `deliveryUnknown` when the provider may have taken the - message. + message. The batches of Postmark and Mailjet are cut by their bytes as well as their count. - **`./breaker`:** a circuit breaker that opens after `failureThreshold` failures of the provider, answers with a retryable `service-unavailable` receipt while open (which `./pool` - fails over with), and lets exactly one probe through after `resetTimeoutMs`. + fails over with), and lets exactly one probe through after `resetTimeoutMs`, for at most + `probeTimeoutMs`. - **`./ratelimit`:** a token bucket per `key`, waiting first in, first out, or refusing with a retryable `rate-limit` receipt; an aborted send leaves the queue, and a cost over `burst` is - refused. + refused. At most `maxKeys` buckets are kept in the process, or a `store` of buckets that + processes share holds them (`take(key, cost, limits)`). - **`./idempotency`:** a key reserved in a store before the send, so a repeat is answered with the first receipt, a send in flight as of unknown delivery, and the same key for another message refused. A reservation lasts `leaseMs` (10 minutes), so a key a crash left reserved is @@ -105,7 +109,9 @@ additive but the defaults listed below. kept `ttlMs` (a day). `createMemoryIdempotencyStore` is the in-process store; a shared one implements `reserve`, `complete` and `release` with the token. - **`./hooks`:** `beforeSend` may replace or refuse a message, `afterSend`, `onRetry` and - `onError` hear each send, with a context made for each. + `onError` hear each send, with a context made for each that carries its signal. +- **A `logger` for `./breaker`, `./ratelimit`, `./idempotency` and `./tee`,** pino-like, with a + code for each event (`breaker.open`, `ratelimit.waiting`, `idempotency.in_flight`…). - **`./tee`:** mirrors each message to other transports, with one Message-ID, each with a timeout of its own and not awaited by default, the primary's receipt the one returned; disposal waits for the copies in flight. @@ -120,6 +126,13 @@ additive but the defaults listed below. it meanwhile is refused as `pending`. `replayStore` is required where a verifier keeps ids (`createMemoryReplayStore()` in one process, a shared store of `reserve`, `commit` and `release`, or `null`), and a refusal carries the `status` to answer the provider with. + `onVerify` hears each request taken or refused, and an event carries the `rfcMessageId` it was + sent with, where the provider tells it, to find the events of a message of unknown delivery. +- **`./pool` `verifyAll()`:** how each transport did, `verified`, `failed` or `unverifiable`, + where `verify()` passes when one does. +- **A failure of unknown delivery stands out:** `./otel` sets `mailer.delivery_unknown` on its + span and its message counter, and `./logger` writes `deliveryUnknown` in `email.failed`, for + the messages to reconcile. - **Bun and Deno:** CI runs the unit suite and the new `tests/runtime` (the package by its name, with no mock: sockets, TLS, DKIM, `fetch`, a child process) on Bun 1.4.3 and Deno 2.9.6, as well as on Node 22 and 24. `package.json` declares `"type": "commonjs"`, which Deno needs and diff --git a/README.md b/README.md index 46138ff..b3cc0d2 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ audit log and suppression. Zero runtime dependencies. verified, AUTH never goes over plain text, CRLF in a header is rejected, and no file or URL is read for an attachment unless you ask. - **What happens after `send()`.** The webhooks of fifteen providers, each verified before it is - read, give one model of delivery events (delivered, bounced, complained, …). Next: a persistent - queue and outbox, an append-only audit log and suppression. None of the + read and taken as handled only once your handler commits it, give one model of delivery events + (delivered, bounced, complained, …). Next: a persistent queue and outbox, an append-only audit + log and suppression. None of the [fourteen libraries compared](https://mailerat.vercel.app/guide/comparison) has the three. - **Sent once, or known to be unknown.** Errors are classified as permanent or temporary, and a failure after which the provider may have taken the message is marked `deliveryUnknown`: diff --git a/docs/transports/pool.md b/docs/transports/pool.md index 443b0a2..b807d6d 100644 --- a/docs/transports/pool.md +++ b/docs/transports/pool.md @@ -491,6 +491,17 @@ resolves when one of them does: the pool can send when one of its transports can When none does, it throws an `AggregateError` of their errors; when none can verify, it has nothing to check. +`verifyAll()` verifies every enabled transport, all at once, and never throws for +one: it gives `{ id, index, status, error? }` for each, `status` being +`verified`, `failed` or `unverifiable` (no `verify()`). A key revoked at one +provider shows there while `verify()` still passes on the others: + +```js +for (const { id, status, error } of await pool.verifyAll()) { + if (status === 'failed') logger.error({ id, err: error }, 'a transport of the pool cannot send'); +} +``` + ## Testing with pools Pool transports integrate seamlessly with testing workflows using mock diff --git a/src/pool/index.d.ts b/src/pool/index.d.ts index 3dd4780..96b11df 100644 --- a/src/pool/index.d.ts +++ b/src/pool/index.d.ts @@ -265,7 +265,10 @@ export declare class PoolTransport */ sendRaw(message: RawMessage, options?: TransportOptions): Promise>; /** - * Verifies the enabled transports that can verify: it resolves when one of them verifies, and throws an `AggregateError` of their errors when none does; it has nothing to check when none can verify. {@link PoolTransport.verifyAll} tells how each did. + * Verifies the enabled transports that can verify: it resolves when one of + * them verifies, and throws an `AggregateError` of their errors when none + * does; it has nothing to check when none can verify. + * {@link PoolTransport.verifyAll} tells how each did. */ verify(options?: TransportOptions): Promise; /** From a11c2cdcf0cfe390529cc2b6ddb16b5a5bdebaff Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:34:08 +0300 Subject: [PATCH 094/143] docs(comparison): the shared store of the rate limit, webhooks taken once handled Co-Authored-By: Claude Opus 5.5 --- docs/comparison.json | 4 ++-- docs/guide/comparison.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/comparison.json b/docs/comparison.json index 9855560..9fc5a75 100644 --- a/docs/comparison.json +++ b/docs/comparison.json @@ -328,7 +328,7 @@ }, "rateLimit": { "mark": "✅", - "text": "token bucket per key, FIFO" + "text": "token bucket per key, FIFO, shared store" }, "circuitBreaker": { "mark": "✅", @@ -434,7 +434,7 @@ "webhooks": { "mark": "✅", "text": "15 providers, verified before parsing", - "note": "A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time and a replay store." + "note": "A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time, and an id kept as taken only once the handler commits its events." }, "inbound": { "mark": "❌", diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md index 91f944a..d28af74 100644 --- a/docs/guide/comparison.md +++ b/docs/guide/comparison.md @@ -67,7 +67,7 @@ it. Failover between providers✅ RR, weighted, priority, circuit breaker2⚠️ host DNS fallback only❌✅ RR, weighted, priority✅ fallback, RR, weighted, tee✅ fallback + weighted❌❌—❌❌❌❌❌⚠️ multi-region endpoint on the AWS side SMTP connection pool✅ on by default, 5, 100 messages each✅ max 5, 100 msg❌ one connection✅ on by default✅ off by default✅ off by default❌✅——————— Retry / backoff✅ jitter, Retry-After, no blind resend2⚠️ requeue in the pool only⚠️ 1 retry on 450/451✅ jitter, Retry-After✅ 5 modes, jitter✅ no jitter, retries SMTP 5xx❌❌ retryHint only—❌❌❌❌❌✅ standard / adaptive -Rate limiting✅ token bucket per key, FIFO✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry +Rate limiting✅ token bucket per key, FIFO, shared store✅ rateLimit in the pool❌⚠️ sendMany only✅ token bucket⚠️ pool + sendBulk❌❌—❌⚠️ contact importer only❌⚠️ 429 → error❌⚠️ adaptive retry Circuit breaker✅ one probe in half-open❌❌❌✅ half-open without a single probe⚠️ cooldownMs❌❌—❌❌❌❌❌❌ Timeout / AbortSignal✅ signal + timeout over the body⚠️ timeouts, no AbortSignal⚠️ timeout, no signal✅ signal + timeout⚠️ signal for HTTP, not SMTP⚠️ SMTP connect only⚠️ socket timeouts✅ signal on connect and send—⚠️ signal, no timeout⚠️ global signal only⚠️ timeout, no signal⚠️ 180 s timeout⚠️ hard 10 s✅ abortSignal + requestTimeout Persistent queue (survives a restart)🔜 MongoDB outbox, BullMQ driver3❌❌❌🔜 planned❌❌⚠️ sample outbox only—❌❌❌❌❌❌ @@ -131,7 +131,7 @@ it. 4. Nothing is read implicitly: fromPath and fromUrl are called by the application, fromUrl checks every redirect against allowUrl (no private address by default) and bounds time and bytes. 5. A transport that cannot schedule refuses the message before sending it, rather than sending it at once. 6. The other five have no request that checks a sending key without sending a message. -7. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time and a replay store. +7. A signature where the provider signs (SNS, Svix, ECDSA, HMAC), its Basic credentials or token where it does not; a window of time, and an id kept as taken only once the handler commits its events. From 0576e08cc4a9b633247894e8e11b9da03013aa33 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:37:55 +0300 Subject: [PATCH 095/143] fix(webhooks): one rule for the mapping of events, pinned for every provider The guide now states the rule every verifier follows: deferred while the provider tries again; bounced hard, soft or undetermined as it says; rejected when nothing was sent; unknown rather than a guess. A table test pins each provider's mapping, and the cases that broke the rule are fixed: - Azure: an engagement of any type but a click was an open, and a quarantined message deferred, which Azure does not try again. - Mailgun, SendGrid, Mailjet: a failure of no severity, a bounce of no type and one without hard_bounce had a kind; SendGrid's expired, given up after its retries, was hard. - Resend, SES: a bounce without its detail, or naming no recipient, had no kind. - Postmark: an undocumented bounce type was a bounce, and a suppression after a bounce or a complaint an unsubscribe. CLAUDE.md lists the bugs of 1.1 found in its review (R1 to R20). The budget of ./webhooks goes from 14.5 to 15 KB for the maps of the rule. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 107 ++++- docs/guide/webhooks.md | 48 +- scripts/size-budgets.json | 2 +- src/webhooks/azure.js | 11 +- src/webhooks/mailgun.js | 6 +- src/webhooks/mailjet.js | 5 +- src/webhooks/postmark.js | 17 +- src/webhooks/resend.js | 7 +- src/webhooks/sendgrid.js | 10 +- src/webhooks/ses.js | 7 +- tests/unit/webhooks-mailgun.test.js | 3 +- tests/unit/webhooks-mapping.test.js | 670 +++++++++++++++++++++++++++ tests/unit/webhooks-others.test.js | 19 +- tests/unit/webhooks-sendgrid.test.js | 1 + 14 files changed, 879 insertions(+), 34 deletions(-) create mode 100644 tests/unit/webhooks-mapping.test.js diff --git a/CLAUDE.md b/CLAUDE.md index dcc71b7..fd89b87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -643,6 +643,98 @@ fails on upstream's behavior): the application pins). ZeptoMail, whose signing is not documented, Plunk, which posts no events, and Cloudflare, whose events go to a Queue, have none. +**Bugs of 1.1 found in review before its release** (each in its own `fix:` commit, or `perf:`, +with a test that fails without it; R for review): + +- R1. webhooks: an id was kept as taken inside `verify()`, before the application handled its + events, so a delivery whose handler failed came back as a replay, was answered 2xx and lost; + and a store was made silently per verifier, which neither several processes nor a verifier + made per request share. `verify()` reserves the id for `leaseMs` and returns the events with + `commit()` and `release()`, over a store of `reserve`, `commit` and `release` (`claimDelivery` + in `request.js`); another delivery meanwhile is `pending` (503); `replayStore` is required, + `null` to keep none; a refusal carries the `status` to answer. +- R2. render: `htmlToText` read the end of a string built with `+=`, which V8 flattens at each + read, so 1 MB of HTML took ten seconds, against the "linear" of U14. The text is built in + chunks, its last character and trailing whitespace kept aside. +- R3. attachments: the media type in the pattern of `inlineDataUrls` was unbounded, quadratic on + `.pem` with no query, four reads at most at once, and + `certificateTimeoutMs` and `signatureVersions` are options; a `validationUrl` must be on + `*.eventgrid.azure.net`, and a token is checked when the request carries one. +- R6. smtp: `createHttpProxyConnector` sent `Proxy-Authorization` over `http:` to any host and had + only an idle timeout. Credentials go in clear to a loopback proxy alone, unless + `allowPlaintextCredentials`, and the CONNECT has one deadline. +- R7. ratelimit: `maxKeys` only started a sweep, so the buckets grew without bound; disposal left + its timers and waiters; an abort between the messages of a `sendMany` was not heard. The least + recently used bucket goes past `maxKeys`, disposal rejects the waiters with an `AbortError`, + and each wait checks the signal. +- R8. tee and every warning of a decorator: disposal closed the mirrors with copies in flight; a + failed copy, a hook or a store that threw became a warning with its raw text, addresses + included, one per failure. Disposal waits for the copies, a copy carries the primary's + Message-ID, and warnings are redacted (`describeThrown`) and held to ten a minute per code + (`warnLimited`, `internal/callback.js`). +- R9. idempotency: one `ttlMs` held a reservation and a result, so a crash or a `break` after + `reserve` left the key pending for a day, every send of it `in_flight`; a holder whose time + ran out could complete another's reservation; the receipts of the decorator had no category + and were classified by the words of the key (`order-timeout-1` was a `timeout`). A reservation + has `leaseMs` (10 minutes) and a token that `complete` and `release` take, a message abandoned + is released, and every receipt names its category. +- R10. core, pool: a refusal by a provider's limits before any request (`scheduledAt` where it + cannot schedule, a second tag for Postmark) was `validation`, at which `./pool` stops its + failover. It is the category `unsupported`: not retried, failed over, not counted by the + breaker. +- R11. mailtrap: its own `parseRetryAfter` had not the bound of bug 29, and gave 0 for a past date. + It uses the core one. +- R12. the gate of the decorators (`internal/gate.js`): a `sendMany` its caller left (`break`) did + not settle the messages read ahead, so a probe stayed out, keys stayed pending and hooks went + unheard; and every answer given in place ended the wrapped `sendMany`, cutting the batches of a + batch API to one or two messages. `abandon` settles them in a `finally`, and one wrapped + `sendMany` runs, the answers in place put in the order of their messages. +- R13. breaker: a probe that never ended kept the circuit half-open for good, an `isFailure` that + threw broke its state, a failed batch counted a failure per message, and a late result counted + in a later period. A probe has `probeTimeoutMs`, the callbacks are guarded, a batch counts one + failure, and a result counts in the period it was sent in. +- R14. sendmail: no timeout, no bound on the processes, success on an exit with 0 before the whole + message was written, and an abort sent SIGTERM alone. `timeout` (60 seconds; SIGTERM, then + SIGKILL 5 seconds on; `deliveryUnknown` once the message was written), `maxConcurrent` (10), + and such an exit fails. +- R15. postmark, mailjet: a batch was cut by its count alone (500, 50), so its attachments were + held whole and could pass the provider's limit, which refuses the whole batch. It is cut by its + bytes too (50 MB, 15 MB). +- R16. hooks: the hooks had no signal, a rejection of an async `onRetry` went unheard, and + `beforeSend` could return anything. The context carries the signal, an abort during + `beforeSend` sends nothing, a rejected `onRetry` is a warning, and a value not shaped as a + message refuses it. +- R17. breaker, ratelimit, the memory stores and the replay store: intervals on `Date.now`, which + a step of the clock moves. They are measured on `performance.now()`. +- R18. webhooks: Basic and Bearer credentials were compared with an early exit on a length apart; + the store was given ids as they came, one of which names a recipient (Mailtrap without + `event_id`); the verifiers of providers that sign no time took `toleranceMs` and `now` and + never read them; a batch reserved its ids one round trip at a time. Credentials compare as + HMACs, the store is given a SHA-256 of each id, those options throw, and the store is called + 64 ids at a time. +- R19. core: `metadata` had no bound on its keys or values. 64 keys, values of 1024 characters. +- R20. webhooks: the mapping of events guessed where the provider did not say. An Azure + engagement of any type but a click was an open, and a quarantined message deferred, which + Azure does not try again; a Mailgun failure of no severity, a SendGrid bounce of no type and a + Mailjet bounce without `hard_bounce` had a kind (and SendGrid's `expired`, given up after its + retries, was hard); a Resend bounce without its detail and an SES bounce of no recipient had + none; Postmark made any undocumented bounce type a bounce, and any suppression an + unsubscribe. The rule of `docs/guide/webhooks.md` (`deferred` while the provider tries again; + `bounced` hard, soft or undetermined as it says; `rejected` with nothing sent; `unknown` rather + than a guess) holds for all, and `tests/unit/webhooks-mapping.test.js` pins every provider's + table. + ## Repository layout ``` @@ -656,8 +748,8 @@ src/ delivery-unknown.js, config.js, loopback.js, validate-ssl.js, verify.js), receipts (provider.js, refusal.js), message fields (headers.js, raw.js, schedule.js, tracking.js, capabilities.js, batch.js, base64.js, uuid.js), - the decorators of 1.1 (gate.js, callback.js, options.js), and redact.js, - redact-addresses.js, sleep.js, log.js + the decorators of 1.1 (gate.js, callback.js, options.js, decorator.js), + and redact.js, redact-addresses.js, sleep.js, log.js tests/ unit/*.test.js node:test suites, one per area (-.test.js for a subpath) runtime/*.test.js the package by its name on Node, Bun and Deno, with no mock @@ -690,10 +782,10 @@ The root entry is the contract every subpath shares, ported from `@upyo/core` (` `verify()`. - **`Receipt`**: `{ successful: true, messageId }` or `{ successful: false, errorMessages, errors, retryable }`. A `ReceiptError` carries a `category` - (auth, rate-limit, network, timeout, validation, rejected, server-error, service-unavailable, - configuration, unknown), `retryable`, `statusCode` and `retryAfterMilliseconds`; - `classifyHttpStatus` and `classifyReceiptError` classify. A transport reports a failure in a - receipt rather than by throwing. + (auth, rate-limit, network, timeout, validation, rejected, unsupported, server-error, + service-unavailable, configuration, unknown), `retryable`, `statusCode` and + `retryAfterMilliseconds`; `classifyHttpStatus` and `classifyReceiptError` classify. A transport + reports a failure in a receipt rather than by throwing. The contracts of the reliability layer, still to be designed (Phase 0): @@ -751,7 +843,8 @@ circuit breaker with a single probe in half-open, and no switch after an ambiguo without an idempotency key (`./pool` has strategies and failover, `./breaker` of 1.1 takes a failing provider out of it; no health checks yet); OpenTelemetry spans and metrics (time in queue, attempts, failure rate per provider, outbox size; `./otel` covers sends); token-bucket rate -limits per provider and per tenant (SES quotas; `./ratelimit` of 1.1, by any key). +limits per provider and per tenant (SES quotas; `./ratelimit` of 1.1, by any key, with a +store of buckets that processes share). **Phase 3, multi-tenant (P1–P2).** Provider credentials per tenant (bring your own SES), domains and DKIM per tenant, quotas; List-Unsubscribe one-click (RFC 8058, done in 1.1) and the split diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 1778338..d7d4194 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -79,9 +79,22 @@ as many events. | `tags`, `metadata` | The tags and metadata of the message, as the transport sent them. | | `raw` | The provider's record. | -`rejected` is a message the provider never attempted, such as one to a -suppressed address; `deferred` one it keeps trying; `unknown` an event the model -has no name for, kept with its `raw` record. +Every provider's events are read by one rule, so that the same outcome has the +same type whoever reports it: + +| What the provider says | `type` | `bounce.kind` | +| --- | --- | --- | +| It has not sent the message and will not: a suppressed, blocked or invalid address, a policy, a template that did not render. | `rejected` | | +| An attempt failed and it will try again (Mailgun's temporary failure, Postmark's `Transient`). The message stays accepted. | `deferred` | `soft`, when it gives a bounce | +| It gave up after an attempt, and says the failure is permanent. Suppress the address. | `bounced` | `hard` | +| It gave up after an attempt, and says the failure is temporary or soft (a full mailbox, a block). | `bounced` | `soft` | +| It gave up after an attempt, and does not say which (Azure, SparkPost's class 1). | `bounced` | `undetermined` | +| An open or a click it documents. | `opened`, `clicked` | | +| Anything it does not document, or a name the model has none for. | `unknown` | | + +An `unknown` event is never a guess: it keeps the provider's `raw` record, +for the application to read. `tests/unit/webhooks-mapping.test.js` holds the +mapping of every provider's events to this table. A receipt of unknown delivery (`deliveryUnknown`) has no `messageId` of the provider to find its events by. Give the message a `messageId` of its own, or @@ -198,8 +211,9 @@ Turn on the Signed Event Webhook in SendGrid and pass its verification key (base64, as SendGrid shows it, or PEM). The signature is ECDSA P-256 over the timestamp and the body. SendGrid posts events in batches: each event is taken once by its `sg_event_id`, so a batch sent again keeps only what is not -committed, and is refused as `replay` when nothing is left. A `bounce` is hard, a `blocked` one soft; a -`dropped` message is `rejected`; an unsubscribe is `unsubscribed`. The +committed, and is refused as `replay` when nothing is left. A bounce of the type +`bounce` is hard, a `blocked` or `expired` one soft, and one of no type +undetermined; a `dropped` message is `rejected`; an unsubscribe is `unsubscribed`. The categories are the tags of the message and its custom arguments its metadata; `messageId` is the `X-Message-Id` of the receipt, without the filter SendGrid adds after it. @@ -219,8 +233,9 @@ neither the body nor the event in it, so a timestamp, token and signature seen once can carry any event until the token is taken: give every process the same shared `replayStore`, as one store per process takes the same token once in each. A permanent -failure is a hard bounce, a temporary one a deferred soft bounce, and a -failure for a suppressed address `rejected`. The user variables are the +failure is a hard bounce, a temporary one a deferred soft bounce, one of no +severity an undetermined bounce, and a failure for a suppressed address +`rejected`. The user variables are the metadata, and `messageId` the id of the receipt, in angle brackets. ## Postmark @@ -233,9 +248,14 @@ const postmark = createPostmarkWebhookVerifier({ username: 'hook', password: 's3 Postmark signs nothing, so the URL of the webhook carries credentials, which the verifier requires and compares in constant time. With no time or id signed, nothing holds a request to a window or takes it once, and it needs no -`replayStore`: deduplicate by the `id` of the events. A bounce is read by its type: `HardBounce` is hard, -`SoftBounce` soft, `Transient` deferred, `SpamComplaint` a complaint, -`ManuallyDeactivated` a rejection. +`replayStore`: deduplicate by the `id` of the events. A bounce is read by its +type: `HardBounce` is hard, `SoftBounce` soft, `Transient` deferred, +`SpamComplaint` a complaint, `ManuallyDeactivated` a rejection, `Unknown` +undetermined, and a notice that follows no attempt (`VirusNotification`, +`OpenRelayTest`) or a type Postmark does not document `unknown`. A subscription +change is `unsubscribed` when it suppresses the address by hand or by a link +(`ManualSuppression`); one that follows a bounce or a complaint is `unknown`, +that event having come on its own. ## MailerSend and Mailtrap @@ -281,7 +301,8 @@ const sparkpost = createSparkPostWebhookVerifier({ username: 'hook', password: ' Neither signs its events, so their webhooks carry Basic credentials (in Mailjet's callback URL, SparkPost's `auth_type: basic`), which the verifiers require. Mailjet's message id is the MessageUUID of the receipt, its custom -campaign the tag and its event payload the metadata; SparkPost's message id is +campaign the tag and its event payload the metadata, and a bounce is hard or +soft by its `hard_bounce`, undetermined without it; SparkPost's message id is the transmission's, and a bounce is hard for the classes 10, 30 and 90, undetermined for 1, soft for the others. As for Postmark, nothing holds a request to a window or takes it once. @@ -350,8 +371,9 @@ validation of a new subscription carries no event and is handed back for the application to answer, its token checked when it carries one; its `validationUrl`, for a validation by hand, must be on `*.eventgrid.azure.net` over HTTPS, or the request is refused. Delivery reports are `delivered`, `bounced`, -`rejected` (Suppressed) or `deferred` (Quarantined); engagement reports -`opened` or `clicked`. The message id is that of the send operation, the +`rejected` (Suppressed), or `unknown` for Quarantined, which Azure does not try +again; an engagement report is +`opened` for a `view`, `clicked` for a `click`, and `unknown` for another. The message id is that of the send operation, the receipt's. ## Scaleway diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 7aa0ad4..56dbedc 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4.5, "./render": 3, - "./webhooks": 14.5, + "./webhooks": 15, "./smtp": 23, "./sendmail": 11.5, "./jmap": 13, diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index 13262b9..fef5a60 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -33,7 +33,14 @@ const DELIVERY_TYPES = new Map([ ['Failed', 'bounced'], ['FilteredSpam', 'bounced'], ['Suppressed', 'rejected'], - ['Quarantined', 'deferred'], + // Held by the recipient's system, which Azure does not try again: the model + // has no name for it. + ['Quarantined', 'unknown'], +]); +// The engagement types Azure documents; another is unknown, not an open. +const ENGAGEMENT_TYPES = new Map([ + ['view', 'opened'], + ['click', 'clicked'], ]); const eventOf = (record) => { @@ -45,7 +52,7 @@ const eventOf = (record) => { type = DELIVERY_TYPES.get(data.status) ?? 'unknown'; at = data.deliveryAttemptTimeStamp ?? at; } else if (record.eventType === ENGAGEMENT) { - type = data.engagementType === 'click' ? 'clicked' : 'opened'; + type = ENGAGEMENT_TYPES.get(data.engagementType) ?? 'unknown'; at = data.userActionTimeStamp ?? at; if (type === 'clicked') url = stringOf(data.engagementContext); } diff --git a/src/webhooks/mailgun.js b/src/webhooks/mailgun.js index a35e893..d5f9dbc 100644 --- a/src/webhooks/mailgun.js +++ b/src/webhooks/mailgun.js @@ -44,7 +44,11 @@ const failureOf = (data) => { const status = isRecord(data['delivery-status']) ? data['delivery-status'] : {}; const reason = stringOf(data.reason); if (reason?.startsWith('suppress-')) return { type: 'rejected', reason }; - const bounce = { kind: data.severity === 'temporary' ? 'soft' : 'hard' }; + // A failure that gives no severity does not say which it is. + let kind = 'undetermined'; + if (data.severity === 'temporary') kind = 'soft'; + else if (data.severity === 'permanent') kind = 'hard'; + const bounce = { kind }; if (status.code !== undefined && status.code !== null) bounce.code = String(status.code); const diagnostic = stringOf(status.description) ?? stringOf(status.message); if (diagnostic !== undefined) bounce.diagnostic = diagnostic; diff --git a/src/webhooks/mailjet.js b/src/webhooks/mailjet.js index 6401b0e..9fd34b6 100644 --- a/src/webhooks/mailjet.js +++ b/src/webhooks/mailjet.js @@ -32,7 +32,10 @@ const eventOf = (record) => { const type = TYPES.get(record.event) ?? 'unknown'; let bounce; if (type === 'bounced') { - bounce = { kind: record.hard_bounce === true ? 'hard' : 'soft' }; + let kind = 'undetermined'; + if (record.hard_bounce === true) kind = 'hard'; + else if (record.hard_bounce === false) kind = 'soft'; + bounce = { kind }; if (stringOf(record.error) !== undefined) bounce.diagnostic = record.error; } // The MessageUUID of the send's answer, the receipt's id. diff --git a/src/webhooks/postmark.js b/src/webhooks/postmark.js index e999b3e..a66a260 100644 --- a/src/webhooks/postmark.js +++ b/src/webhooks/postmark.js @@ -44,6 +44,10 @@ const BOUNCES = new Map([ ['Subscribe', { type: 'unknown' }], ['ChallengeVerification', { type: 'unknown' }], ['InboundError', { type: 'unknown' }], + ['VirusNotification', { type: 'unknown' }], + ['OpenRelayTest', { type: 'unknown' }], + ['Unknown', { type: 'bounced', kind: 'undetermined' }], + ['Unconfirmed', { type: 'bounced', kind: 'undetermined' }], ]); const TYPES = new Map([ ['Delivery', 'delivered'], @@ -54,7 +58,11 @@ const TYPES = new Map([ const describeRecord = (record) => { if (record.RecordType === 'Bounce') { - const known = BOUNCES.get(record.Type) ?? { type: 'bounced', kind: 'undetermined' }; + // A bounce that gives no type is a bounce of a kind not said; a type + // Postmark does not document is not a guess. + const known = + BOUNCES.get(record.Type) ?? + (record.Type === undefined ? { type: 'bounced', kind: 'undetermined' } : { type: 'unknown' }); let bounce; if (known.kind !== undefined) { bounce = { kind: known.kind }; @@ -66,7 +74,12 @@ const describeRecord = (record) => { } if (record.RecordType === 'SubscriptionChange') { return { - type: record.SuppressSending === true ? 'unsubscribed' : 'unknown', + // A suppression after a bounce or a complaint followed that event; + // only one by hand or by a link is an unsubscribe. + type: + record.SuppressSending === true && record.SuppressionReason === 'ManualSuppression' + ? 'unsubscribed' + : 'unknown', reason: stringOf(record.SuppressionReason), at: record.ChangedAt, }; diff --git a/src/webhooks/resend.js b/src/webhooks/resend.js index d4422ad..ebfdd8f 100644 --- a/src/webhooks/resend.js +++ b/src/webhooks/resend.js @@ -91,9 +91,10 @@ const eventsOf = (payload, id) => { const type = TYPES.get(payload.type) ?? 'unknown'; const { tags, metadata } = splitNamedTags(data.tags); let bounce; - if (type === 'bounced' && isRecord(data.bounce)) { - bounce = { kind: bounceKindOf(data.bounce.type) }; - if (stringOf(data.bounce.message) !== undefined) bounce.diagnostic = data.bounce.message; + if (type === 'bounced') { + const detail = isRecord(data.bounce) ? data.bounce : {}; + bounce = { kind: bounceKindOf(detail.type) }; + if (stringOf(detail.message) !== undefined) bounce.diagnostic = detail.message; } const detail = data.bounce ?? data.failed ?? data.suppressed ?? data.delivery_delayed; const reason = isRecord(detail) diff --git a/src/webhooks/sendgrid.js b/src/webhooks/sendgrid.js index b0785b5..80b1371 100644 --- a/src/webhooks/sendgrid.js +++ b/src/webhooks/sendgrid.js @@ -112,11 +112,19 @@ const messageIdOf = (record) => { return dot > 0 ? id.slice(0, dot) : id; }; +// A bounce is hard, a block or a message that expired after its retries soft; +// one of no type does not say. +const SENDGRID_BOUNCE_KINDS = new Map([ + ['bounce', 'hard'], + ['blocked', 'soft'], + ['expired', 'soft'], +]); + const eventOf = (record, index, timestamp) => { const type = TYPES.get(record.event) ?? 'unknown'; let bounce; if (record.event === 'bounce') { - bounce = { kind: record.type === 'blocked' ? 'soft' : 'hard' }; + bounce = { kind: SENDGRID_BOUNCE_KINDS.get(record.type) ?? 'undetermined' }; if (stringOf(record.status) !== undefined) bounce.code = record.status; if (stringOf(record.reason) !== undefined) bounce.diagnostic = record.reason; } diff --git a/src/webhooks/ses.js b/src/webhooks/ses.js index 7cef6e4..78f936a 100644 --- a/src/webhooks/ses.js +++ b/src/webhooks/ses.js @@ -71,7 +71,7 @@ const onlyDestinationOf = (mail) => { const bounceOf = (bounce) => { const kind = bounceKindOf(bounce.bounceType); - return recipientsOf(bounce.bouncedRecipients, (item) => { + const recipients = recipientsOf(bounce.bouncedRecipients, (item) => { const recipient = stringOf(item?.emailAddress); if (recipient === undefined) return undefined; const details = { kind }; @@ -79,6 +79,11 @@ const bounceOf = (bounce) => { if (stringOf(item.diagnosticCode) !== undefined) details.diagnostic = item.diagnosticCode; return { recipient, bounce: details, reason: stringOf(bounce.bounceSubType) }; }); + // A bounce that names no recipient is still a bounce, of its kind. + if (recipients.length === 0) { + recipients.push({ bounce: { kind }, reason: stringOf(bounce.bounceSubType) }); + } + return recipients; }; const emailsOf = (list) => diff --git a/tests/unit/webhooks-mailgun.test.js b/tests/unit/webhooks-mailgun.test.js index 3cb3c67..c602291 100644 --- a/tests/unit/webhooks-mailgun.test.js +++ b/tests/unit/webhooks-mailgun.test.js @@ -221,7 +221,8 @@ test('Webhooks Postmark: a bounce is read by its type', async () => { ['Unsubscribe', 'unsubscribed', undefined], ['ManuallyDeactivated', 'rejected', undefined], ['AutoResponder', 'unknown', undefined], - ['SomethingNew', 'bounced', 'undetermined'], + // A type Postmark does not document is unknown, not a guess. + ['SomethingNew', 'unknown', undefined], ]; for (const [type, eventType, kind] of cases) { const { events } = await verifier.verify(bounce(type)); diff --git a/tests/unit/webhooks-mapping.test.js b/tests/unit/webhooks-mapping.test.js new file mode 100644 index 0000000..4ca64be --- /dev/null +++ b/tests/unit/webhooks-mapping.test.js @@ -0,0 +1,670 @@ +// Not ported: the mapping of every provider's events by the verifiers of +// ./webhooks to the type and bounce kind of a DeliveryEvent, one table per +// provider (docs/guide/webhooks.md): deferred when the provider tries again, +// bounced when it gave up after an attempt, hard, soft or undetermined as it +// says, rejected when it sent nothing, unknown for anything else. +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createHmac, generateKeyPairSync, sign } = require('node:crypto'); +const { + createAzureWebhookVerifier, + createBrevoWebhookVerifier, + createLettermintWebhookVerifier, + createMailerSendWebhookVerifier, + createMailerooWebhookVerifier, + createMailgunWebhookVerifier, + createMailjetWebhookVerifier, + createMailtrapWebhookVerifier, + createPostmarkWebhookVerifier, + createResendWebhookVerifier, + createScalewayWebhookVerifier, + createSendGridWebhookVerifier, + createSesWebhookVerifier, + createSmtp2goWebhookVerifier, + createSparkPostWebhookVerifier, +} = require('../../src/webhooks/index.js'); +const { SECRET, signSvix } = require('../helpers/webhooks/svix.js'); +const { + TOPIC, + certificateFetch, + notification, + scaleway, + signSns, +} = require('../helpers/webhooks/sns.js'); + +const NOW = 1_800_000_000_000; +const SNS_NOW = Date.parse('2027-01-01T12:00:30Z'); +const hmac = (secret, body) => createHmac('sha256', secret).update(body).digest('hex'); +const basic = (user, password) => `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; + +// A row is [input, type, kind]: what the provider sends (its event name, or +// the fields that tell it apart), the type of the event and the kind of its +// bounce, absent when it has none. +const fieldsOf = (input, name) => (typeof input === 'string' ? { [name]: input } : input); +const expectedOf = (rows) => rows.map(([input, type, kind]) => [input, type, kind]); +const mappedOf = (input, event) => [input, event.type, event.bounce?.kind]; + +// The events of one request holding every row, in the order of the rows. +const batchOf = (rows, events) => events.map((event, index) => mappedOf(rows[index]?.[0], event)); + +// The events of one request per row; a row that gives several events, or +// none, shows in the diff. +const eachOf = async (rows, verify) => { + const actual = []; + for (let index = 0; index < rows.length; index++) { + const input = rows[index][0]; + const { events } = await verify(input, index); + for (const event of events) actual.push(mappedOf(input, event)); + } + return actual; +}; + +test('Webhooks mapping: Brevo, by event, a bounce hard or soft by its name', async () => { + const rows = [ + ['request', 'accepted'], + ['sent', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + ['hard_bounce', 'bounced', 'hard'], + ['hardBounce', 'bounced', 'hard'], + ['soft_bounce', 'bounced', 'soft'], + ['softBounce', 'bounced', 'soft'], + ['blocked', 'rejected'], + ['invalid', 'rejected'], + ['invalid_email', 'rejected'], + ['error', 'rejected'], + ['spam', 'complained'], + ['complaint', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['unsubscribed', 'unsubscribed'], + ['opened', 'opened'], + ['unique_opened', 'opened'], + ['uniqueOpened', 'opened'], + ['proxy_open', 'opened'], + ['click', 'clicked'], + ['unique_click', 'clicked'], + ['list_addition', 'unknown'], + ['no_such_event', 'unknown'], + ]; + const verifier = createBrevoWebhookVerifier({ token: 'brevo-token' }); + const body = JSON.stringify( + rows.map(([event], index) => ({ + event, + email: 'bob@example.com', + 'message-id': ``, + ts_event: 1_800_000_000, + })), + ); + const { events } = await verifier.verify({ + body, + headers: { Authorization: 'Bearer brevo-token' }, + }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); + +test('Webhooks mapping: Azure, by delivery status and engagement type', async () => { + const rows = [ + [{ status: 'Delivered' }, 'delivered'], + [{ status: 'Bounced' }, 'bounced', 'undetermined'], + [{ status: 'Failed' }, 'bounced', 'undetermined'], + [{ status: 'FilteredSpam' }, 'bounced', 'undetermined'], + [{ status: 'Suppressed' }, 'rejected'], + [{ status: 'Quarantined' }, 'unknown'], + [{ status: 'Expanded' }, 'unknown'], + [{ status: 'NoSuchStatus' }, 'unknown'], + [{ engagementType: 'view' }, 'opened'], + [{ engagementType: 'click' }, 'clicked'], + [{ engagementType: 'forward' }, 'unknown'], + [{ eventType: 'Microsoft.Communication.NoSuchEvent' }, 'unknown'], + ]; + const recordOf = ({ status, engagementType, eventType }, index) => { + const id = `az-${index}`; + if (status !== undefined) { + return { + id, + eventType: 'Microsoft.Communication.EmailDeliveryReportReceived', + data: { status, messageId: 'op-1', recipient: 'bob@example.com' }, + }; + } + if (engagementType !== undefined) { + return { + id, + eventType: 'Microsoft.Communication.EmailEngagementTrackingReportReceived', + data: { engagementType, messageId: 'op-1', recipient: 'bob@example.com' }, + }; + } + return { id, eventType, data: {} }; + }; + const verifier = createAzureWebhookVerifier({ token: 'eg-token', replayStore: null }); + const { events } = await verifier.verify({ + body: JSON.stringify(rows.map(([input], index) => recordOf(input, index))), + headers: { 'aeg-event-type': 'Notification', Authorization: 'Bearer eg-token' }, + }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); + +test('Webhooks mapping: Lettermint, by event, a bounce hard or soft by its name', async () => { + const rows = [ + ['message.created', 'accepted'], + ['message.sent', 'accepted'], + ['message.delivered', 'delivered'], + ['message.hard_bounced', 'bounced', 'hard'], + ['message.soft_bounced', 'bounced', 'soft'], + ['message.spam_complaint', 'complained'], + ['message.failed', 'rejected'], + ['message.suppressed', 'rejected'], + ['message.policy_rejected', 'rejected'], + ['message.unsubscribed', 'unsubscribed'], + ['message.opened', 'opened'], + ['message.clicked', 'clicked'], + ['message.no_such_event', 'unknown'], + ]; + const secret = 'whsec_lettermint'; + const verifier = createLettermintWebhookVerifier({ secret, now: () => NOW, replayStore: null }); + const actual = await eachOf(rows, (event, index) => { + const body = JSON.stringify({ + id: `lm-${index}`, + event, + timestamp: '2027-01-15T08:00:00Z', + data: { message_id: 'lm-msg-1', recipient: 'bob@example.com' }, + }); + const t = '1800000000'; + return verifier.verify({ + body, + headers: { 'X-Lettermint-Signature': `t=${t},v1=${hmac(secret, `${t}.${body}`)}` }, + }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: MailerSend, by activity, a bounce hard or soft by its name', async () => { + const rows = [ + ['activity.sent', 'accepted'], + ['activity.delivered', 'delivered'], + ['activity.deferred', 'deferred'], + ['activity.soft_bounced', 'bounced', 'soft'], + ['activity.hard_bounced', 'bounced', 'hard'], + ['activity.suppressed', 'rejected'], + ['activity.spam_complaint', 'complained'], + ['activity.unsubscribed', 'unsubscribed'], + ['activity.opened', 'opened'], + ['activity.opened_unique', 'opened'], + ['activity.clicked', 'clicked'], + ['activity.clicked_unique', 'clicked'], + ['activity.no_such_event', 'unknown'], + ]; + const verifier = createMailerSendWebhookVerifier({ secret: 'ms-secret', replayStore: null }); + const actual = await eachOf(rows, (type, index) => { + const body = JSON.stringify({ + type, + created_at: '2027-01-15T08:00:00.000000Z', + data: { id: `act-${index}`, message_id: 'msg-1', recipient: 'bob@example.com' }, + }); + return verifier.verify({ body, headers: { Signature: hmac('ms-secret', body) } }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: Mailgun, a failure by its severity and reason', async () => { + const failed = (fields) => ({ event: 'failed', ...fields }); + const rows = [ + ['accepted', 'accepted'], + ['delivered', 'delivered'], + [failed({ severity: 'temporary' }), 'deferred', 'soft'], + [failed({ severity: 'temporary', reason: 'generic' }), 'deferred', 'soft'], + [failed({ severity: 'permanent' }), 'bounced', 'hard'], + [failed({ severity: 'permanent', reason: 'old' }), 'bounced', 'hard'], + // A failure that gives no severity does not say which it is. + ['failed', 'bounced', 'undetermined'], + [failed({ severity: 'permanent', reason: 'suppress-bounce' }), 'rejected'], + [failed({ severity: 'permanent', reason: 'suppress-unsubscribe' }), 'rejected'], + [failed({ severity: 'permanent', reason: 'suppress-complaint' }), 'rejected'], + ['rejected', 'rejected'], + ['complained', 'complained'], + ['unsubscribed', 'unsubscribed'], + ['opened', 'opened'], + ['clicked', 'clicked'], + ['stored', 'unknown'], + ['no_such_event', 'unknown'], + ]; + const key = 'key-mailgun-signing'; + const verifier = createMailgunWebhookVerifier({ + signingKey: key, + now: () => NOW, + replayStore: null, + }); + const actual = await eachOf(rows, (input, index) => { + const timestamp = '1800000000'; + const token = `token-${index}`; + const body = JSON.stringify({ + signature: { timestamp, token, signature: hmac(key, timestamp + token) }, + 'event-data': { + id: `mg-${index}`, + timestamp: 1_799_999_990, + recipient: 'bob@example.com', + message: { headers: { 'message-id': '20270115.1@mg.example.com' } }, + ...fieldsOf(input, 'event'), + }, + }); + return verifier.verify({ body, headers: {} }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: Maileroo, by event, a failure hard and a deferral soft', async () => { + const rows = [ + ['accepted', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred', 'soft'], + ['failed', 'bounced', 'hard'], + ['rejected', 'rejected'], + ['complained', 'complained'], + ['opened', 'opened'], + ['clicked', 'clicked'], + ['no_such_event', 'unknown'], + ]; + const verifier = createMailerooWebhookVerifier({ secret: 'mr-secret', replayStore: null }); + const actual = await eachOf(rows, (type, index) => { + const body = JSON.stringify({ + event_id: `mr-${index}`, + event_type: type, + event_time: 1_800_000_000, + message_reference_id: 'ref-1', + event_data: { to: 'bob@example.com' }, + }); + return verifier.verify({ + body, + headers: { 'x-maileroo-signature': hmac('mr-secret', body) }, + }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: Mailjet, a bounce hard or soft by hard_bounce', async () => { + const rows = [ + ['sent', 'delivered'], + [{ event: 'bounce', hard_bounce: true }, 'bounced', 'hard'], + [{ event: 'bounce', hard_bounce: false }, 'bounced', 'soft'], + // A bounce that does not say is undetermined. + ['bounce', 'bounced', 'undetermined'], + ['blocked', 'rejected'], + ['spam', 'complained'], + ['unsub', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], + ['no_such_event', 'unknown'], + ]; + const verifier = createMailjetWebhookVerifier({ username: 'mj', password: 'pw' }); + const body = JSON.stringify( + rows.map(([input], index) => ({ + email: 'bob@example.com', + time: 1_800_000_000, + MessageID: index, + ...fieldsOf(input, 'event'), + })), + ); + const { events } = await verifier.verify({ body, headers: { authorization: basic('mj', 'pw') } }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); + +test('Webhooks mapping: Postmark, a bounce by its Type', async () => { + const bounce = (Type) => ({ RecordType: 'Bounce', Type }); + const change = (fields) => ({ RecordType: 'SubscriptionChange', ...fields }); + const rows = [ + ['Delivery', 'delivered'], + ['SpamComplaint', 'complained'], + ['Open', 'opened'], + ['Click', 'clicked'], + [bounce('HardBounce'), 'bounced', 'hard'], + [bounce('BadEmailAddress'), 'bounced', 'hard'], + [bounce('DMARCPolicy'), 'bounced', 'hard'], + [bounce('SoftBounce'), 'bounced', 'soft'], + [bounce('DnsError'), 'bounced', 'soft'], + [bounce('Blocked'), 'bounced', 'soft'], + [bounce('Transient'), 'deferred', 'soft'], + [bounce('SpamComplaint'), 'complained'], + [bounce('SpamNotification'), 'complained'], + [bounce('Unsubscribe'), 'unsubscribed'], + [bounce('ManuallyDeactivated'), 'rejected'], + [bounce('SMTPApiError'), 'rejected'], + [bounce('TemplateRenderingFailed'), 'rejected'], + [bounce('AutoResponder'), 'unknown'], + [bounce('AddressChange'), 'unknown'], + [bounce('Subscribe'), 'unknown'], + [bounce('ChallengeVerification'), 'unknown'], + [bounce('InboundError'), 'unknown'], + // The bounces Postmark cannot classify. + [bounce('Unknown'), 'bounced', 'undetermined'], + [bounce('Unconfirmed'), 'bounced', 'undetermined'], + // Notices that follow no attempt, and a type Postmark does not document. + [bounce('VirusNotification'), 'unknown'], + [bounce('OpenRelayTest'), 'unknown'], + [bounce('NoSuchBounce'), 'unknown'], + [{ RecordType: 'Bounce' }, 'bounced', 'undetermined'], + // Only a suppression by hand or by a link is an unsubscribe; one after a + // bounce or a complaint followed that event. + [change({ SuppressSending: true }), 'unknown'], + [change({ SuppressSending: true, SuppressionReason: 'ManualSuppression' }), 'unsubscribed'], + [change({ SuppressSending: true, SuppressionReason: 'HardBounce' }), 'unknown'], + [change({ SuppressSending: true, SuppressionReason: 'SpamComplaint' }), 'unknown'], + [change({ SuppressSending: false }), 'unknown'], + ['NoSuchRecord', 'unknown'], + ]; + const verifier = createPostmarkWebhookVerifier({ username: 'hook', password: 's3cret' }); + const actual = await eachOf(rows, (input, index) => + verifier.verify({ + body: JSON.stringify({ + ID: index, + MessageID: 'pm-1', + Recipient: 'bob@example.com', + ...fieldsOf(input, 'RecordType'), + }), + headers: { Authorization: basic('hook', 's3cret') }, + }), + ); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: Mailtrap, by event, a bounce hard or soft by its name', async () => { + const rows = [ + ['delivery', 'delivered'], + ['soft bounce', 'bounced', 'soft'], + ['bounce', 'bounced', 'hard'], + ['suspension', 'rejected'], + ['reject', 'rejected'], + ['spam', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], + ['no_such_event', 'unknown'], + ]; + const verifier = createMailtrapWebhookVerifier({ secret: 'mt-secret', replayStore: null }); + const body = JSON.stringify({ + events: rows.map(([event], index) => ({ + event, + event_id: `mt-${index}`, + message_id: 'mt-msg-1', + email: 'bob@example.com', + timestamp: 1_800_000_000, + })), + }); + const { events } = await verifier.verify({ + body, + headers: { 'Mailtrap-Signature': hmac('mt-secret', body) }, + }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); + +test('Webhooks mapping: Resend, a bounce by the type SES gives it', async () => { + const bounced = (bounce) => ({ type: 'email.bounced', bounce }); + const rows = [ + ['email.sent', 'accepted'], + ['email.scheduled', 'accepted'], + ['email.delivered', 'delivered'], + ['email.delivery_delayed', 'deferred'], + [bounced({ type: 'Permanent' }), 'bounced', 'hard'], + [bounced({ type: 'Transient' }), 'bounced', 'soft'], + [bounced({ type: 'Undetermined' }), 'bounced', 'undetermined'], + [bounced({ message: 'no type' }), 'bounced', 'undetermined'], + // With no bounce object, the bounce does not say its kind. + ['email.bounced', 'bounced', 'undetermined'], + ['email.failed', 'rejected'], + ['email.suppressed', 'rejected'], + ['email.complained', 'complained'], + ['email.opened', 'opened'], + ['email.clicked', 'clicked'], + ['contact.created', 'unknown'], + ['email.no_such_event', 'unknown'], + ]; + const verifier = createResendWebhookVerifier({ + secret: SECRET, + now: () => NOW, + replayStore: null, + }); + const actual = await eachOf(rows, (input, index) => { + const { type, ...data } = fieldsOf(input, 'type'); + const body = JSON.stringify({ + type, + created_at: '2027-01-15T08:00:00.000Z', + data: { email_id: 'e-1', to: ['bob@example.com'], ...data }, + }); + return verifier.verify(signSvix(body, `msg_${index}`)); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: SendGrid, a bounce hard, soft when blocked or expired', async () => { + const bounce = (type) => ({ event: 'bounce', type }); + const rows = [ + ['processed', 'accepted'], + ['delivered', 'delivered'], + ['deferred', 'deferred'], + [bounce('bounce'), 'bounced', 'hard'], + [bounce('blocked'), 'bounced', 'soft'], + // Expired: given up after its retries. + [bounce('expired'), 'bounced', 'soft'], + ['bounce', 'bounced', 'undetermined'], + ['dropped', 'rejected'], + ['spamreport', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['group_unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], + ['group_resubscribe', 'unknown'], + ['no_such_event', 'unknown'], + ]; + const { publicKey, privateKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); + const verifier = createSendGridWebhookVerifier({ + publicKey: publicKey.export({ format: 'der', type: 'spki' }).toString('base64'), + now: () => NOW, + replayStore: null, + }); + const body = JSON.stringify( + rows.map(([input], index) => ({ + email: 'bob@example.com', + timestamp: 1_799_999_990, + sg_event_id: `sg-${index}`, + sg_message_id: 'x-msg-1.filter-1', + ...fieldsOf(input, 'event'), + })), + ); + const timestamp = '1800000000'; + const { events } = await verifier.verify({ + body, + headers: { + 'X-Twilio-Email-Event-Webhook-Signature': sign( + 'sha256', + Buffer.from(timestamp + body), + privateKey, + ).toString('base64'), + 'X-Twilio-Email-Event-Webhook-Timestamp': timestamp, + }, + }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); + +test('Webhooks mapping: Scaleway, by type, a drop hard and a deferral soft', async () => { + const rows = [ + ['email_queued', 'accepted'], + ['email_delivered', 'delivered'], + ['email_deferred', 'deferred', 'soft'], + ['email_dropped', 'bounced', 'hard'], + ['email_mailbox_not_found', 'bounced', 'hard'], + ['email_blocklisted', 'rejected'], + ['email_spam', 'complained'], + ['email_no_such_event', 'unknown'], + ]; + const topic = 'arn:scw:sns:fr-par:project-1:tem-events'; + const verifier = createScalewayWebhookVerifier({ + topicArns: [topic], + trustChain: scaleway.trustChain, + fetch: certificateFetch(scaleway.certificate).fetch, + now: () => SNS_NOW, + replayStore: null, + }); + const actual = await eachOf(rows, (type, index) => { + const message = signSns( + { + Type: 'Notification', + MessageId: `scw-${index}`, + TopicArn: topic, + Message: JSON.stringify({ + id: `scw-event-${index}`, + type, + created_at: '2027-01-01T11:59:00Z', + email_id: 'em-1', + email_to: 'bob@example.com', + }), + Timestamp: '2027-01-01T12:00:00.000Z', + SigningCertURL: 'https://messaging.s3.fr-par.scw.cloud/fr-par/sns/cert-1.pem', + }, + '2', + scaleway.privateKey, + ); + return verifier.verify({ body: JSON.stringify(message) }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: SES, by event type, a bounce by its bounceType', async () => { + const BOB = [{ emailAddress: 'bob@example.com' }]; + const bounce = (fields) => ({ + eventType: 'Bounce', + bounce: { bouncedRecipients: BOB, ...fields }, + }); + const subscription = (unsubscribeAll) => ({ + eventType: 'Subscription', + subscription: { newTopicPreferences: { unsubscribeAll } }, + }); + const rows = [ + [{ eventType: 'Send' }, 'accepted'], + [{ eventType: 'Delivery', delivery: { recipients: ['bob@example.com'] } }, 'delivered'], + [bounce({ bounceType: 'Permanent' }), 'bounced', 'hard'], + [bounce({ bounceType: 'Transient' }), 'bounced', 'soft'], + [bounce({ bounceType: 'Undetermined' }), 'bounced', 'undetermined'], + [bounce({}), 'bounced', 'undetermined'], + // A bounce that names no recipient gives one event, of its kind. + [{ eventType: 'Bounce', bounce: { bounceType: 'Permanent' } }, 'bounced', 'hard'], + [{ eventType: 'Complaint', complaint: { complainedRecipients: BOB } }, 'complained'], + [{ eventType: 'DeliveryDelay', deliveryDelay: { delayedRecipients: BOB } }, 'deferred'], + [{ eventType: 'Reject', reject: { reason: 'Bad content' } }, 'rejected'], + [{ eventType: 'Rendering Failure', failure: { errorMessage: 'No name' } }, 'rejected'], + // SES writes it with a space; the name without is not one of its types. + [{ eventType: 'RenderingFailure' }, 'unknown'], + [{ eventType: 'Open' }, 'opened'], + [{ eventType: 'Click', click: { link: 'https://example.com/a' } }, 'clicked'], + [subscription(true), 'unsubscribed'], + [subscription(false), 'unknown'], + // The notifications of an identity, rather than of a configuration set. + [ + { notificationType: 'Bounce', bounce: { bounceType: 'Permanent', bouncedRecipients: BOB } }, + 'bounced', + 'hard', + ], + [{ notificationType: 'Complaint', complaint: { complainedRecipients: BOB } }, 'complained'], + [{ notificationType: 'Delivery', delivery: { recipients: ['bob@example.com'] } }, 'delivered'], + [{ eventType: 'NoSuchEvent' }, 'unknown'], + ]; + const verifier = createSesWebhookVerifier({ + topicArns: [TOPIC], + fetch: certificateFetch().fetch, + now: () => SNS_NOW, + replayStore: null, + }); + const actual = await eachOf(rows, (input, index) => { + const record = { + mail: { + timestamp: '2027-01-01T11:59:00.000Z', + messageId: 'ses-message-1', + destination: ['bob@example.com'], + }, + ...input, + }; + const message = notification(record, { MessageId: `sns-${index}` }); + return verifier.verify({ body: JSON.stringify(message), headers: {} }); + }); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: SMTP2GO, a bounce by its bounce field', async () => { + const bounce = (kind) => ({ event: 'bounce', bounce: kind }); + const rows = [ + ['processed', 'accepted'], + ['delivered', 'delivered'], + [bounce('hard'), 'bounced', 'hard'], + [bounce('soft'), 'bounced', 'soft'], + [bounce('other'), 'bounced', 'undetermined'], + ['bounce', 'bounced', 'undetermined'], + ['reject', 'rejected'], + ['spam', 'complained'], + ['unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['click', 'clicked'], + ['resubscribe', 'unknown'], + ['no_such_event', 'unknown'], + ]; + const verifier = createSmtp2goWebhookVerifier({ username: 'hook', password: 'pw' }); + const actual = await eachOf(rows, (input) => + verifier.verify({ + body: JSON.stringify({ + email_id: '1abc-2def', + rcpt: 'bob@example.com', + time: '2027-01-15T08:00:00Z', + ...fieldsOf(input, 'event'), + }), + headers: { authorization: basic('hook', 'pw') }, + }), + ); + assert.deepStrictEqual(actual, expectedOf(rows)); +}); + +test('Webhooks mapping: SparkPost, a bounce by its bounce class', async () => { + const classed = (type, bounceClass) => ({ type, bounce_class: bounceClass }); + const rows = [ + ['injection', 'accepted'], + ['delivery', 'delivered'], + ['delay', 'deferred'], + // A delay carries no bounce, whatever its class. + [classed('delay', '20'), 'deferred'], + [classed('bounce', '10'), 'bounced', 'hard'], + [classed('bounce', 30), 'bounced', 'hard'], + [classed('bounce', '90'), 'bounced', 'hard'], + [classed('bounce', '20'), 'bounced', 'soft'], + [classed('bounce', '25'), 'bounced', 'soft'], + [classed('bounce', '50'), 'bounced', 'soft'], + [classed('bounce', '100'), 'bounced', 'soft'], + [classed('bounce', '1'), 'bounced', 'undetermined'], + ['bounce', 'bounced', 'undetermined'], + [classed('out_of_band', '10'), 'bounced', 'hard'], + [classed('out_of_band', '20'), 'bounced', 'soft'], + ['out_of_band', 'bounced', 'undetermined'], + ['spam_complaint', 'complained'], + ['policy_rejection', 'rejected'], + ['generation_failure', 'rejected'], + ['generation_rejection', 'rejected'], + ['list_unsubscribe', 'unsubscribed'], + ['link_unsubscribe', 'unsubscribed'], + ['open', 'opened'], + ['initial_open', 'opened'], + ['amp_open', 'opened'], + ['amp_initial_open', 'opened'], + ['click', 'clicked'], + ['amp_click', 'clicked'], + ['no_such_event', 'unknown'], + ]; + const verifier = createSparkPostWebhookVerifier({ username: 'sp', password: 'pw' }); + const body = JSON.stringify( + rows.map(([input], index) => ({ + msys: { + message_event: { + event_id: `sp-${index}`, + transmission_id: 't-1', + rcpt_to: 'bob@example.com', + ...fieldsOf(input, 'type'), + }, + }, + })), + ); + const { events } = await verifier.verify({ body, headers: { Authorization: basic('sp', 'pw') } }); + assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); +}); diff --git a/tests/unit/webhooks-others.test.js b/tests/unit/webhooks-others.test.js index d2e0fdf..9837cae 100644 --- a/tests/unit/webhooks-others.test.js +++ b/tests/unit/webhooks-others.test.js @@ -500,11 +500,28 @@ test('Webhooks Azure: Event Grid events with the token, and the validation hande eventType: 'Microsoft.Communication.EmailEngagementTrackingReportReceived', data: { engagementType: 'view' }, }); + // Before: an engagement of any type but a click was an open. + many.push({ + id: 's-forward', + eventType: 'Microsoft.Communication.EmailEngagementTrackingReportReceived', + data: { engagementType: 'forward' }, + }); many.push({ id: 's-other', eventType: 'Microsoft.Communication.Other' }); const typed = await verifier.verify({ body: JSON.stringify(many), headers }); assert.deepStrictEqual( typed.events.map((event) => event.type), - ['delivered', 'bounced', 'bounced', 'rejected', 'deferred', 'unknown', 'opened', 'unknown'], + [ + 'delivered', + 'bounced', + 'bounced', + 'rejected', + // Quarantined: held by the recipient's system, not tried again. + 'unknown', + 'unknown', + 'opened', + 'unknown', + 'unknown', + ], ); await rejects(verifier.verify({ body: '{}', headers }), 'payload'); const custom = createAzureWebhookVerifier({ diff --git a/tests/unit/webhooks-sendgrid.test.js b/tests/unit/webhooks-sendgrid.test.js index 6d55e42..fe4e0a0 100644 --- a/tests/unit/webhooks-sendgrid.test.js +++ b/tests/unit/webhooks-sendgrid.test.js @@ -53,6 +53,7 @@ const rejects = async (promise, reason) => { test('Webhooks SendGrid: a signed batch is an event per record', async () => { const records = [ record('bounce', { + type: 'bounce', status: '5.1.1', reason: '550 5.1.1 unknown user', bounce_classification: 'Invalid Address', From 3c896d72593088f60c28b2ff3a49b68004df7cb6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:40:34 +0300 Subject: [PATCH 096/143] test(sendmail): a send that waits for a place leaves the queue when aborted The wait for a place among maxConcurrent processes had no test of an abort while waiting, nor of a signal already aborted when no place is free, and function coverage fell under its gate. Co-Authored-By: Claude Opus 5.5 --- tests/unit/sendmail.test.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/sendmail.test.js b/tests/unit/sendmail.test.js index df88afa..8b6653b 100644 --- a/tests/unit/sendmail.test.js +++ b/tests/unit/sendmail.test.js @@ -483,3 +483,32 @@ test('Sendmail: no more than maxConcurrent processes run at once', async () => { await assert.rejects(queued, /gone/); await Promise.all(held); }); + +test('Sendmail: a send that waits for a place leaves the queue when it is aborted', async () => { + const children = []; + const spawn = () => { + const child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = () => true; + child.stdin.resume(); + children.push(child); + return child; + }; + const transport = new SendmailTransport({ spawn, maxConcurrent: 1 }); + const held = transport.send(message()); + while (children.length === 0) await new Promise((resolve) => setImmediate(resolve)); + const controller = new AbortController(); + const queued = transport.send(message(), { signal: controller.signal }); + for (let i = 0; i < 5; i++) await new Promise((resolve) => setImmediate(resolve)); + controller.abort(new Error('gone')); + await assert.rejects(queued, /gone/); + // A signal aborted by the time no place is free is refused at once. + const envelope = { from: 'a@example.com', to: ['b@example.com'] }; + const late = AbortSignal.abort(new Error('late')); + await assert.rejects(transport.deliver(envelope, [], 'id', late), /late/); + children[0].stdin.on('finish', () => children[0].emit('close', 0, null)); + if (children[0].stdin.writableFinished) children[0].emit('close', 0, null); + assert.strictEqual((await held).successful, true); + assert.strictEqual(children.length, 1); +}); From c2fa30296fb55999a562246d0bb4c67fdaa6b96f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:57:11 +0300 Subject: [PATCH 097/143] refactor(resend): the client on createApiClient, ResendApiError over its error The client of the HTTP providers moves to internal/api-client.js, which api.js keeps exporting for the providers of 1.1, and takes the hooks a provider with errors of its own needs: read, reject, timeoutError, finalError, acceptedError and isAnswer; an error answer whose body cannot be read keeps its status. Resend's client is its request, authorization and messages over it, ResendApiError extends ProviderApiError, and the transport's receipts of failure and its abort check are the shared ones. The receipts are those of before. The shared client is larger than Resend's own was: ./resend goes from 6.5 to 7.5 KB, and the hooks take ./mailjet to 8.5, ./zeptomail to 7.5, ./smtp2go to 8 and ./azure to 9. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 10 +- src/internal/api-client.js | 265 +++++++++++++++++++++++++++++++++ src/internal/api.js | 208 ++------------------------ src/resend/http-client.js | 215 +++++++++----------------- src/resend/resend-transport.js | 41 +---- 5 files changed, 357 insertions(+), 382 deletions(-) create mode 100644 src/internal/api-client.js diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 56dbedc..6e4a64f 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -8,7 +8,7 @@ "./sendmail": 11.5, "./jmap": 13, "./ses": 8, - "./resend": 6.5, + "./resend": 7.5, "./sendgrid": 7, "./mailgun": 6.5, "./mailtrap": 7, @@ -17,12 +17,12 @@ "./plunk": 6, "./postmark": 7.5, "./brevo": 7.5, - "./mailjet": 8, + "./mailjet": 8.5, "./mailersend": 7.5, "./sparkpost": 8, - "./zeptomail": 7, - "./smtp2go": 7.5, - "./azure": 8.5, + "./zeptomail": 7.5, + "./smtp2go": 8, + "./azure": 9, "./scaleway": 7.5, "./cloudflare": 8, "./pool": 3.5, diff --git a/src/internal/api-client.js b/src/internal/api-client.js new file mode 100644 index 0000000..570078e --- /dev/null +++ b/src/internal/api-client.js @@ -0,0 +1,265 @@ +// Not ported: the client of a provider's API that every HTTP provider uses, +// those of 1.1 with its defaults, those ported from upyo with their own errors +// and messages through its hooks: the error of their API, an authenticated +// request through `requestWithRetry`, and the receipt of a failure. + +const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); +const { readText } = require('./body.js'); +const { isDeliveryUnknown } = require('./delivery-unknown.js'); +const { requestWithRetry } = require('./http.js'); +const { certificateFailure, refusalFailure } = require('./refusal.js'); +const { createLog } = require('./log.js'); + +const MAX_ERROR_TEXT = 500; + +/** + * An error of a provider's API: its status, its own code and the category the + * provider gives it, the wait it asks for, and the attempts made. The public + * errors of the providers ported from upyo (`SendGridApiError`…) extend it. + * + * @internal + */ +class ProviderApiError extends Error { + constructor(provider, message, details = {}) { + super(message, details.cause === undefined ? undefined : { cause: details.cause }); + this.name = 'ProviderApiError'; + this.provider = provider; + this.statusCode = details.statusCode; + this.retryAfterMilliseconds = details.retryAfterMilliseconds; + this.attempts = details.attempts; + this.category = details.category; + this.retryable = details.retryable; + this.providerDetails = details.providerDetails; + // The code of the receipt, apart from `code`, which classifyReceiptError + // reads as a system error code. + this.receiptCode = details.code; + } +} + +const parseJson = (text) => { + try { + return JSON.parse(text); + } catch { + return undefined; + } +}; + +const shorten = (text) => + text.length > MAX_ERROR_TEXT ? `${text.slice(0, MAX_ERROR_TEXT)}…` : text; + +// The message an API gives for an error, in the fields most of them use. +const defaultMessageOf = (json) => { + if (json == null || typeof json !== 'object') return undefined; + if (typeof json.message === 'string') return json.message; + if (typeof json.Message === 'string') return json.Message; + if (typeof json.error === 'string') return json.error; + if (typeof json.error?.message === 'string') return json.error.message; + const first = Array.isArray(json.errors) ? json.errors[0] : undefined; + if (typeof first === 'string') return first; + if (typeof first?.message === 'string') return first.message; + return undefined; +}; + +// The answer of a 2xx: its JSON, an empty body as null. +const defaultRead = (provider) => (response, text) => { + const json = text.trim() === '' ? null : parseJson(text); + if (json === undefined) { + throw new ProviderApiError(provider.id, `${provider.label} answered with no JSON.`, { + statusCode: response.status, + }); + } + return { status: response.status, headers: response.headers, json }; +}; + +const defaultTimeoutError = (config, provider) => () => + new ProviderApiError( + provider.id, + `${provider.label} API request timed out after ${config.timeout} ms.`, + ); + +const defaultFinalError = (provider) => (error, attempts) => + error instanceof ProviderApiError + ? error + : new ProviderApiError(provider.id, `${provider.label}: ${error.message}`, { + attempts, + cause: error, + }); + +// The text of an error answer, or undefined when it cannot be read: the +// error then keeps the status, which says more than the failure to read. +const readErrorText = async (response) => { + try { + return await readText(response); + } catch { + return undefined; + } +}; + +/** + * Makes the client of a provider's API. + * + * @param config The resolved configuration: `fetch`, `headers`, `retries`, + * `timeout` and `logger`. + * @param provider `id`, the provider's id; `label`, its name in messages; + * `authorize(headers, request)`, which adds its credentials; optional + * `messageOf(json, text)`, the message of an error answer, and + * `classify(status, json)`, `{ category, retryable, code }` of one. A + * provider with errors of its own replaces any of: `read(response, text, + * attempt)`, the result of a 2xx; `reject(response, text, attempt)`, the + * error of another answer, `text` undefined when it could not be read; + * `timeoutError()`; `finalError(error, attempts)`; `acceptedError(error, + * attempts)`, a failure after a 2xx; and `isAnswer(error)`, a refusal in a + * 2xx (see `requestWithRetry`). + * @returns `{ log, fetchWithAuth, request }`. + * @internal + */ +const createApiClient = (config, provider) => { + const log = createLog(config.logger, { transport: provider.id }); + + const fetchWithAuth = async (url, options, signal) => { + const headers = new Headers(options.headers); + await provider.authorize(headers, { url, method: options.method ?? 'GET', body: options.body }); + for (const name in config.headers) { + if (Object.hasOwn(config.headers, name)) headers.set(name, config.headers[name]); + } + // A redirect comes back as a response rather than being followed, so the + // credentials in the headers never reach another origin. + const fetch = config.fetch ?? globalThis.fetch; + return fetch(url, { ...options, headers, signal, redirect: 'manual' }); + }; + + const rejectionOf = (response, read, attempt) => { + const text = read ?? ''; + const json = parseJson(text); + const message = + (provider.messageOf ?? defaultMessageOf)(json, text) ?? + (text.trim() === '' ? `HTTP ${response.status}` : shorten(text.trim())); + const classified = provider.classify?.(response.status, json) ?? {}; + return new ProviderApiError(provider.id, `${provider.label}: ${message}`, { + statusCode: response.status, + retryAfterMilliseconds: parseRetryAfter(response.headers.get('Retry-After')), + attempts: attempt + 1, + category: classified.category, + retryable: classified.retryable, + code: classified.code, + providerDetails: json ?? undefined, + }); + }; + + const timeoutError = provider.timeoutError ?? defaultTimeoutError(config, provider); + const finalError = provider.finalError ?? defaultFinalError(provider); + const read = provider.read ?? defaultRead(provider); + const reject = provider.reject ?? rejectionOf; + + /** + * Sends one request and reads its answer: `{ status, headers, json }` of a + * 2xx, an empty body as `null`, or what the provider's `read` makes of it. + * A 2xx whose body is not JSON fails as of unknown delivery, as the + * provider may have taken the request. A body of FormData goes as it is, + * any other as JSON, serialized once for every attempt. + */ + const request = (options) => { + const isForm = options.body instanceof FormData; + const headers = { Accept: 'application/json', ...options.headers }; + let body; + if (isForm) body = options.body; + else if (options.body !== undefined) { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify(options.body); + } + return requestWithRetry({ + log, + signal: options.signal, + onRetry: options.onRetry, + retries: config.retries, + timeoutMs: config.timeout, + hasIdempotencyKey: options.hasIdempotencyKey === true, + isIdempotent: options.isIdempotent === true, + send: (signal) => + fetchWithAuth(options.url, { method: options.method ?? 'POST', headers, body }, signal), + read: async (response, attempt) => read(response, await readText(response), attempt), + reject: async (response, attempt) => reject(response, await readErrorText(response), attempt), + createTimeoutError: timeoutError, + finalError, + wrapAccepted: provider.acceptedError, + isAnswer: provider.isAnswer, + }); + }; + + return { log, fetchWithAuth, request }; +}; + +/** + * The receipt of a failure, with what the error says of it. + * + * @param provider The provider's id. + * @param error What was thrown. + * @returns A failed receipt. + * @internal + */ +const providerFailure = (provider, error) => { + const message = error instanceof Error ? error.message : String(error); + const certificate = certificateFailure(provider, message, error); + if (certificate !== undefined) return certificate; + if (error instanceof ProviderApiError) { + return createFailedReceipt(message, { + provider, + deliveryUnknown: isDeliveryUnknown(error), + statusCode: error.statusCode, + retryAfterMilliseconds: error.retryAfterMilliseconds, + attempts: error.attempts, + category: error.category, + retryable: error.retryable, + code: error.receiptCode, + providerDetails: error.providerDetails, + }); + } + return ( + refusalFailure(provider, error) ?? + createFailedReceipt(message, { + provider, + deliveryUnknown: isDeliveryUnknown(error), + attempts: typeof error?.attempts === 'number' ? error.attempts : undefined, + }) + ); +}; + +/** + * Parses the JSON of a 2xx, or throws what `invalid(error)` makes of a body + * that is not JSON. + * + * @internal + */ +const parseAnswer = (text, invalid) => { + try { + return JSON.parse(text); + } catch (error) { + throw invalid(error); + } +}; + +/** + * Cuts the text of an error answer to 500 characters, as the providers + * ported from upyo do, with `...` after it. + * + * @internal + */ +const truncateErrorText = (text) => + text.length > MAX_ERROR_TEXT ? `${text.slice(0, MAX_ERROR_TEXT)}...` : text; + +/** + * Whether an error is the abort of the caller's own signal. + * + * @internal + */ +const isCallerAbort = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +module.exports = { + ProviderApiError, + createApiClient, + isCallerAbort, + parseAnswer, + providerFailure, + truncateErrorText, +}; diff --git a/src/internal/api.js b/src/internal/api.js index 9bab791..65cc365 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -1,203 +1,21 @@ // Not ported: what the HTTP providers added in 1.1 share, once rather than a -// copy in each: the error of their API, an authenticated JSON request through -// `requestWithRetry`, the receipt of a failure, and the conversion of a batch -// in which a message that cannot be converted fails alone. +// copy in each: the client of api-client.js, and the conversion of a batch in +// which a message that cannot be converted fails alone, its attachments and +// header fields, and the steps of one send. const { readAttachmentContent } = require('../core/attachment.js'); -const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); +const { createFailedReceipt } = require('../core/receipt.js'); const { toBase64 } = require('./base64.js'); -const { readText } = require('./body.js'); const { canonicalHeaderName } = require('./headers.js'); const { convertAttachments, visitCustomHeaders, withCalendarAttachment } = require('./provider.js'); -const { isDeliveryUnknown } = require('./delivery-unknown.js'); -const { requestWithRetry } = require('./http.js'); -const { certificateFailure, refusalFailure } = require('./refusal.js'); -const { createLog } = require('./log.js'); - -const MAX_ERROR_TEXT = 500; - -/** - * An error of a provider's API: its status, its own code and the category the - * provider gives it, the wait it asks for, and the attempts made. - * - * @internal - */ -class ProviderApiError extends Error { - constructor(provider, message, details = {}) { - super(message, details.cause === undefined ? undefined : { cause: details.cause }); - this.name = 'ProviderApiError'; - this.provider = provider; - this.statusCode = details.statusCode; - this.retryAfterMilliseconds = details.retryAfterMilliseconds; - this.attempts = details.attempts; - this.category = details.category; - this.retryable = details.retryable; - this.providerDetails = details.providerDetails; - // The code of the receipt, apart from `code`, which classifyReceiptError - // reads as a system error code. - this.receiptCode = details.code; - } -} - -const parseJson = (text) => { - try { - return JSON.parse(text); - } catch { - return undefined; - } -}; - -const shorten = (text) => - text.length > MAX_ERROR_TEXT ? `${text.slice(0, MAX_ERROR_TEXT)}…` : text; - -// The message an API gives for an error, in the fields most of them use. -const defaultMessageOf = (json) => { - if (json == null || typeof json !== 'object') return undefined; - if (typeof json.message === 'string') return json.message; - if (typeof json.Message === 'string') return json.Message; - if (typeof json.error === 'string') return json.error; - if (typeof json.error?.message === 'string') return json.error.message; - const first = Array.isArray(json.errors) ? json.errors[0] : undefined; - if (typeof first === 'string') return first; - if (typeof first?.message === 'string') return first.message; - return undefined; -}; - -/** - * Makes the client of a provider's API. - * - * @param config The resolved configuration: `fetch`, `headers`, `retries`, - * `timeout` and `logger`. - * @param provider `id`, the provider's id; `label`, its name in messages; - * `authorize(headers, request)`, which adds its credentials; optional - * `messageOf(json, text)`, the message of an error answer, and - * `classify(status, json)`, `{ category, retryable, code }` of one. - * @returns `{ log, fetchWithAuth, request }`. - * @internal - */ -const createApiClient = (config, provider) => { - const log = createLog(config.logger, { transport: provider.id }); - - const fetchWithAuth = async (url, options, signal) => { - const headers = new Headers(options.headers); - await provider.authorize(headers, { url, method: options.method ?? 'GET', body: options.body }); - for (const name in config.headers) { - if (Object.hasOwn(config.headers, name)) headers.set(name, config.headers[name]); - } - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - const fetch = config.fetch ?? globalThis.fetch; - return fetch(url, { ...options, headers, signal, redirect: 'manual' }); - }; - - const rejectionOf = (response, text, attempt) => { - const json = parseJson(text); - const message = - (provider.messageOf ?? defaultMessageOf)(json, text) ?? - (text.trim() === '' ? `HTTP ${response.status}` : shorten(text.trim())); - const classified = provider.classify?.(response.status, json) ?? {}; - return new ProviderApiError(provider.id, `${provider.label}: ${message}`, { - statusCode: response.status, - retryAfterMilliseconds: parseRetryAfter(response.headers.get('Retry-After')), - attempts: attempt + 1, - category: classified.category, - retryable: classified.retryable, - code: classified.code, - providerDetails: json ?? undefined, - }); - }; - - /** - * Sends one request and reads its answer: `{ status, headers, json }` of a - * 2xx, an empty body as `null`. A 2xx whose body is not JSON fails as of - * unknown delivery, as the provider may have taken the request. - */ - const request = (options) => { - const isForm = options.body instanceof FormData; - const headers = { Accept: 'application/json', ...options.headers }; - let body; - if (isForm) body = options.body; - else if (options.body !== undefined) { - headers['Content-Type'] = 'application/json'; - body = JSON.stringify(options.body); - } - return requestWithRetry({ - log, - signal: options.signal, - onRetry: options.onRetry, - retries: config.retries, - timeoutMs: config.timeout, - hasIdempotencyKey: options.hasIdempotencyKey === true, - isIdempotent: options.isIdempotent === true, - send: (signal) => - fetchWithAuth(options.url, { method: options.method ?? 'POST', headers, body }, signal), - read: async (response) => { - const text = await readText(response); - const json = text.trim() === '' ? null : parseJson(text); - if (json === undefined) { - throw new ProviderApiError(provider.id, `${provider.label} answered with no JSON.`, { - statusCode: response.status, - }); - } - return { status: response.status, headers: response.headers, json }; - }, - reject: async (response, attempt) => rejectionOf(response, await readText(response), attempt), - createTimeoutError: () => - new ProviderApiError( - provider.id, - `${provider.label} API request timed out after ${config.timeout} ms.`, - ), - finalError: (error, attempts) => - error instanceof ProviderApiError - ? error - : new ProviderApiError(provider.id, `${provider.label}: ${error.message}`, { - attempts, - cause: error, - }), - }); - }; - - return { log, fetchWithAuth, request }; -}; - -/** - * The receipt of a failure, with what the error says of it. - * - * @param provider The provider's id. - * @param error What was thrown. - * @returns A failed receipt. - * @internal - */ -const providerFailure = (provider, error) => { - const message = error instanceof Error ? error.message : String(error); - const certificate = certificateFailure(provider, message, error); - if (certificate !== undefined) return certificate; - if (error instanceof ProviderApiError) { - return createFailedReceipt(message, { - provider, - deliveryUnknown: isDeliveryUnknown(error), - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - category: error.category, - retryable: error.retryable, - code: error.receiptCode, - providerDetails: error.providerDetails, - }); - } - return ( - refusalFailure(provider, error) ?? - createFailedReceipt(message, { provider, deliveryUnknown: isDeliveryUnknown(error) }) - ); -}; - -/** - * Whether an error is the abort of the caller's own signal. - * - * @internal - */ -const isCallerAbort = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); +const { + ProviderApiError, + createApiClient, + isCallerAbort, + parseAnswer, + providerFailure, + truncateErrorText, +} = require('./api-client.js'); /** * Converts each message of a batch, one that fails to convert failing alone. @@ -363,7 +181,9 @@ module.exports = { headerFieldsOf, isCallerAbort, mergeBatch, + parseAnswer, providerFailure, sendConverted, sentWith, + truncateErrorText, }; diff --git a/src/resend/http-client.js b/src/resend/http-client.js index 4cbdadb..7a4f95f 100644 --- a/src/resend/http-client.js +++ b/src/resend/http-client.js @@ -1,30 +1,21 @@ // Ported from upyo v0.7.0 (packages/resend/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and ResendApiError +// extends its error; the messages and the errors of upstream stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); +const { + ProviderApiError, + createApiClient, + parseAnswer, + truncateErrorText, +} = require('../internal/api-client.js'); /** * Error thrown when a Resend API request fails. */ -class ResendApiError extends Error { - /** - * HTTP status code returned by Resend, if the request reached the API. - */ - statusCode; - - /** - * Retry delay from Resend's `Retry-After` response header. - */ - retryAfterMilliseconds; - - /** - * Number of attempts made before this error was produced. - */ - attempts; - +class ResendApiError extends ProviderApiError { /** * Creates a Resend API error. * @@ -35,14 +26,42 @@ class ResendApiError extends Error { * @param options The cause of the error, such as the failure of `fetch`. */ constructor(message, statusCode, retryAfterMilliseconds, attempts, options) { - super(message, options); + super('resend', message, { + statusCode, + retryAfterMilliseconds, + attempts, + cause: options?.cause, + }); this.name = 'ResendApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } +const messageOf = (text) => { + try { + return JSON.parse(text).message; + } catch { + // Ignore if JSON parsing fails, as the body may be non-JSON + return undefined; + } +}; + +const rejectionOf = (response, read, attempt) => { + const text = read ?? ''; + const parsed = messageOf(text); + const parsedErrorMessage = parsed === '' ? undefined : parsed; + const responseMessage = truncateErrorText(text); + const fallbackMessage = responseMessage === '' ? undefined : responseMessage; + return new ResendApiError( + parsedErrorMessage ?? fallbackMessage ?? `HTTP ${response.status}`, + response.status, + parseRetryAfter(response.headers.get('Retry-After')), + attempt + 1, + ); +}; + +const invalidJson = (error) => + new Error(`Invalid JSON response from Resend API: ${error.message}`, { cause: error }); + /** * HTTP client wrapper for Resend API requests. * @@ -51,10 +70,25 @@ class ResendApiError extends Error { */ class ResendHttpClient { config; + #request; constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'resend' }); + const client = createApiClient(config, { + id: 'resend', + label: 'Resend', + authorize: (headers) => headers.set('Authorization', `Bearer ${config.apiKey}`), + read: (_response, text) => parseAnswer(text, invalidJson), + reject: rejectionOf, + timeoutError: () => new Error(`Resend API request timed out after ${config.timeout} ms.`), + finalError: (error, attempts) => + error instanceof ResendApiError + ? error + : new ResendApiError(error.message, undefined, undefined, attempts, { cause: error }), + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** @@ -67,25 +101,7 @@ class ResendHttpClient { * @returns Promise that resolves to the Resend response. */ sendMessage(messageData, signal, idempotencyKey, onRetry) { - const url = `${this.config.baseUrl}/emails`; - const headers = { - 'Content-Type': 'application/json', - }; - - if (idempotencyKey) { - headers['Idempotency-Key'] = idempotencyKey; - } - - return this.makeRequest( - url, - { - method: 'POST', - headers, - body: JSON.stringify(messageData), - signal, - }, - onRetry, - ); + return this.#send('/emails', messageData, signal, idempotencyKey, onRetry); } /** @@ -98,114 +114,21 @@ class ResendHttpClient { * @returns Promise that resolves to the Resend batch response. */ sendBatch(messagesData, signal, idempotencyKey, onRetry) { - const url = `${this.config.baseUrl}/emails/batch`; - const headers = { - 'Content-Type': 'application/json', - }; - - if (idempotencyKey) { - headers['Idempotency-Key'] = idempotencyKey; - } - - return this.makeRequest( - url, - { - method: 'POST', - headers, - body: JSON.stringify(messagesData), - signal, - }, - onRetry, - ); + return this.#send('/emails/batch', messagesData, signal, idempotencyKey, onRetry); } - /** - * Makes an HTTP request to Resend API with retry logic. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @returns Promise that resolves to the parsed response. - */ - makeRequest(url, options, onRetry) { - return requestWithRetry({ - log: this.log, - signal: options.signal, + #send(path, body, signal, idempotencyKey, onRetry) { + // Resend answers a repeated Idempotency-Key with its first answer. + const hasKey = typeof idempotencyKey === 'string' && idempotencyKey !== ''; + return this.#request({ + url: `${this.config.baseUrl}${path}`, + body, + signal, onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - // Resend answers a repeated Idempotency-Key with its first answer. - hasIdempotencyKey: options.headers?.['Idempotency-Key'] != null, - send: (signal) => this.fetchWithAuth(url, options, signal), - read: async (response) => { - const text = await readText(response); - try { - return JSON.parse(text); - } catch (error) { - throw new Error( - `Invalid JSON response from Resend API: ${ - error instanceof Error ? error.message : String(error) - }`, - { cause: error }, - ); - } - }, - reject: async (response, attempt) => { - const text = await readText(response); - let errorMessage; - try { - errorMessage = JSON.parse(text).message; - } catch { - // Ignore if JSON parsing fails, as the body may be non-JSON - } - - const parsedErrorMessage = errorMessage === '' ? undefined : errorMessage; - const responseMessage = truncateErrorBody(text); - const fallbackMessage = responseMessage === '' ? undefined : responseMessage; - - return new ResendApiError( - parsedErrorMessage ?? fallbackMessage ?? `HTTP ${response.status}`, - response.status, - parseRetryAfter(response.headers.get('Retry-After')), - attempt + 1, - ); - }, - createTimeoutError: () => - new Error(`Resend API request timed out after ${this.config.timeout} ms.`), - finalError: (error, attempts) => - error instanceof ResendApiError - ? error - : new ResendApiError(error.message, undefined, undefined, attempts, { cause: error }), + headers: hasKey ? { 'Idempotency-Key': idempotencyKey } : undefined, + hasIdempotencyKey: hasKey, }); } - - /** - * Makes a fetch request with authentication headers. - * - * @param url The URL to fetch. - * @param options Fetch options. - * @param signal The signal of the attempt, with its deadline. - * @returns Promise that resolves to the Response. - */ - fetchWithAuth(url, options, signal) { - const headers = new Headers(options.headers); - - // Add authentication - headers.set('Authorization', `Bearer ${this.config.apiKey}`); - - // Add custom headers from config - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - const fetch = this.config.fetch ?? globalThis.fetch; - return fetch(url, { ...options, headers, signal, redirect: 'manual' }); - } } -const truncateErrorBody = (text) => { - return text.length > 500 ? `${text.slice(0, 500)}...` : text; -}; - module.exports = { ResendApiError, ResendHttpClient }; diff --git a/src/resend/resend-transport.js b/src/resend/resend-transport.js index ceb408c..6d234b0 100644 --- a/src/resend/resend-transport.js +++ b/src/resend/resend-transport.js @@ -1,19 +1,17 @@ // Ported from upyo v0.7.0 (packages/resend/src/resend-transport.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -const { createFailedReceipt } = require('../core/receipt.js'); const { createResendConfig } = require('./config.js'); const { verifyByRequest } = require('../internal/verify.js'); -const { ResendApiError, ResendHttpClient } = require('./http-client.js'); +const { ResendHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage, convertMessagesBatch, generateIdempotencyKey, } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Resend transport implementation for sending emails via Resend API. @@ -166,9 +164,7 @@ class ResendTransport { throw error; } - const errorMessage = error instanceof Error ? error.message : String(error); - - return createResendFailure(errorMessage, error); + return providerFailure('resend', error); } } @@ -302,11 +298,9 @@ class ResendTransport { throw error; } - const errorMessage = error instanceof Error ? error.message : String(error); - // If batch fails, yield error receipt for each message for (let i = 0; i < messages.length; i++) { - yield createResendFailure(errorMessage, error); + yield providerFailure('resend', error); } } } @@ -342,33 +336,6 @@ class ResendTransport { } } -const createResendFailure = (message, error) => { - const known = refusalFailure('resend', error) ?? certificateFailure('resend', message, error); - if (known !== undefined) return known; - if (error instanceof ResendApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'resend', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'resend', - }); -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - // Resend answers a key that may only send with `restricted_api_key`. const isRestrictedKey = (body) => { try { From f959b1009e5967d9595409c38bdac8e6452b9e61 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:57:12 +0300 Subject: [PATCH 098/143] refactor(sendgrid): the client on createApiClient, SendGridApiError over its error SendGridApiError extends ProviderApiError, its errors the providerDetails of the receipt, and the 202 SendGrid answers is checked in the read of the shared client. The transport's receipts are the shared ones, unchanged. ./sendgrid goes from 7 to 7.5 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/sendgrid/http-client.js | 232 +++++++++++------------------ src/sendgrid/sendgrid-transport.js | 39 +---- 3 files changed, 92 insertions(+), 181 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 6e4a64f..0bdf490 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -9,7 +9,7 @@ "./jmap": 13, "./ses": 8, "./resend": 7.5, - "./sendgrid": 7, + "./sendgrid": 7.5, "./mailgun": 6.5, "./mailtrap": 7, "./maileroo": 6.5, diff --git a/src/sendgrid/http-client.js b/src/sendgrid/http-client.js index 21cb47c..b31f326 100644 --- a/src/sendgrid/http-client.js +++ b/src/sendgrid/http-client.js @@ -1,155 +1,21 @@ // Ported from upyo v0.7.0 (packages/sendgrid/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and SendGridApiError +// extends its error; the messages and the errors of upstream stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); - -/** - * HTTP client wrapper for SendGrid API requests. - * - * This class handles authentication, request formatting, error handling, - * and retry logic for SendGrid API calls. - */ -class SendGridHttpClient { - config; - - constructor(config) { - this.config = config; - this.log = createLog(config.logger, { transport: 'sendgrid' }); - } - - /** - * Sends a message via SendGrid API. - * - * @param messageData The JSON data to send to SendGrid. - * @param signal Optional AbortSignal for cancellation. - * @param onRetry Optional callback told of each retry. - * @returns Promise that resolves to the SendGrid response. - */ - sendMessage(messageData, signal, onRetry) { - const url = `${this.config.baseUrl}/mail/send`; - - return this.makeRequest( - url, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(messageData), - signal, - }, - onRetry, - ); - } - - /** - * Makes an HTTP request to SendGrid API with retry logic. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @returns Promise that resolves to the parsed response. - * @throws {DOMException} If the caller aborts the request. - * @throws {SendGridApiError} If SendGrid returns a client error or all retry - * attempts are exhausted. - */ - makeRequest(url, options, onRetry) { - return requestWithRetry({ - log: this.log, - signal: options.signal, - onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (signal) => this.fetchWithAuth(url, options, signal), - read: async (response, attempt) => { - const text = await readText(response); - // SendGrid returns 202 for successful sends - if (response.status !== 202) throw createSendGridApiError(response, text, attempt); - return { - statusCode: response.status, - body: text, - headers: this.headersToRecord(response.headers), - }; - }, - reject: async (response, attempt) => - createSendGridApiError(response, await readText(response), attempt), - createTimeoutError: () => - new Error(`SendGrid API request timed out after ${this.config.timeout} ms.`), - finalError: (error, attempts) => - error instanceof SendGridApiError - ? error - : new SendGridApiError(error.message, undefined, undefined, undefined, attempts, { - cause: error, - }), - }); - } - - /** - * Makes a fetch request with SendGrid authentication. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @param signal The signal of the attempt, with its deadline. - * @returns Promise that resolves to the fetch response. - */ - fetchWithAuth(url, options, signal) { - const headers = new Headers(options.headers); - - // Add authentication header - headers.set('Authorization', `Bearer ${this.config.apiKey}`); - - // Add custom headers from config - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - const fetch = this.config.fetch ?? globalThis.fetch; - return fetch(url, { ...options, headers, signal, redirect: 'manual' }); - } - - /** - * Converts Headers object to a plain Record. - * - * @param headers The Headers object to convert. - * @returns A plain object with header key-value pairs. - */ - headersToRecord(headers) { - const record = {}; - for (const [key, value] of headers.entries()) { - record[key] = value; - } - return record; - } -} +const { ProviderApiError, createApiClient } = require('../internal/api-client.js'); /** * Error thrown when a SendGrid API request fails. */ -class SendGridApiError extends Error { - /** - * HTTP status code returned by SendGrid, if the request reached the API. - */ - statusCode; - +class SendGridApiError extends ProviderApiError { /** * Provider-supplied SendGrid error details. */ errors; - /** - * Retry delay from SendGrid's `Retry-After` response header. - */ - retryAfterMilliseconds; - - /** - * Number of attempts made before this error was produced. - */ - attempts; - /** * Creates a SendGrid API error. * @@ -161,16 +27,20 @@ class SendGridApiError extends Error { * @param options The cause of the error, such as the failure of `fetch`. */ constructor(message, statusCode, errors, retryAfterMilliseconds, attempts, options) { - super(message, options); + super('sendgrid', message, { + statusCode, + retryAfterMilliseconds, + attempts, + cause: options?.cause, + providerDetails: errors, + }); this.name = 'SendGridApiError'; - this.statusCode = statusCode; this.errors = errors; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } -const createSendGridApiError = (response, text, attempt) => { +const createSendGridApiError = (response, read, attempt) => { + const text = read ?? ''; let errorData; try { errorData = JSON.parse(text); @@ -187,4 +57,78 @@ const createSendGridApiError = (response, text, attempt) => { ); }; +/** + * Converts Headers object to a plain Record. + * + * @param headers The Headers object to convert. + * @returns A plain object with header key-value pairs. + */ +const headersToRecord = (headers) => { + const record = {}; + for (const [key, value] of headers.entries()) { + record[key] = value; + } + return record; +}; + +/** + * HTTP client wrapper for SendGrid API requests. + * + * This class handles authentication, request formatting, error handling, + * and retry logic for SendGrid API calls. + */ +class SendGridHttpClient { + config; + #request; + + constructor(config) { + this.config = config; + const client = createApiClient(config, { + id: 'sendgrid', + label: 'SendGrid', + authorize: (headers) => headers.set('Authorization', `Bearer ${config.apiKey}`), + read: (response, text, attempt) => { + // SendGrid returns 202 for successful sends + if (response.status !== 202) throw createSendGridApiError(response, text, attempt); + return { + statusCode: response.status, + body: text, + headers: headersToRecord(response.headers), + }; + }, + reject: createSendGridApiError, + timeoutError: () => new Error(`SendGrid API request timed out after ${config.timeout} ms.`), + finalError: (error, attempts) => + error instanceof SendGridApiError + ? error + : new SendGridApiError(error.message, undefined, undefined, undefined, attempts, { + cause: error, + }), + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; + } + + /** + * Sends a message via SendGrid API. + * + * @param messageData The JSON data to send to SendGrid. + * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. + * @returns Promise that resolves to the SendGrid response. + * @throws {DOMException} If the caller aborts the request. + * @throws {SendGridApiError} If SendGrid returns a client error or all retry + * attempts are exhausted. + */ + sendMessage(messageData, signal, onRetry) { + return this.#request({ + url: `${this.config.baseUrl}/mail/send`, + body: messageData, + signal, + onRetry, + }); + } +} + module.exports = { SendGridHttpClient, SendGridApiError }; diff --git a/src/sendgrid/sendgrid-transport.js b/src/sendgrid/sendgrid-transport.js index e93be12..ce4e40e 100644 --- a/src/sendgrid/sendgrid-transport.js +++ b/src/sendgrid/sendgrid-transport.js @@ -2,14 +2,12 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { generateId } = require('../core/id.js'); -const { createFailedReceipt } = require('../core/receipt.js'); const { createSendGridConfig } = require('./config.js'); const { verifyByRequest } = require('../internal/verify.js'); -const { SendGridApiError, SendGridHttpClient } = require('./http-client.js'); +const { SendGridHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage } = require('./message-converter.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); /** @@ -166,10 +164,7 @@ class SendGridTransport { if (isCallerAbort(error, options?.signal)) { throw error; } - - const errorMessage = error instanceof Error ? error.message : String(error); - - return createSendGridFailure(errorMessage, error); + return providerFailure('sendgrid', error); } } @@ -258,32 +253,4 @@ class SendGridTransport { } } -const createSendGridFailure = (message, error) => { - const known = refusalFailure('sendgrid', error) ?? certificateFailure('sendgrid', message, error); - if (known !== undefined) return known; - if (error instanceof SendGridApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'sendgrid', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - providerDetails: error.errors, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'sendgrid', - }); -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - module.exports = { SendGridTransport }; From 838bd1043b15a7d1d676aabdccbb1acdfa463e77 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 03:57:12 +0300 Subject: [PATCH 099/143] refactor(mailgun): the client on createApiClient, MailgunApiError over its error The form requests of messages and messages.mime go through the shared client with Basic credentials; MailgunApiError extends ProviderApiError, and the transport's receipts are the shared ones, a refusal among them. ./mailgun goes from 6.5 to 7.5 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/mailgun/http-client.js | 242 ++++++++++++------------------- src/mailgun/mailgun-transport.js | 41 +----- 3 files changed, 100 insertions(+), 185 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 0bdf490..55e9f0e 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -10,7 +10,7 @@ "./ses": 8, "./resend": 7.5, "./sendgrid": 7.5, - "./mailgun": 6.5, + "./mailgun": 7.5, "./mailtrap": 7, "./maileroo": 6.5, "./lettermint": 6.5, diff --git a/src/mailgun/http-client.js b/src/mailgun/http-client.js index 9816063..a07a834 100644 --- a/src/mailgun/http-client.js +++ b/src/mailgun/http-client.js @@ -1,24 +1,100 @@ // Ported from upyo v0.7.0 (packages/mailgun/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and MailgunApiError +// extends its error; the messages and the errors of upstream stay. -const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); const { Buffer } = require('node:buffer'); +const { parseRetryAfter } = require('../core/receipt.js'); +const { + ProviderApiError, + createApiClient, + parseAnswer, + truncateErrorText, +} = require('../internal/api-client.js'); + +/** + * Error thrown when a Mailgun API request fails. + */ +class MailgunApiError extends ProviderApiError { + /** + * Creates a Mailgun API error. + * + * @param message Error message. + * @param statusCode HTTP status code returned by Mailgun. + * @param retryAfterMilliseconds Retry delay from the response. + * @param attempts Number of attempts made before this error. + * @param options The cause of the error, such as the failure of `fetch`. + */ + constructor(message, statusCode, retryAfterMilliseconds, attempts, options) { + super('mailgun', message, { + statusCode, + retryAfterMilliseconds, + attempts, + cause: options?.cause, + }); + this.name = 'MailgunApiError'; + } +} + +const rejectionOf = (response, read, attempt) => { + const text = read ?? ''; + let errorMessage; + try { + // Try to parse the error message from the JSON body + errorMessage = JSON.parse(text)?.message; + } catch { + // Ignore if JSON parsing fails, as the body may be non-JSON + } + + // Fallback logic for creating a meaningful error message. + // 1. Use the parsed `errorMessage` if available. + // 2. Otherwise, use the raw `text` response. + // 3. If the raw `text` is also empty, fall back to the HTTP status. + // Using `||` is intentional here to treat empty strings as falsy + // and ensure a non-empty error message for the tests. + return new MailgunApiError( + errorMessage || truncateErrorText(text) || `HTTP ${response.status}`, + response.status, + parseRetryAfter(response.headers.get('Retry-After')), + attempt + 1, + ); +}; + +// This handles cases where the request was successful (e.g., 2xx status) +// but the response body is not valid JSON. +const invalidJson = (error) => + new Error(`Successfully received response but failed to parse JSON: ${error.message}`, { + cause: error, + }); /** * HTTP client wrapper for Mailgun API requests. - * - * This class handles authentication, request formatting, error handling, - * and retry logic for Mailgun API calls. */ class MailgunHttpClient { config; + #request; constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'mailgun' }); + const client = createApiClient(config, { + id: 'mailgun', + label: 'Mailgun', + authorize: (headers) => { + const auth = Buffer.from(`api:${config.apiKey}`).toString('base64'); + headers.set('Authorization', `Basic ${auth}`); + }, + read: (_response, text) => parseAnswer(text, invalidJson), + reject: rejectionOf, + timeoutError: () => new Error(`Mailgun API request timed out after ${config.timeout} ms.`), + finalError: (error, attempts) => + error instanceof MailgunApiError + ? error + : new MailgunApiError(error.message, undefined, undefined, attempts, { cause: error }), + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** @@ -28,20 +104,11 @@ class MailgunHttpClient { * @param signal Optional AbortSignal for cancellation. * @param onRetry Optional callback told of each retry. * @returns Promise that resolves to the Mailgun response. + * @throws {MailgunApiError} If Mailgun returns an error response or all + * retry attempts are exhausted. */ sendMessage(formData, signal, onRetry) { - // The domain is a path segment: encoded, it cannot climb the path. - const url = `${this.config.baseUrl}/${encodeURIComponent(this.config.domain)}/messages`; - - return this.makeRequest( - url, - { - method: 'POST', - body: formData, - signal, - }, - onRetry, - ); + return this.#send('messages', formData, signal, onRetry); } /** @@ -53,138 +120,19 @@ class MailgunHttpClient { * @returns Promise that resolves to the Mailgun response. */ sendMime(formData, signal, onRetry) { - const url = `${this.config.baseUrl}/${encodeURIComponent(this.config.domain)}/messages.mime`; - return this.makeRequest(url, { method: 'POST', body: formData, signal }, onRetry); + return this.#send('messages.mime', formData, signal, onRetry); } - /** - * Makes an HTTP request to Mailgun API with retry logic. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @returns Promise that resolves to the parsed response. - * @throws {MailgunApiError} If Mailgun returns an error response or all - * retry attempts are exhausted. - */ - makeRequest(url, options, onRetry) { - return requestWithRetry({ - log: this.log, - signal: options.signal, + #send(endpoint, formData, signal, onRetry) { + // The domain is a path segment: encoded, it cannot climb the path. + const domain = encodeURIComponent(this.config.domain); + return this.#request({ + url: `${this.config.baseUrl}/${domain}/${endpoint}`, + body: formData, + signal, onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (signal) => this.fetchWithAuth(url, options, signal), - read: async (response) => { - const text = await readText(response); - try { - return JSON.parse(text); - } catch (error) { - // This handles cases where the request was successful (e.g., 2xx status) - // but the response body is not valid JSON. - throw new Error( - `Successfully received response but failed to parse JSON: ${error.message}`, - { cause: error }, - ); - } - }, - reject: async (response, attempt) => { - const text = await readText(response); - let errorMessage; - try { - // Try to parse the error message from the JSON body - errorMessage = JSON.parse(text)?.message; - } catch { - // Ignore if JSON parsing fails, as the body may be non-JSON - } - - // Fallback logic for creating a meaningful error message. - // 1. Use the parsed `errorMessage` if available. - // 2. Otherwise, use the raw `text` response. - // 3. If the raw `text` is also empty, fall back to the HTTP status. - // Using `||` is intentional here to treat empty strings as falsy - // and ensure a non-empty error message for the tests. - return new MailgunApiError( - errorMessage || truncateErrorBody(text) || `HTTP ${response.status}`, - response.status, - parseRetryAfter(response.headers.get('Retry-After')), - attempt + 1, - ); - }, - createTimeoutError: () => - new Error(`Mailgun API request timed out after ${this.config.timeout} ms.`), - finalError: (error, attempts) => - error instanceof MailgunApiError - ? error - : new MailgunApiError(error.message, undefined, undefined, attempts, { cause: error }), }); } - - /** - * Makes a fetch request with Mailgun authentication. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @param signal The signal of the attempt, with its deadline. - * @returns Promise that resolves to the fetch response. - */ - fetchWithAuth(url, options, signal) { - const headers = new Headers(options.headers); - - // Add authentication header - const auth = Buffer.from(`api:${this.config.apiKey}`).toString('base64'); - headers.set('Authorization', `Basic ${auth}`); - - // Add custom headers from config - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - const fetch = this.config.fetch ?? globalThis.fetch; - return fetch(url, { ...options, headers, signal, redirect: 'manual' }); - } } -/** - * Error thrown when a Mailgun API request fails. - */ -class MailgunApiError extends Error { - /** - * HTTP status code returned by Mailgun, if the request reached the API. - */ - statusCode; - - /** - * Retry delay from Mailgun's `Retry-After` response header. - */ - retryAfterMilliseconds; - - /** - * Number of attempts made before this error was produced. - */ - attempts; - - /** - * Creates a Mailgun API error. - * - * @param message Error message. - * @param statusCode HTTP status code returned by Mailgun. - * @param retryAfterMilliseconds Retry delay from the response. - * @param attempts Number of attempts made before this error. - * @param options The cause of the error, such as the failure of `fetch`. - */ - constructor(message, statusCode, retryAfterMilliseconds, attempts, options) { - super(message, options); - this.name = 'MailgunApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; - } -} - -const truncateErrorBody = (text) => { - return text.length > 500 ? `${text.slice(0, 500)}...` : text; -}; - module.exports = { MailgunHttpClient, MailgunApiError }; diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index a4c3dcf..74b0efa 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -1,16 +1,14 @@ // Ported from upyo v0.7.0 (packages/mailgun/src/mailgun-transport.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee -const { createFailedReceipt } = require('../core/receipt.js'); const { createMailgunConfig } = require('./config.js'); const { verifyByRequest } = require('../internal/verify.js'); -const { MailgunApiError, MailgunHttpClient } = require('./http-client.js'); +const { MailgunHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage } = require('./message-converter.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); -const { certificateFailure } = require('../internal/refusal.js'); /** * Mailgun transport implementation for sending emails via Mailgun API. @@ -150,10 +148,7 @@ class MailgunTransport { if (isCallerAbort(error, options?.signal)) { throw error; } - - const errorMessage = error instanceof Error ? error.message : String(error); - - return createMailgunFailure(errorMessage, error); + return providerFailure('mailgun', error); } } @@ -178,8 +173,7 @@ class MailgunTransport { return { successful: true, messageId: response.id, provider: 'mailgun' }; } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - const errorMessage = error instanceof Error ? error.message : String(error); - return rawMessageFailure('mailgun', error) ?? createMailgunFailure(errorMessage, error); + return rawMessageFailure('mailgun', error) ?? providerFailure('mailgun', error); } } @@ -246,31 +240,4 @@ class MailgunTransport { } } -const createMailgunFailure = (message, error) => { - const certificate = certificateFailure('mailgun', message, error); - if (certificate !== undefined) return certificate; - if (error instanceof MailgunApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'mailgun', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'mailgun', - }); -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - module.exports = { MailgunTransport }; From c84d28238e8803b579e8e59bd7a34a805c6e1b9a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 04:03:21 +0300 Subject: [PATCH 100/143] refactor(maileroo): the client on createApiClient, its errors over the shared one MailerooApiError and MailerooTimeoutError extend ProviderApiError, the timeout carrying the category, code and retryability its receipt had; the transport's receipts are the shared ones, unchanged. ./maileroo goes from 6.5 to 7 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/maileroo/http-client.js | 206 ++++++++++++----------------- src/maileroo/maileroo-transport.js | 54 +------- 3 files changed, 87 insertions(+), 175 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 55e9f0e..cd30e53 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -12,7 +12,7 @@ "./sendgrid": 7.5, "./mailgun": 7.5, "./mailtrap": 7, - "./maileroo": 6.5, + "./maileroo": 7, "./lettermint": 6.5, "./plunk": 6, "./postmark": 7.5, diff --git a/src/maileroo/http-client.js b/src/maileroo/http-client.js index a9b7f10..997d66e 100644 --- a/src/maileroo/http-client.js +++ b/src/maileroo/http-client.js @@ -1,21 +1,21 @@ // Ported from upyo v0.7.0 (packages/maileroo/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and the +// errors extend its error; the messages and the errors of upstream stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); - -const maxErrorMessageLength = 500; +const { + ProviderApiError, + createApiClient, + parseAnswer, + truncateErrorText, +} = require('../internal/api-client.js'); /** * Maileroo API error class for API-specific failures. */ -class MailerooApiError extends Error { - statusCode; - retryAfterMilliseconds; - attempts; - +class MailerooApiError extends ProviderApiError { /** * Creates a Maileroo API error. * @@ -25,28 +25,20 @@ class MailerooApiError extends Error { * @param attempts Number of attempts made before this error. */ constructor(message, statusCode, retryAfterMilliseconds, attempts) { - super(message); + super('maileroo', message, { statusCode, retryAfterMilliseconds, attempts }); this.name = 'MailerooApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } /** * Maileroo request timeout error. */ -class MailerooTimeoutError extends Error { +class MailerooTimeoutError extends ProviderApiError { /** * Request timeout in milliseconds. */ timeout; - /** - * Number of attempts made before this error was produced. - */ - attempts; - /** * Creates a Maileroo request timeout error. * @@ -54,85 +46,14 @@ class MailerooTimeoutError extends Error { * @param attempts Number of attempts made before this error. */ constructor(timeout, attempts) { - super(`Maileroo API request timed out after ${timeout} ms.`); + super('maileroo', `Maileroo API request timed out after ${timeout} ms.`, { + attempts, + category: 'timeout', + code: 'timeout', + retryable: true, + }); this.name = 'MailerooTimeoutError'; this.timeout = timeout; - this.attempts = attempts; - } -} - -/** - * HTTP client wrapper for Maileroo API requests. - */ -class MailerooHttpClient { - config; - - /** - * Creates a new Maileroo HTTP client. - * - * @param config Resolved Maileroo configuration. - */ - constructor(config) { - this.config = config; - this.log = createLog(config.logger, { transport: 'maileroo' }); - } - - /** - * Sends a single message via Maileroo API. - * - * @param messageData The JSON data to send to Maileroo. - * @param signal Optional AbortSignal for cancellation. - * @returns Promise that resolves to the Maileroo response. - * @throws {MailerooApiError} If Maileroo returns an API error. - * @throws {MailerooTimeoutError} If the request timeout elapses. - */ - sendMessage(messageData, signal, onRetry) { - const url = `${this.config.baseUrl}/emails`; - return this.makeRequest(url, messageData, signal, onRetry); - } - - makeRequest(url, body, signal, onRetry) { - // Serialized once rather than on every attempt. - const json = JSON.stringify(body); - return requestWithRetry({ - log: this.log, - signal, - onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (attemptSignal) => this.fetchWithAuth(url, json, attemptSignal), - read: async (response) => parseResponse(await readText(response)), - reject: async (response, attempt) => - new MailerooApiError( - parseErrorMessage(await readText(response), response.status), - response.status, - parseRetryAfter(response.headers.get('Retry-After')), - attempt + 1, - ), - createTimeoutError: () => new MailerooTimeoutError(this.config.timeout), - finalError: withAttempts, - }); - } - - fetchWithAuth(url, json, signal) { - const headers = new Headers({ - 'Content-Type': 'application/json', - 'X-API-Key': this.config.apiKey, - }); - - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - return (this.config.fetch ?? globalThis.fetch)(url, { - method: 'POST', - headers, - body: json, - signal, - redirect: 'manual', - }); } } @@ -150,45 +71,84 @@ const parseErrorMessage = (text, statusCode) => { try { const errorBody = JSON.parse(text); if (errorBody != null && typeof errorBody === 'object') { - const mailerooError = errorBody; - if (typeof mailerooError.message === 'string' && mailerooError.message !== '') { - return truncateErrorMessage(mailerooError.message); + if (typeof errorBody.message === 'string' && errorBody.message !== '') { + return truncateErrorText(errorBody.message); } - if (typeof mailerooError.error === 'string' && mailerooError.error !== '') { - return truncateErrorMessage(mailerooError.error); + if (typeof errorBody.error === 'string' && errorBody.error !== '') { + return truncateErrorText(errorBody.error); } - if (Array.isArray(mailerooError.errors) && mailerooError.errors.length > 0) { - return truncateErrorMessage(JSON.stringify(mailerooError.errors)); + if (Array.isArray(errorBody.errors) && errorBody.errors.length > 0) { + return truncateErrorText(JSON.stringify(errorBody.errors)); } } } catch { // Ignore if JSON parsing fails, as the body may be non-JSON. } - return truncateErrorMessage(text) || `HTTP ${statusCode}`; + return truncateErrorText(text) || `HTTP ${statusCode}`; }; +const invalidJson = (error) => + new SyntaxError(`Invalid JSON response from Maileroo API: ${error.message}.`, { cause: error }); + const parseResponse = (text) => { - try { - const response = JSON.parse(text); - if (response == null || typeof response !== 'object') { - throw new SyntaxError('response is not an object'); - } - return response; - } catch (error) { - throw new SyntaxError( - `Invalid JSON response from Maileroo API: ${ - error instanceof Error ? error.message : String(error) - }.`, - { cause: error }, - ); + const response = parseAnswer(text, invalidJson); + if (response == null || typeof response !== 'object') { + throw invalidJson(new SyntaxError('response is not an object')); } + return response; }; -const truncateErrorMessage = (message) => { - return message.length > maxErrorMessageLength - ? `${message.slice(0, maxErrorMessageLength)}...` - : message; -}; +/** + * HTTP client wrapper for Maileroo API requests. + */ +class MailerooHttpClient { + config; + #request; + + /** + * Creates a new Maileroo HTTP client. + * + * @param config Resolved Maileroo configuration. + */ + constructor(config) { + this.config = config; + const client = createApiClient(config, { + id: 'maileroo', + label: 'Maileroo', + authorize: (headers) => headers.set('X-API-Key', config.apiKey), + read: (_response, text) => parseResponse(text), + reject: (response, text, attempt) => + new MailerooApiError( + parseErrorMessage(text ?? '', response.status), + response.status, + parseRetryAfter(response.headers.get('Retry-After')), + attempt + 1, + ), + timeoutError: () => new MailerooTimeoutError(config.timeout), + finalError: withAttempts, + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; + } + + /** + * Sends a message via Maileroo API. + * + * @param messageData The JSON data to send to Maileroo. + * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. + * @returns Promise that resolves to the Maileroo response. + */ + sendMessage(messageData, signal, onRetry) { + return this.#request({ + url: `${this.config.baseUrl}/emails`, + body: messageData, + signal, + onRetry, + }); + } +} module.exports = { MailerooApiError, MailerooTimeoutError, MailerooHttpClient }; diff --git a/src/maileroo/maileroo-transport.js b/src/maileroo/maileroo-transport.js index c7b06c5..4a54484 100644 --- a/src/maileroo/maileroo-transport.js +++ b/src/maileroo/maileroo-transport.js @@ -3,12 +3,11 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createMailerooConfig } = require('./config.js'); -const { MailerooApiError, MailerooHttpClient, MailerooTimeoutError } = require('./http-client.js'); +const { MailerooHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Maileroo transport implementation for sending emails via Maileroo API. @@ -92,7 +91,7 @@ class MailerooTransport { return responseToReceipt(response); } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - return createMailerooFailure(error instanceof Error ? error.message : String(error), error); + return providerFailure('maileroo', error); } } @@ -143,51 +142,4 @@ const responseToReceipt = (response) => { }; }; -const createMailerooFailure = (message, error) => { - const known = refusalFailure('maileroo', error) ?? certificateFailure('maileroo', message, error); - if (known !== undefined) return known; - if (error instanceof MailerooApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'maileroo', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - if (error instanceof MailerooTimeoutError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'maileroo', - category: 'timeout', - code: 'timeout', - retryable: true, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'maileroo', - attempts: getErrorAttempts(error), - }); -}; - -const getErrorAttempts = (error) => { - if (typeof error !== 'object' || error == null || !('attempts' in error)) { - return undefined; - } - const attempts = error.attempts; - return typeof attempts === 'number' ? attempts : undefined; -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - module.exports = { MailerooTransport }; From ecfa38de043571d9c50126797f39a08b3af2e17c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 04:03:21 +0300 Subject: [PATCH 101/143] refactor(lettermint): the client on createApiClient, its errors over the shared one The Idempotency-Key goes in the request of the shared client, which retries it as before; LettermintApiError and LettermintTimeoutError extend ProviderApiError, and the transport's receipts are the shared ones. ./lettermint goes from 6.5 to 7 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/lettermint/http-client.js | 191 ++++++++++--------------- src/lettermint/lettermint-transport.js | 68 +-------- 3 files changed, 85 insertions(+), 176 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index cd30e53..db16122 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -13,7 +13,7 @@ "./mailgun": 7.5, "./mailtrap": 7, "./maileroo": 7, - "./lettermint": 6.5, + "./lettermint": 7, "./plunk": 6, "./postmark": 7.5, "./brevo": 7.5, diff --git a/src/lettermint/http-client.js b/src/lettermint/http-client.js index e300b48..3af70b1 100644 --- a/src/lettermint/http-client.js +++ b/src/lettermint/http-client.js @@ -1,19 +1,16 @@ // Ported from upyo v0.7.0 (packages/lettermint/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and the +// errors extend its error; the messages and the errors of upstream stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); +const { ProviderApiError, createApiClient, parseAnswer } = require('../internal/api-client.js'); /** * Lettermint API error class for API-specific failures. */ -class LettermintApiError extends Error { - statusCode; - retryAfterMilliseconds; - attempts; - +class LettermintApiError extends ProviderApiError { /** * Creates a Lettermint API error. * @@ -23,28 +20,20 @@ class LettermintApiError extends Error { * @param attempts Number of attempts made before this error. */ constructor(message, statusCode, retryAfterMilliseconds, attempts) { - super(message); + super('lettermint', message, { statusCode, retryAfterMilliseconds, attempts }); this.name = 'LettermintApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } /** * Lettermint request timeout error. */ -class LettermintTimeoutError extends Error { +class LettermintTimeoutError extends ProviderApiError { /** * Request timeout in milliseconds. */ timeout; - /** - * Number of attempts made before this error was produced. - */ - attempts; - /** * Creates a Lettermint request timeout error. * @@ -52,18 +41,57 @@ class LettermintTimeoutError extends Error { * @param attempts Number of attempts made before this error. */ constructor(timeout, attempts) { - super(`Lettermint API request timed out after ${timeout} ms.`); + super('lettermint', `Lettermint API request timed out after ${timeout} ms.`, { + attempts, + category: 'timeout', + code: 'timeout', + retryable: true, + }); this.name = 'LettermintTimeoutError'; this.timeout = timeout; - this.attempts = attempts; } } +const withAttempts = (error, attempts) => { + if (error instanceof LettermintTimeoutError) { + return new LettermintTimeoutError(error.timeout, attempts); + } + if (error instanceof LettermintApiError) { + return error; + } + return Object.assign(error, { attempts }); +}; + +const parseErrorMessage = (text, statusCode) => { + try { + const errorBody = JSON.parse(text); + if (typeof errorBody.message === 'string' && errorBody.message !== '') { + return errorBody.message; + } + if (typeof errorBody.error === 'string' && errorBody.error !== '') { + return errorBody.error; + } + if (Array.isArray(errorBody.errors) && errorBody.errors.length > 0) { + return JSON.stringify(errorBody.errors); + } + } catch { + // Ignore if JSON parsing fails, as the body may be non-JSON. + } + + return text || `HTTP ${statusCode}`; +}; + +const invalidJson = (error) => + new SyntaxError(`Invalid JSON response from Lettermint API: ${error.message}.`, { + cause: error, + }); + /** * HTTP client wrapper for Lettermint API requests. */ class LettermintHttpClient { config; + #request; /** * Creates a new Lettermint HTTP client. @@ -72,7 +100,24 @@ class LettermintHttpClient { */ constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'lettermint' }); + const client = createApiClient(config, { + id: 'lettermint', + label: 'Lettermint', + authorize: (headers) => headers.set('x-lettermint-token', config.apiToken), + read: (_response, text) => parseAnswer(text, invalidJson), + reject: (response, text, attempt) => + new LettermintApiError( + parseErrorMessage(text ?? '', response.status), + response.status, + parseRetryAfter(response.headers.get('Retry-After')), + attempt + 1, + ), + timeoutError: () => new LettermintTimeoutError(config.timeout), + finalError: withAttempts, + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** @@ -81,116 +126,38 @@ class LettermintHttpClient { * @param messageData The JSON data to send to Lettermint. * @param signal Optional AbortSignal for cancellation. * @param idempotencyKey Optional idempotency key for request deduplication. + * @param onRetry Optional callback told of each retry. * @returns Promise that resolves to the Lettermint response. */ sendMessage(messageData, signal, idempotencyKey, onRetry) { - const url = `${this.config.baseUrl}/v1/send`; - return this.makeRequest(url, messageData, signal, idempotencyKey, onRetry); + return this.#send('/v1/send', messageData, signal, idempotencyKey, onRetry); } /** * Sends multiple messages via Lettermint batch API. * - * @param messagesData The messages to send to Lettermint. + * @param messagesData Array of message data objects to send. * @param signal Optional AbortSignal for cancellation. * @param idempotencyKey Optional idempotency key for request deduplication. + * @param onRetry Optional callback told of each retry. * @returns Promise that resolves to the Lettermint batch response. */ sendBatch(messagesData, signal, idempotencyKey, onRetry) { - const url = `${this.config.baseUrl}/v1/send/batch`; - return this.makeRequest(url, messagesData, signal, idempotencyKey, onRetry); + return this.#send('/v1/send/batch', messagesData, signal, idempotencyKey, onRetry); } - makeRequest(url, body, signal, idempotencyKey, onRetry) { - // Serialized once rather than on every attempt. - const json = JSON.stringify(body); - return requestWithRetry({ - log: this.log, + #send(path, body, signal, idempotencyKey, onRetry) { + // Lettermint deduplicates a repeated Idempotency-Key. + const hasKey = typeof idempotencyKey === 'string' && idempotencyKey !== ''; + return this.#request({ + url: `${this.config.baseUrl}${path}`, + body, signal, onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - // Lettermint deduplicates a repeated Idempotency-Key. - hasIdempotencyKey: idempotencyKey != null && idempotencyKey !== '', - send: (attemptSignal) => this.fetchWithAuth(url, json, attemptSignal, idempotencyKey), - read: async (response) => { - const text = await readText(response); - try { - return JSON.parse(text); - } catch (error) { - throw new SyntaxError( - `Invalid JSON response from Lettermint API: ${ - error instanceof Error ? error.message : String(error) - }.`, - { cause: error }, - ); - } - }, - reject: async (response, attempt) => - new LettermintApiError( - parseErrorMessage(await readText(response), response.status), - response.status, - parseRetryAfter(response.headers.get('Retry-After')), - attempt + 1, - ), - createTimeoutError: () => new LettermintTimeoutError(this.config.timeout), - finalError: withAttempts, - }); - } - - fetchWithAuth(url, json, signal, idempotencyKey) { - const headers = new Headers({ - 'Content-Type': 'application/json', - 'x-lettermint-token': this.config.apiToken, - }); - - if (idempotencyKey) { - headers.set('Idempotency-Key', idempotencyKey); - } - - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - return (this.config.fetch ?? globalThis.fetch)(url, { - method: 'POST', - headers, - body: json, - signal, - redirect: 'manual', + headers: hasKey ? { 'Idempotency-Key': idempotencyKey } : undefined, + hasIdempotencyKey: hasKey, }); } } -const withAttempts = (error, attempts) => { - if (error instanceof LettermintTimeoutError) { - return new LettermintTimeoutError(error.timeout, attempts); - } - if (error instanceof LettermintApiError) { - return error; - } - return Object.assign(error, { attempts }); -}; - -const parseErrorMessage = (text, statusCode) => { - try { - const errorBody = JSON.parse(text); - if (typeof errorBody.message === 'string' && errorBody.message !== '') { - return errorBody.message; - } - if (typeof errorBody.error === 'string' && errorBody.error !== '') { - return errorBody.error; - } - if (Array.isArray(errorBody.errors) && errorBody.errors.length > 0) { - return JSON.stringify(errorBody.errors); - } - } catch { - // Ignore if JSON parsing fails, as the body may be non-JSON. - } - - return text || `HTTP ${statusCode}`; -}; - module.exports = { LettermintApiError, LettermintTimeoutError, LettermintHttpClient }; diff --git a/src/lettermint/lettermint-transport.js b/src/lettermint/lettermint-transport.js index a41b74a..097adf7 100644 --- a/src/lettermint/lettermint-transport.js +++ b/src/lettermint/lettermint-transport.js @@ -3,15 +3,10 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createLettermintConfig } = require('./config.js'); -const { - LettermintApiError, - LettermintHttpClient, - LettermintTimeoutError, -} = require('./http-client.js'); +const { LettermintHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage, generateIdempotencyKey } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -101,7 +96,7 @@ class LettermintTransport { return responseToReceipt(response); } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - return createLettermintFailure(error instanceof Error ? error.message : String(error), error); + return providerFailure('lettermint', error); } } @@ -151,9 +146,7 @@ class LettermintTransport { receipts.push(undefined); } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - receipts.push( - createLettermintFailure(error instanceof Error ? error.message : String(error), error), - ); + receipts.push(providerFailure('lettermint', error)); } } @@ -187,15 +180,12 @@ class LettermintTransport { } } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - const errorMessage = error instanceof Error ? error.message : String(error); for (const receipt of receipts) { if (receipt !== undefined) { yield receipt; continue; } - yield { - ...createLettermintFailure(errorMessage, error), - }; + yield providerFailure('lettermint', error); } } } @@ -244,46 +234,6 @@ const responseToReceipt = (response) => { }); }; -const createLettermintFailure = (message, error) => { - const known = - refusalFailure('lettermint', error) ?? certificateFailure('lettermint', message, error); - if (known !== undefined) return known; - if (error instanceof LettermintApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'lettermint', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - if (error instanceof LettermintTimeoutError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'lettermint', - category: 'timeout', - code: 'timeout', - retryable: true, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'lettermint', - attempts: getErrorAttempts(error), - }); -}; - -const getErrorAttempts = (error) => { - if (typeof error !== 'object' || error == null || !('attempts' in error)) { - return undefined; - } - const attempts = error.attempts; - return typeof attempts === 'number' ? attempts : undefined; -}; - const isSuccessfulStatus = (status) => { switch (status) { case 'pending': @@ -305,12 +255,4 @@ const isSuccessfulStatus = (status) => { } }; -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - module.exports = { LettermintTransport }; From d914114e17f8d696643f10184a4adecc3f8d6246 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 04:03:21 +0300 Subject: [PATCH 102/143] refactor(plunk): the client on createApiClient, PlunkApiError over the shared one Plunk's refusal in a 200 (success: false) is the isAnswer of the shared client, its validation of the answer the read; the transport's receipts are the shared ones. ./plunk goes from 6 to 7 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/plunk/http-client.js | 218 +++++++++++++---------------------- src/plunk/plunk-transport.js | 38 +----- 3 files changed, 82 insertions(+), 176 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index db16122..71f55f5 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -14,7 +14,7 @@ "./mailtrap": 7, "./maileroo": 7, "./lettermint": 7, - "./plunk": 6, + "./plunk": 7, "./postmark": 7.5, "./brevo": 7.5, "./mailjet": 8.5, diff --git a/src/plunk/http-client.js b/src/plunk/http-client.js index 27cbc78..dd26f21 100644 --- a/src/plunk/http-client.js +++ b/src/plunk/http-client.js @@ -1,30 +1,22 @@ // Ported from upyo v0.7.0 (packages/plunk/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and +// PlunkApiError extends its error; the messages and the errors of upstream +// stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readJson, readText } = require('../internal/body.js'); +const { + ProviderApiError, + createApiClient, + parseAnswer, + truncateErrorText, +} = require('../internal/api-client.js'); /** - * Error thrown when a Plunk API request fails. + * Plunk API error class for handling API-specific errors. */ -class PlunkApiError extends Error { - /** - * HTTP status code returned by Plunk, if the request reached the API. - */ - statusCode; - - /** - * Retry delay from Plunk's `Retry-After` response header. - */ - retryAfterMilliseconds; - - /** - * Number of attempts made before this error was produced. - */ - attempts; - +class PlunkApiError extends ProviderApiError { /** * Creates a Plunk API error. * @@ -32,76 +24,76 @@ class PlunkApiError extends Error { * @param statusCode HTTP status code returned by Plunk. * @param retryAfterMilliseconds Retry delay from the response. * @param attempts Number of attempts made before this error. + * @param options The cause of the error, such as the failure of `fetch`. */ constructor(message, statusCode, retryAfterMilliseconds, attempts, options) { - super(message, options); + super('plunk', message, { + statusCode, + retryAfterMilliseconds, + attempts, + cause: options?.cause, + }); this.name = 'PlunkApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } +const invalidJson = (error) => new Error('Invalid JSON response from Plunk API', { cause: error }); + +const parseResponse = (response, text) => { + const data = parseAnswer(text, invalidJson); + + // Validate response structure + if (typeof data !== 'object' || data === null) { + throw new Error('Invalid response format: expected object'); + } + + if (typeof data.success !== 'boolean') { + throw new Error('Invalid response format: missing success field'); + } + + if (!data.success) { + throw new PlunkApiError( + data.message ?? 'Send operation failed without error details', + response.status, + ); + } + + const responseData = 'data' in data ? data.data : data; + if (typeof responseData !== 'object' || responseData === null) { + throw new Error('Invalid response format: missing data field'); + } + + return { + success: true, + emails: responseData.emails, + timestamp: responseData.timestamp, + }; +}; + /** * HTTP client wrapper for Plunk API requests. - * - * This class handles authentication, request formatting, error handling, - * and retry logic for the Plunk HTTP API. */ class PlunkHttpClient { config; + #request; - /** - * Creates a new Plunk HTTP client instance. - * - * @param config - Resolved Plunk configuration - */ constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'plunk' }); - } - - /** - * Sends a message via the Plunk API. - * - * This method makes a POST request to the `/v1/send` endpoint with proper - * authentication, retry logic, and error handling. - * - * @param emailData - The email data in Plunk API format - * @param signal - Optional AbortSignal for request cancellation - * @returns Promise that resolves to Plunk API response - * @throws PlunkError if the request fails after all retries - */ - sendMessage(emailData, signal, onRetry) { - const url = `${this.config.baseUrl}/v1/send`; - // Serialized once rather than on every attempt. - const json = JSON.stringify(emailData); - - return requestWithRetry({ - log: this.log, - signal, - onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (attemptSignal) => this.makeRequest(url, json, attemptSignal), - read: (response) => this.parseResponse(response), - reject: async (response, attempt) => { - let errorBody; - try { - errorBody = await readText(response); - } catch { - errorBody = 'Failed to read error response'; - } - - return new PlunkApiError( - `HTTP ${response.status}: ${response.statusText}. ${truncateErrorBody(errorBody)}`, + const client = createApiClient(config, { + id: 'plunk', + label: 'Plunk', + authorize: (headers) => headers.set('Authorization', `Bearer ${config.apiKey}`), + read: parseResponse, + reject: (response, text, attempt) => + new PlunkApiError( + `HTTP ${response.status}: ${response.statusText}. ${truncateErrorText( + text ?? 'Failed to read error response', + )}`, response.status, parseRetryAfter(response.headers.get('Retry-After')), attempt + 1, - ); - }, - createTimeoutError: () => - new Error(`Plunk API request timed out after ${this.config.timeout} ms.`), + ), + timeoutError: () => new Error(`Plunk API request timed out after ${config.timeout} ms.`), // Plunk refusing in a 200 (success: false) is a failure of its own. isAnswer: (error) => error instanceof PlunkApiError, finalError: (error, attempts) => @@ -115,81 +107,27 @@ class PlunkHttpClient { ) : new PlunkApiError(error.message, undefined, undefined, attempts, { cause: error }), }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** - * Makes an HTTP request to the Plunk API. + * Sends a message via Plunk API. * - * @param url - The request URL - * @param json - The email data, serialized as JSON - * @param signal - Optional AbortSignal for cancellation - * @returns Promise that resolves to the Response object + * @param emailData The JSON data to send to Plunk. + * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. + * @returns Promise that resolves to the Plunk response. */ - makeRequest(url, json, signal) { - const headers = { - Authorization: `Bearer ${this.config.apiKey}`, - 'Content-Type': 'application/json', - ...this.config.headers, - }; - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - return (this.config.fetch ?? globalThis.fetch)(url, { - method: 'POST', - headers, - body: json, + sendMessage(emailData, signal, onRetry) { + return this.#request({ + url: `${this.config.baseUrl}/v1/send`, + body: emailData, signal, - redirect: 'manual', + onRetry, }); } - - /** - * Parses the response from the Plunk API. - * - * @param response - The Response object from fetch - * @returns Promise that resolves to parsed PlunkResponse - */ - async parseResponse(response) { - try { - const data = await readJson(response); - - // Validate response structure - if (typeof data !== 'object' || data === null) { - throw new Error('Invalid response format: expected object'); - } - - if (typeof data.success !== 'boolean') { - throw new Error('Invalid response format: missing success field'); - } - - if (!data.success) { - throw new PlunkApiError( - data.message ?? 'Send operation failed without error details', - response.status, - ); - } - - const responseData = 'data' in data ? data.data : data; - if (typeof responseData !== 'object' || responseData === null) { - throw new Error('Invalid response format: missing data field'); - } - - return { - success: true, - emails: responseData.emails, - timestamp: responseData.timestamp, - }; - } catch (error) { - if (error instanceof SyntaxError) { - throw new Error('Invalid JSON response from Plunk API', { cause: error }); - } - throw error; - } - } } -const truncateErrorBody = (text) => { - return text.length > 500 ? `${text.slice(0, 500)}...` : text; -}; - module.exports = { PlunkApiError, PlunkHttpClient }; diff --git a/src/plunk/plunk-transport.js b/src/plunk/plunk-transport.js index 22bb202..c2443fc 100644 --- a/src/plunk/plunk-transport.js +++ b/src/plunk/plunk-transport.js @@ -2,14 +2,12 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { generateId } = require('../core/id.js'); -const { createFailedReceipt } = require('../core/receipt.js'); const { createPlunkConfig } = require('./config.js'); -const { PlunkApiError, PlunkHttpClient } = require('./http-client.js'); +const { PlunkHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { sendConcurrently } = require('../internal/batch.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); /** * Plunk transport implementation for sending emails via Plunk API. @@ -133,10 +131,7 @@ class PlunkTransport { if (isCallerAbort(error, options?.signal)) { throw error; } - - const errorMessage = error instanceof Error ? error.message : String(error); - - return createPlunkFailure(errorMessage, error); + return providerFailure('plunk', error); } } @@ -225,31 +220,4 @@ class PlunkTransport { } } -const createPlunkFailure = (message, error) => { - const known = refusalFailure('plunk', error) ?? certificateFailure('plunk', message, error); - if (known !== undefined) return known; - if (error instanceof PlunkApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'plunk', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'plunk', - }); -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - module.exports = { PlunkTransport }; From 2edf46d9ab1ecf56bcd0477782686ec6242fba62 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 04:03:22 +0300 Subject: [PATCH 103/143] refactor(mailtrap): the client on createApiClient, its errors over the shared one The failure after a 2xx is the acceptedError of the shared client, the sandbox's 429 still read as a wait of ten seconds; MailtrapApiError, MailtrapTimeoutError and MailtrapResponseError extend ProviderApiError, and the transport's receipts are the shared ones. ./mailtrap goes from 7 to 7.5 KB for the shared client. With it every HTTP provider requests through createApiClient, but SES, which signs each attempt, and JMAP; a test keeps it so, and another that the errors of 1.0 keep their names, arguments and fields. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/mailtrap/http-client.js | 239 ++++++++++++----------------- src/mailtrap/mailtrap-transport.js | 78 +--------- tests/unit/api.test.js | 70 +++++++++ 4 files changed, 177 insertions(+), 212 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 71f55f5..0c473fd 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -11,7 +11,7 @@ "./resend": 7.5, "./sendgrid": 7.5, "./mailgun": 7.5, - "./mailtrap": 7, + "./mailtrap": 7.5, "./maileroo": 7, "./lettermint": 7, "./plunk": 7, diff --git a/src/mailtrap/http-client.js b/src/mailtrap/http-client.js index 3d0d0c9..ba7113e 100644 --- a/src/mailtrap/http-client.js +++ b/src/mailtrap/http-client.js @@ -1,23 +1,24 @@ // Ported from upyo v0.7.0 (packages/mailtrap/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, and the +// errors extend its error; the messages and the errors of upstream stay. const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); +const { + ProviderApiError, + createApiClient, + parseAnswer, + truncateErrorText, +} = require('../internal/api-client.js'); -const maxErrorMessageLength = 500; const sandboxRateLimitWindowMilliseconds = 10_000; const sandboxRateLimitMessage = 'too many emails per second'; /** * Mailtrap API error class for API-specific failures. */ -class MailtrapApiError extends Error { - statusCode; - retryAfterMilliseconds; - attempts; - +class MailtrapApiError extends ProviderApiError { /** * Creates a Mailtrap API error. * @@ -28,28 +29,20 @@ class MailtrapApiError extends Error { * @param attempts Number of attempts made before this error. */ constructor(message, statusCode, retryAfterMilliseconds, attempts) { - super(message); + super('mailtrap', message, { statusCode, retryAfterMilliseconds, attempts }); this.name = 'MailtrapApiError'; - this.statusCode = statusCode; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } /** * Mailtrap request timeout error. */ -class MailtrapTimeoutError extends Error { +class MailtrapTimeoutError extends ProviderApiError { /** * Request timeout in milliseconds. */ timeout; - /** - * Number of attempts made before this error was produced. - */ - attempts; - /** * Creates a Mailtrap request timeout error. * @@ -57,10 +50,14 @@ class MailtrapTimeoutError extends Error { * @param attempts Number of attempts made before this error. */ constructor(timeout, attempts) { - super(`Mailtrap API request timed out after ${timeout} ms.`); + super('mailtrap', `Mailtrap API request timed out after ${timeout} ms.`, { + attempts, + category: 'timeout', + code: 'timeout', + retryable: true, + }); this.name = 'MailtrapTimeoutError'; this.timeout = timeout; - this.attempts = attempts; } } @@ -68,12 +65,7 @@ class MailtrapTimeoutError extends Error { * Failure to read a successful Mailtrap response after the request was accepted. * Retrying the request could send the message again. */ -class MailtrapResponseError extends Error { - /** - * Number of attempts made before this error was produced. - */ - attempts; - +class MailtrapResponseError extends ProviderApiError { /** * Creates an error for an unreadable successful response. * @@ -81,17 +73,65 @@ class MailtrapResponseError extends Error { * @param attempts Number of attempts made before this error. */ constructor(cause, attempts) { - super(cause instanceof Error ? cause.message : String(cause), { cause }); + const timedOut = cause instanceof MailtrapTimeoutError; + super('mailtrap', cause instanceof Error ? cause.message : String(cause), { + cause, + attempts, + category: timedOut ? 'timeout' : undefined, + code: timedOut ? 'timeout' : undefined, + retryable: false, + }); this.name = 'MailtrapResponseError'; - this.attempts = attempts; } } +const withAttempts = (error, attempts) => { + if (error instanceof MailtrapTimeoutError) { + return new MailtrapTimeoutError(error.timeout, attempts); + } + if (error instanceof MailtrapApiError) { + return new MailtrapApiError( + error.message, + error.statusCode, + error.retryAfterMilliseconds, + attempts, + ); + } + return Object.assign(error, { attempts }); +}; + +const inferSandboxRetryAfter = (message, statusCode, sandbox) => { + if (sandbox && statusCode === 429 && message.toLowerCase().includes(sandboxRateLimitMessage)) { + return sandboxRateLimitWindowMilliseconds; + } + return undefined; +}; + +const parseErrorMessage = (text, statusCode) => { + try { + const errorBody = JSON.parse(text); + if (typeof errorBody.message === 'string' && errorBody.message !== '') { + return truncateErrorText(errorBody.message); + } + if (Array.isArray(errorBody.errors) && errorBody.errors.length > 0) { + return truncateErrorText(errorBody.errors.join('; ')); + } + } catch { + // Ignore if JSON parsing fails, as the body may be non-JSON. + } + + return truncateErrorText(text) || `HTTP ${statusCode}`; +}; + +const invalidJson = (error) => + new SyntaxError(`Invalid JSON response from Mailtrap API: ${error.message}.`, { cause: error }); + /** * HTTP client wrapper for Mailtrap API requests. */ class MailtrapHttpClient { config; + #request; /** * Creates a new Mailtrap HTTP client. @@ -100,7 +140,32 @@ class MailtrapHttpClient { */ constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'mailtrap' }); + const client = createApiClient(config, { + id: 'mailtrap', + label: 'Mailtrap', + authorize: (headers) => { + headers.set('Api-Token', config.apiToken); + headers.set('User-Agent', config.userAgent); + }, + read: (_response, text) => parseAnswer(text, invalidJson), + reject: (response, text, attempt) => { + const message = parseErrorMessage(text ?? '', response.status); + return new MailtrapApiError( + message, + response.status, + parseRetryAfter(response.headers.get('Retry-After')) ?? + inferSandboxRetryAfter(message, response.status, config.sandbox), + attempt + 1, + ); + }, + timeoutError: () => new MailtrapTimeoutError(config.timeout), + // A 2xx response means Mailtrap may already have sent the message. + acceptedError: (error, attempts) => new MailtrapResponseError(error, attempts), + finalError: withAttempts, + }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** @@ -108,11 +173,11 @@ class MailtrapHttpClient { * * @param messageData The JSON data to send to Mailtrap. * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. * @returns Promise that resolves to the Mailtrap response. */ sendMessage(messageData, signal, onRetry) { - const url = this.resolveUrl('send'); - return this.makeRequest(url, messageData, signal, onRetry); + return this.#send('send', messageData, signal, onRetry); } /** @@ -120,11 +185,11 @@ class MailtrapHttpClient { * * @param requests The messages to send to Mailtrap. * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. * @returns Promise that resolves to the Mailtrap batch response. */ sendBatch(requests, signal, onRetry) { - const url = this.resolveUrl('batch'); - return this.makeRequest(url, { requests }, signal, onRetry); + return this.#send('batch', { requests }, signal, onRetry); } resolveUrl(kind) { @@ -136,113 +201,11 @@ class MailtrapHttpClient { return `${baseUrl}/api/${kind}${suffix}`; } - makeRequest(url, body, signal, onRetry) { - // Serialized once rather than on every attempt. - const json = JSON.stringify(body); - return requestWithRetry({ - log: this.log, - signal, - onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (attemptSignal) => this.fetchWithAuth(url, json, attemptSignal), - read: async (response) => { - const text = await readText(response); - try { - return JSON.parse(text); - } catch (error) { - throw new SyntaxError( - `Invalid JSON response from Mailtrap API: ${ - error instanceof Error ? error.message : String(error) - }.`, - { cause: error }, - ); - } - }, - reject: async (response, attempt) => { - const message = parseErrorMessage(await readText(response), response.status); - return new MailtrapApiError( - message, - response.status, - parseRetryAfter(response.headers.get('Retry-After')) ?? - inferSandboxRetryAfter(message, response.status, this.config.sandbox), - attempt + 1, - ); - }, - createTimeoutError: () => new MailtrapTimeoutError(this.config.timeout), - // A 2xx response means Mailtrap may already have sent the message. - wrapAccepted: (error, attempts) => new MailtrapResponseError(error, attempts), - finalError: withAttempts, - }); - } - - fetchWithAuth(url, json, signal) { - const headers = new Headers({ - 'Content-Type': 'application/json', - 'Api-Token': this.config.apiToken, - 'User-Agent': this.config.userAgent, - }); - - for (const [key, value] of Object.entries(this.config.headers)) { - headers.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - return (this.config.fetch ?? globalThis.fetch)(url, { - method: 'POST', - headers, - body: json, - signal, - redirect: 'manual', - }); + #send(kind, body, signal, onRetry) { + return this.#request({ url: this.resolveUrl(kind), body, signal, onRetry }); } } -const withAttempts = (error, attempts) => { - if (error instanceof MailtrapTimeoutError) { - return new MailtrapTimeoutError(error.timeout, attempts); - } - if (error instanceof MailtrapApiError) { - return new MailtrapApiError( - error.message, - error.statusCode, - error.retryAfterMilliseconds, - attempts, - ); - } - return Object.assign(error, { attempts }); -}; - -const inferSandboxRetryAfter = (message, statusCode, sandbox) => { - if (sandbox && statusCode === 429 && message.toLowerCase().includes(sandboxRateLimitMessage)) { - return sandboxRateLimitWindowMilliseconds; - } - return undefined; -}; - -const parseErrorMessage = (text, statusCode) => { - try { - const errorBody = JSON.parse(text); - if (typeof errorBody.message === 'string' && errorBody.message !== '') { - return truncateErrorMessage(errorBody.message); - } - if (Array.isArray(errorBody.errors) && errorBody.errors.length > 0) { - return truncateErrorMessage(errorBody.errors.join('; ')); - } - } catch { - // Ignore if JSON parsing fails, as the body may be non-JSON. - } - - return truncateErrorMessage(text) || `HTTP ${statusCode}`; -}; - -const truncateErrorMessage = (message) => { - return message.length > maxErrorMessageLength - ? `${message.slice(0, maxErrorMessageLength)}...` - : message; -}; - module.exports = { MailtrapApiError, MailtrapTimeoutError, diff --git a/src/mailtrap/mailtrap-transport.js b/src/mailtrap/mailtrap-transport.js index ff9455b..7f92ff2 100644 --- a/src/mailtrap/mailtrap-transport.js +++ b/src/mailtrap/mailtrap-transport.js @@ -3,16 +3,10 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { createMailtrapConfig } = require('./config.js'); -const { - MailtrapApiError, - MailtrapHttpClient, - MailtrapResponseError, - MailtrapTimeoutError, -} = require('./http-client.js'); +const { MailtrapHttpClient } = require('./http-client.js'); +const { isCallerAbort, providerFailure } = require('../internal/api-client.js'); const { convertMessage } = require('./message-converter.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); -const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); -const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const MAX_BATCH_SIZE = 500; @@ -100,7 +94,7 @@ class MailtrapTransport { return responseToReceipt(response); } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - return createMailtrapFailure(error instanceof Error ? error.message : String(error), error); + return providerFailure('mailtrap', error); } } @@ -139,9 +133,7 @@ class MailtrapTransport { receipts.push(undefined); } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - receipts.push( - createMailtrapFailure(error instanceof Error ? error.message : String(error), error), - ); + receipts.push(providerFailure('mailtrap', error)); } } @@ -193,13 +185,12 @@ class MailtrapTransport { } } catch (error) { if (isCallerAbort(error, options?.signal)) throw error; - const errorMessage = error instanceof Error ? error.message : String(error); for (const receipt of receipts) { if (receipt !== undefined) { yield receipt; continue; } - yield createMailtrapFailure(errorMessage, error); + yield providerFailure('mailtrap', error); } } } @@ -250,68 +241,9 @@ const toReceipt = (response, options) => { }; }; -const createMailtrapFailure = (message, error) => { - const known = refusalFailure('mailtrap', error) ?? certificateFailure('mailtrap', message, error); - if (known !== undefined) return known; - if (error instanceof MailtrapResponseError) { - return createFailedReceipt(message, { - deliveryUnknown: true, - provider: 'mailtrap', - ...(error.cause instanceof MailtrapTimeoutError - ? { category: 'timeout', code: 'timeout' } - : {}), - retryable: false, - attempts: error.attempts, - }); - } - - if (error instanceof MailtrapApiError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'mailtrap', - statusCode: error.statusCode, - retryAfterMilliseconds: error.retryAfterMilliseconds, - attempts: error.attempts, - }); - } - - if (error instanceof MailtrapTimeoutError) { - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'mailtrap', - category: 'timeout', - code: 'timeout', - retryable: true, - attempts: error.attempts, - }); - } - - return createFailedReceipt(message, { - deliveryUnknown: isDeliveryUnknown(error), - provider: 'mailtrap', - attempts: getErrorAttempts(error), - }); -}; - const formatErrors = (errors) => { if (errors == null || errors.length === 0) return undefined; return errors.join('; '); }; -const getErrorAttempts = (error) => { - if (typeof error !== 'object' || error == null || !('attempts' in error)) { - return undefined; - } - const attempts = error.attempts; - return typeof attempts === 'number' ? attempts : undefined; -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - module.exports = { MailtrapTransport }; diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js index 03b87ae..39169f0 100644 --- a/tests/unit/api.test.js +++ b/tests/unit/api.test.js @@ -146,3 +146,73 @@ test('API: a refusal of the converter is a non-retryable unsupported failure', ( const range = providerFailure('x', new RangeError('The body is over 4 MiB.')); assert.notStrictEqual(range.errors[0].category, 'unsupported'); }); + +test('API: every HTTP provider requests through createApiClient, but SES and JMAP', () => { + // Before: the seven clients ported from upyo each carried their own fetch, + // retry wiring, timeouts and truncation, and a fix of one (bug 29) missed + // another. SES signs each attempt with SigV4 and JMAP has its own session. + const fs = require('node:fs'); + const path = require('node:path'); + const src = path.join(__dirname, '../../src'); + const direct = []; + for (const entry of fs.readdirSync(src, { withFileTypes: true })) { + const file = path.join(src, entry.name, 'http-client.js'); + if (!entry.isDirectory() || !fs.existsSync(file)) continue; + const text = fs.readFileSync(file, 'utf8'); + if (text.includes('requestWithRetry')) direct.push(entry.name); + else assert.match(text, /createApiClient\(/, entry.name); + } + assert.deepStrictEqual(direct.sort(), ['jmap', 'ses']); +}); + +test('API: the errors of 1.0 keep their names, arguments and fields over the shared error', () => { + const { SendGridApiError } = require('../../src/sendgrid/index.js'); + const { MailgunApiError } = require('../../src/mailgun/index.js'); + const maileroo = require('../../src/maileroo/index.js'); + const lettermint = require('../../src/lettermint/index.js'); + const mailtrap = require('../../src/mailtrap/index.js'); + const sendgrid = new SendGridApiError('bad', 400, [{ message: 'x' }], 1000, 2, { cause: 'c' }); + assert.ok(sendgrid instanceof Error && sendgrid instanceof ProviderApiError); + assert.deepStrictEqual( + [sendgrid.name, sendgrid.message, sendgrid.statusCode, sendgrid.retryAfterMilliseconds], + ['SendGridApiError', 'bad', 400, 1000], + ); + assert.deepStrictEqual(sendgrid.errors, [{ message: 'x' }]); + assert.deepStrictEqual([sendgrid.attempts, sendgrid.cause], [2, 'c']); + assert.deepStrictEqual(providerFailure('sendgrid', sendgrid).errors[0].providerDetails, [ + { message: 'x' }, + ]); + const mailgun = new MailgunApiError('down', 503, undefined, 3); + assert.deepStrictEqual( + [mailgun.name, mailgun.statusCode, mailgun.attempts], + ['MailgunApiError', 503, 3], + ); + for (const [module, prefix] of [ + [maileroo, 'Maileroo'], + [lettermint, 'Lettermint'], + [mailtrap, 'Mailtrap'], + ]) { + const api = new module[`${prefix}ApiError`]('no', 401, 5, 1); + assert.deepStrictEqual( + [api.name, api.statusCode, api.retryAfterMilliseconds, api.attempts], + [`${prefix}ApiError`, 401, 5, 1], + ); + const timeout = new module[`${prefix}TimeoutError`](1500, 2); + assert.deepStrictEqual( + [timeout.name, timeout.timeout, timeout.attempts], + [`${prefix}TimeoutError`, 1500, 2], + ); + assert.strictEqual(timeout.message, `${prefix} API request timed out after 1500 ms.`); + const receipt = providerFailure(prefix.toLowerCase(), timeout); + assert.deepStrictEqual( + [receipt.errors[0].category, receipt.errors[0].code, receipt.retryable, receipt.attempts], + ['timeout', 'timeout', true, 2], + ); + } + const read = new mailtrap.MailtrapResponseError(new mailtrap.MailtrapTimeoutError(10), 1); + assert.deepStrictEqual( + [read.name, read.attempts, read.retryable], + ['MailtrapResponseError', 1, false], + ); + assert.strictEqual(providerFailure('mailtrap', read).errors[0].category, 'timeout'); +}); From c0793822fe0dc8fb3984450f6004e4a08b693256 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 04:03:49 +0300 Subject: [PATCH 104/143] docs: the HTTP clients of the upyo providers on the shared one CLAUDE.md says that the clients ported from upyo are their requests over createApiClient, what stays upstream's, and the test that keeps it; the changelog of 1.1 tells what a user of 1.0 sees of it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++++ CLAUDE.md | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73a3303..adadd70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,11 @@ additive but the defaults listed below. transport that can send it; and a TLS certificate refused in the handshake is a non-retryable `configuration` failure at the first attempt, where SMTP reported it as `unknown` and the HTTP transports retried it and reported a retryable `network` failure (`deliveryUnknown` on Deno). +- **The HTTP providers of 1.0 share one client:** Resend, SendGrid, Mailgun, Maileroo, + Lettermint, Plunk and Mailtrap request through the client of the providers of 1.1. Their errors + (`SendGridApiError`, `MailtrapTimeoutError`…) keep their names, arguments and fields, and carry + `provider`, `category` and `retryable` besides; an error answer whose body cannot be read keeps + its status; and a message Mailgun cannot express is an `unsupported` failure, as elsewhere. - **On Deno, give the HTTP transports a `fetch` that keeps no idle connection** (`Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`): Deno's own sends a request again when a reused connection closes before its answer. See diff --git a/CLAUDE.md b/CLAUDE.md index fd89b87..8d77539 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,6 +126,16 @@ Copyright 2025 Hong Minhee; the notice is in `LICENSE`): one subpath per upyo pa logger passed in, fallbacks for runtimes older than Node 22 dropped, and the `@since` tags of upyo versions removed. A known upstream bug is fixed afterwards in its own `fix:` commit with a test that fails without it, not during the port. +- **The HTTP clients are this package's, the API of each provider upstream's.** The fixes 24 to + 39 had to land in seven copies of the same client, and one missed a copy (Mailtrap, bug 29), so + since 1.1 the clients of Resend, SendGrid, Mailgun, Maileroo, Lettermint, Plunk and Mailtrap + are their requests over `createApiClient` (`src/internal/api-client.js`), as those of 1.1 are: + what stays upstream's is the provider's side (its authorization, the messages it is read for, + what its 2xx must hold, its error classes), passed in as hooks. Their public errors + (`SendGridApiError`…) extend `ProviderApiError` with the names, arguments and fields of 1.0, and + the transports build their receipts with `providerFailure`. SES, which signs each attempt with + SigV4, and JMAP, with its session, keep their own clients; `tests/unit/api.test.js` fails when + another provider calls `requestWithRetry` on its own. - A lint rule that upstream code breaks is suppressed on the line, `// oxlint-disable-next-line -- upstream`, rather than rewritten; lifting those is a later refactor, like TS `private` fields, which stay plain properties (the tests reach into them). @@ -744,7 +754,8 @@ src/ version.js the package name and version (OpenTelemetry scope), kept equal to package.json / one directory per subpath: index.js + index.d.ts + its modules internal/ private modules shared by subpaths, not in the exports map: the request - layer (http.js, api.js for the providers of 1.1, body.js, unsent.js, + layer (http.js, api-client.js for every HTTP provider but SES and JMAP, + api.js for the conversions of the providers of 1.1, body.js, unsent.js, delivery-unknown.js, config.js, loopback.js, validate-ssl.js, verify.js), receipts (provider.js, refusal.js), message fields (headers.js, raw.js, schedule.js, tracking.js, capabilities.js, batch.js, base64.js, uuid.js), From 58c157482e7c8c45caf19db4fb6197ff27367b7f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:16:47 +0300 Subject: [PATCH 105/143] fix(gate): a message the wrapped sendMany never took is settled as unsent The first message of a run was admitted before the wrapped sendMany() took it, and its slot was made only once taken, so a sendMany() that threw or returned at once left it neither failed nor abandoned: its idempotency key stayed pending, a probe held, a hook unheard. Its slot is there from the start, and a message never taken goes to gate.unsent when the gate has it, or to fail and abandon as before. ./idempotency frees the key of an unsent message, as nothing went; only those the wrapped transport took are of unknown delivery. MAX_IN_PLACE is said for what it counts: the answers in place that wait for earlier receipts. Co-Authored-By: Claude Opus 5.5 --- src/idempotency/idempotency-transport.js | 3 ++ src/internal/gate.js | 49 ++++++++++++++---------- tests/unit/gate.test.js | 40 +++++++++++++++++++ tests/unit/idempotency.test.js | 23 +++++++++++ 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index 56468a7..f17a351 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -99,6 +99,9 @@ class IdempotencyTransport { }, fail: (error, admitted) => this.#thrown(admitted.entry, error, options?.signal), abandon: (admitted) => this.#abandon(admitted.entry), + // A message the wrapped transport never took was not sent: its key is + // free for the next attempt. + unsent: (admitted) => this.#release(admitted.entry), }); } diff --git a/src/internal/gate.js b/src/internal/gate.js index e55a3d6..240c87f 100644 --- a/src/internal/gate.js +++ b/src/internal/gate.js @@ -42,10 +42,10 @@ const iteratorOf = (messages) => ? messages[Symbol.asyncIterator]() : messages[Symbol.iterator](); -// The most answers in place a run holds while a message handed to the -// wrapped transport waits for its receipt: past it, the run ends, so that a -// long streak of them, a circuit open over an endless input, is not read -// ahead without bound. +// The most answers in place a run holds while the messages handed to the +// wrapped transport before them wait for their receipts: past it, the run +// ends, so that a circuit open over an endless input is not read ahead +// without bound, and the answers held stay few. const MAX_IN_PLACE = 64; /** @@ -54,17 +54,19 @@ const MAX_IN_PLACE = 64; * wrapped transport. The admitted ones go through one `transport.sendMany()`, * the answers in place taking their turn among its receipts, so that a * transport that batches keeps its batches; a run ends only at the end of the - * input or past `MAX_IN_PLACE` answers in place in a row. Each receipt goes - * through `gate.settle(receipt, admitted)` before it is yielded. When the - * wrapped `sendMany()` throws, `gate.fail(error, admitted)` hears of each - * message still waiting for its receipt; when the caller stops reading, or - * the wrapped `sendMany()` ends short, `gate.abandon(admitted)` does, as a - * transport that reads ahead may have sent them or not. + * input or when `MAX_IN_PLACE` answers in place wait for earlier receipts. + * Each receipt goes through `gate.settle(receipt, admitted)` before it is + * yielded. When the wrapped `sendMany()` throws, `gate.fail(error, admitted)` + * hears of each message still waiting for its receipt; when the caller stops + * reading, or the wrapped `sendMany()` ends short, `gate.abandon(admitted)` + * does, as a transport that reads ahead may have sent them or not. A message + * admitted that the wrapped `sendMany()` never took goes to + * `gate.unsent(admitted)` instead, when the gate has one: it was not sent. * * @param transport The wrapped transport. * @param messages The messages, sync or async. * @param options The caller's options, passed on. - * @param gate `admit`, and optional `settle`, `fail` and `abandon`. + * @param gate `admit`, and optional `settle`, `fail`, `abandon` and `unsent`. * @returns The receipts, one per message, in order. * @internal */ @@ -83,7 +85,11 @@ const gatedSendMany = async function* (transport, messages, options, gate) { } // In the order of the messages: `{ admitted }` waiting for its receipt, // or `{ receipt }` answered in place. - const run = { iterator, first: decision, slots: new Fifo(), inPlace: 0, exhausted: false }; + // The first is admitted before the wrapped sendMany() takes it, which it + // may never do: its slot is there from the start, not taken yet. + const first = { admitted: decision, taken: false }; + const run = { iterator, first, slots: new Fifo(), inPlace: 0, exhausted: false }; + run.slots.push(first); yield* sendRun(transport, run, options, gate); while (run.slots.size > 0) yield run.slots.shift().receipt; if (run.exhausted) return; @@ -96,8 +102,8 @@ const gatedSendMany = async function* (transport, messages, options, gate) { // The admitted messages of a run, read until the input ends or too many // answers in place wait. const feedRun = async function* (run, gate) { - run.slots.push({ admitted: run.first }); - yield run.first.message; + run.first.taken = true; + yield run.first.admitted.message; for (;;) { if (run.inPlace >= MAX_IN_PLACE) return; const step = await run.iterator.next(); @@ -111,19 +117,21 @@ const feedRun = async function* (run, gate) { run.inPlace++; continue; } - run.slots.push({ admitted: decision }); + run.slots.push({ admitted: decision, taken: true }); yield decision.message; } }; // The messages of a run still waiting for their receipts, to be failed or -// abandoned; the answers in place among them stay for the caller. -const drainAdmitted = async (run, each) => { +// abandoned, or told unsent when the wrapped sendMany() never took them; the +// answers in place among them stay for the caller. +const drainAdmitted = async (run, each, unsent) => { const kept = new Fifo(); while (run.slots.size > 0) { const slot = run.slots.shift(); if (slot.admitted === undefined) kept.push(slot); - else await each(slot.admitted); + else if (!slot.taken && unsent !== undefined) await unsent(slot.admitted); + else if (each !== undefined) await each(slot.admitted); } run.slots = kept; }; @@ -142,12 +150,13 @@ const sendRun = async function* (transport, run, options, gate) { } } catch (error) { failed = true; - if (gate.fail !== undefined) await drainAdmitted(run, (admitted) => gate.fail(error, admitted)); + const fail = gate.fail === undefined ? undefined : (admitted) => gate.fail(error, admitted); + await drainAdmitted(run, fail, gate.unsent); throw error; } finally { // The caller stopped reading, or the transport answered fewer messages // than it was handed: those it was handed are neither settled nor failed. - if (!failed && gate.abandon !== undefined) await drainAdmitted(run, gate.abandon); + if (!failed) await drainAdmitted(run, gate.abandon, gate.unsent); } }; diff --git a/tests/unit/gate.test.js b/tests/unit/gate.test.js index 83d7de2..a0c2acb 100644 --- a/tests/unit/gate.test.js +++ b/tests/unit/gate.test.js @@ -95,6 +95,46 @@ test('Gate: what a transport reading ahead was handed is abandoned on a break', assert.deepStrictEqual(abandoned, [1, 2]); }); +test('Gate: a message the wrapped sendMany never took is told unsent, or failed', async () => { + // Before: the first message of a run was admitted before the wrapped + // sendMany() took it, and its slot made only once taken, so a sendMany() + // that threw or returned at once left it neither failed nor abandoned. + const told = []; + const gate = { + admit: (n) => ({ message: n }), + fail: (error, admitted) => { + told.push(['fail', admitted.message, error.message]); + }, + abandon: (admitted) => { + told.push(['abandon', admitted.message]); + }, + }; + const throwing = { + // oxlint-disable-next-line require-yield -- it throws before any message + async *sendMany() { + throw new Error('down'); + }, + }; + const empty = { + async *sendMany() {}, + }; + await assert.rejects(collect(gatedSendMany(throwing, [1, 2], undefined, gate)), /down/); + assert.deepStrictEqual(await collect(gatedSendMany(empty, [3], undefined, gate)), []); + assert.deepStrictEqual(told, [ + ['fail', 1, 'down'], + ['abandon', 3], + ]); + // A gate with unsent hears of them there: they were not sent. + told.length = 0; + const unsentGate = { ...gate, unsent: (admitted) => told.push(['unsent', admitted.message]) }; + await assert.rejects(collect(gatedSendMany(throwing, [1], undefined, unsentGate)), /down/); + assert.deepStrictEqual(await collect(gatedSendMany(empty, [3], undefined, unsentGate)), []); + assert.deepStrictEqual(told, [ + ['unsent', 1], + ['unsent', 3], + ]); +}); + test('Gate: answers in place do not split the batch of the wrapped transport', async () => { // Before: each answer in place ended the run, so a batching transport got // batches of the messages between two of them. diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js index ca43c19..cc71537 100644 --- a/tests/unit/idempotency.test.js +++ b/tests/unit/idempotency.test.js @@ -497,3 +497,26 @@ test('Idempotency: a key a batch left before its answer is held for the lease al const later = await idempotent.send(message({ idempotencyKey: 'k-2', subject: 'Second' })); assert.strictEqual(later.successful, true); }); + +test('Idempotency: a key the wrapped sendMany never took is free again', async () => { + // Before: the first message of a batch was reserved before the wrapped + // sendMany() took it; one that threw at once left its key pending, every + // send of it answered in_flight for the lease. + let throws = true; + const inner = fakeTransport(); + const original = inner.sendMany.bind(inner); + inner.sendMany = (messages, options) => { + if (!throws) return original(messages, options); + // oxlint-disable-next-line require-yield -- it throws before any message + return (async function* () { + throw new Error('not yet'); + })(); + }; + const once = new IdempotencyTransport(inner); + const keyed = message({ idempotencyKey: 'welcome:9' }); + await assert.rejects(collect(once.sendMany([keyed])), /not yet/); + throws = false; + const receipts = await collect(once.sendMany([keyed])); + assert.strictEqual(receipts[0].successful, true); + assert.strictEqual(inner.calls.length, 1); +}); From 64bc5cd1afa4de7ec545d707e37d191b971240a6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:17:27 +0300 Subject: [PATCH 106/143] fix(breaker): a clock that throws gives the last time it gave A now() that threw fell back to Date.now, another clock than the monotonic default or the caller's, so a circuit that opened on one and was read on the other waited about fifty years to half-open. The breaker keeps the last time its clock gave and uses it when the clock throws. Co-Authored-By: Claude Opus 5.5 --- src/breaker/breaker-transport.js | 7 +++++-- tests/unit/breaker.test.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 6e834c9..5230c6e 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -50,6 +50,7 @@ class CircuitBreakerTransport { // Grows at each change of state: a result counts only in the period it // was sent in, not in one the circuit came to since. #period = 0; + #lastNow = 0; #log; /** @@ -74,13 +75,15 @@ class CircuitBreakerTransport { return this.#state; } + // A clock that throws gives the last time it gave, never one of another + // clock, which would put the intervals decades off. #now() { try { - return this.config.now(); + this.#lastNow = this.config.now(); } catch (error) { warnThrown('The now of CircuitBreakerTransport', error, 'MAILER_BREAKER_LISTENER_FAILED'); - return Date.now(); } + return this.#lastNow; } /** diff --git a/tests/unit/breaker.test.js b/tests/unit/breaker.test.js index 2a05ba2..aec99ea 100644 --- a/tests/unit/breaker.test.js +++ b/tests/unit/breaker.test.js @@ -384,3 +384,32 @@ test('Breaker: a jump of the time of day does not move the circuit', async (t) = shift = 60 * 60 * 1000; assert.strictEqual(breaker.state, 'open'); }); + +test('Breaker: a clock that throws gives the last time it gave', async (t) => { + // Before: a now() that threw fell back to Date.now, another clock, so a + // circuit opened on a monotonic clock waited about fifty years to retry. + const warned = []; + t.mock.method(process, 'emitWarning', (_text, options) => warned.push(options?.code)); + let time = 1000; + let throws = false; + const breaker = new CircuitBreakerTransport( + fakeTransport(() => failed('network')), + { + failureThreshold: 1, + resetTimeoutMs: 60_000, + now: () => { + if (throws) throw new Error('clock gone'); + return time; + }, + }, + ); + await breaker.send(message()); + throws = true; + const refused = await breaker.send(message()); + assert.strictEqual(refused.successful, false); + assert.ok(refused.errors[0].retryAfterMilliseconds <= 60_000); + assert.ok(warned.includes('MAILER_BREAKER_LISTENER_FAILED')); + throws = false; + time += 60_000; + assert.strictEqual(breaker.state, 'half-open'); +}); From a8da325eff6391ce4f7776e403d70c20f2ff8866 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:22:27 +0300 Subject: [PATCH 107/143] fix(ratelimit): full buckets dropped first, a bounded queue, the abort and the waits - maxKeys dropped the least used bucket with no waiter, full or not, so a limited key rotated out came back full, silently. A full bucket, which loses nothing, goes first; a bucket not yet full goes only when none is, and a warning says so (MAILER_RATELIMIT_EVICTED, or ratelimit.evicted). - maxWaitMs: Infinity let any number of messages wait; maxQueue (10000) bounds the waiters of a bucket, one more refused as rate-limit. - A message whose tokens were there did not hear a signal that aborted between two messages of a batch, and went. - A shared store that answers { taken: false } without waitMs was asked again every millisecond; it is asked after the time the rate gives. - ratelimit.waited tells the time a send waited, once it has its tokens. warnLimited moves to internal/warn.js, so a decorator that warns needs no redaction of addresses. The docs give the clock as performance.now(), and the d.ts keeps the comment of RateLimitConfig on it. ./ratelimit goes from 3 to 3.5 KB. Co-Authored-By: Claude Opus 5.5 --- docs/transports/ratelimit.md | 16 ++-- scripts/size-budgets.json | 2 +- src/internal/callback.js | 34 +------- src/internal/warn.js | 37 +++++++++ src/ratelimit/config.js | 1 + src/ratelimit/index.d.ts | 31 +++++--- src/ratelimit/ratelimit-transport.js | 75 +++++++++++++++--- tests/unit/ratelimit.test.js | 111 +++++++++++++++++++++++++++ 8 files changed, 249 insertions(+), 58 deletions(-) create mode 100644 src/internal/warn.js diff --git a/docs/transports/ratelimit.md b/docs/transports/ratelimit.md index d289c7b..3b23435 100644 --- a/docs/transports/ratelimit.md +++ b/docs/transports/ratelimit.md @@ -36,9 +36,12 @@ takes `cost(message)` of them, 1 without `cost`. `key(message)` chooses the bucket, such as a tenant's, so one tenant's burst does not hold back the others. No more than `maxKeys` buckets are kept: past -it, the least used one with no waiter is dropped, and its key starts full when -it comes back, the price of a bound on memory. Disposing of the transport stops -the waits, each rejecting with an `AbortError`. +it, the least used full one is dropped, which loses nothing; when none is full, +the least used one with no waiter goes, its key starting full when it comes +back, the price of a bound on memory, and a warning says so +(`MAILER_RATELIMIT_EVICTED`, or `ratelimit.evicted` to the logger). No more than +`maxQueue` messages wait in one bucket, whatever `maxWaitMs` allows. Disposing +of the transport stops the waits, each rejecting with an `AbortError`. ```js const perTenant = new RateLimitTransport(transport, { @@ -93,10 +96,11 @@ const limited = new RateLimitTransport(transport, { limit: 14, store }); | `maxWaitMs` | `30000` | The longest wait, from the arrival. | | `key` | one bucket | The bucket of a message. | | `cost` | `1` | The tokens a message takes. | -| `maxKeys` | `10000` | The most buckets kept, the least used dropped first. | +| `maxKeys` | `10000` | The most buckets kept, the least used full one dropped first. | +| `maxQueue` | `10000` | The most messages waiting in one bucket. | | `store` | | The buckets the processes share, in place of this process's. | -| `now` | `Date.now` | The clock. | -| `logger` | | A pino-like logger: a refused send (`ratelimit.limited`) at `info`, a send that waits at `debug`. | +| `now` | `performance.now()` | The clock of the intervals, which no change of the time of day moves. | +| `logger` | | A pino-like logger: a refused send (`ratelimit.limited`) at `info`, a send that waits (`ratelimit.waiting`) and the time it waited (`ratelimit.waited`) at `debug`, buckets dropped before they were full (`ratelimit.evicted`) at `warn`. | `sendMany()` goes through the `sendMany()` of the wrapped transport, each message handed to it once it has its tokens, so a batch is throttled like as diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 0c473fd..edccd1f 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -31,7 +31,7 @@ "./otel": 5, "./logger": 2.5, "./breaker": 3, - "./ratelimit": 3, + "./ratelimit": 3.5, "./idempotency": 4, "./hooks": 2.5, "./tee": 3 diff --git a/src/internal/callback.js b/src/internal/callback.js index 0bcb5b1..70e4d12 100644 --- a/src/internal/callback.js +++ b/src/internal/callback.js @@ -3,39 +3,7 @@ // becomes a process warning rather than vanishing, as a logger's does. const { redactAddresses } = require('./redact-addresses.js'); - -// A listener or a mirror that fails for every message would fill the output -// of the process: no more than BURST warnings of a code go out a minute, and -// the first after it says how many were left out. -const BURST = 10; -const WINDOW_MS = 60 * 1000; -const periods = new Map(); - -/** - * Emits a process warning, at most `BURST` of a code a minute. - * - * @param code The code of the warning. - * @param text What it says. - * @internal - */ -const warnLimited = (code, text) => { - const now = performance.now(); - let period = periods.get(code); - let said = text; - if (period === undefined || now - period.start >= WINDOW_MS) { - if (period !== undefined && period.skipped > 0) { - said += ` (${period.skipped} more warnings of ${code} were left out in the last minute)`; - } - period = { start: now, count: 0, skipped: 0 }; - periods.set(code, period); - } - if (period.count >= BURST) { - period.skipped++; - return; - } - period.count++; - process.emitWarning(said, { code }); -}; +const { warnLimited } = require('./warn.js'); /** * The text of what was thrown, with the addresses in it redacted: an SMTP diff --git a/src/internal/warn.js b/src/internal/warn.js new file mode 100644 index 0000000..38f3e4b --- /dev/null +++ b/src/internal/warn.js @@ -0,0 +1,37 @@ +// Not ported: process warnings held to a few of each code a minute, so that +// a listener, a mirror or a logger that fails for every message does not +// fill the output of the process. + +// No more than BURST warnings of a code go out a minute, and the first after +// it says how many were left out. +const BURST = 10; +const WINDOW_MS = 60 * 1000; +const periods = new Map(); + +/** + * Emits a process warning, at most `BURST` of a code a minute. + * + * @param code The code of the warning. + * @param text What it says. + * @internal + */ +const warnLimited = (code, text) => { + const now = performance.now(); + let period = periods.get(code); + let said = text; + if (period === undefined || now - period.start >= WINDOW_MS) { + if (period !== undefined && period.skipped > 0) { + said += ` (${period.skipped} more warnings of ${code} were left out in the last minute)`; + } + period = { start: now, count: 0, skipped: 0 }; + periods.set(code, period); + } + if (period.count >= BURST) { + period.skipped++; + return; + } + period.count++; + process.emitWarning(said, { code }); +}; + +module.exports = { warnLimited }; diff --git a/src/ratelimit/config.js b/src/ratelimit/config.js index 7acc45b..96f7381 100644 --- a/src/ratelimit/config.js +++ b/src/ratelimit/config.js @@ -45,6 +45,7 @@ const createRateLimitConfig = (config = {}) => { key: checkOptionalFunction(config.key, 'key'), cost: checkOptionalFunction(config.cost, 'cost'), maxKeys: checkPositiveInteger(config.maxKeys ?? 10000, 'maxKeys'), + maxQueue: checkPositiveInteger(config.maxQueue ?? 10000, 'maxQueue'), now: checkOptionalFunction(config.now, 'now') ?? monotonic, logger: config.logger, store: checkStore(config.store), diff --git a/src/ratelimit/index.d.ts b/src/ratelimit/index.d.ts index 8e97d73..6113d83 100644 --- a/src/ratelimit/index.d.ts +++ b/src/ratelimit/index.d.ts @@ -14,9 +14,6 @@ import type { VerifiableTransport, } from '../../index.js'; -/** - * The options of {@link RateLimitTransport}. - */ /** * A store of token buckets that the processes of an application share, so * that N of them send no more than the one limit of the provider: on Redis, @@ -37,6 +34,9 @@ export interface RateLimitStore { | Promise<{ readonly taken: boolean; readonly waitMs?: number }>; } +/** + * The options of {@link RateLimitTransport}. + */ export interface RateLimitConfig { /** The tokens a bucket gains every `intervalMs`: the sustained rate. */ readonly limit: number; @@ -73,25 +73,37 @@ export interface RateLimitConfig { */ readonly cost?: (message: Message | RawMessage) => number; /** - * The most buckets kept: past it, the least used bucket with no waiter is - * dropped, and its key starts full when it comes back. + * The most buckets kept: past it, the least used full bucket is dropped, + * which loses nothing; when none is full, the least used with no waiter, + * whose key starts full again when it comes back, which is told + * (`MAILER_RATELIMIT_EVICTED`, or `ratelimit.evicted` at `warn`). * @default 10000 */ readonly maxKeys?: number; + /** + * The most messages that wait in one bucket: one more is refused with a + * retryable `rate-limit` receipt, however long `maxWaitMs` allows. + * @default 10000 + */ + readonly maxQueue?: number; /** * The clock of the intervals, in milliseconds: `performance.now()` by * default, which no change of the time of day moves. */ readonly now?: () => number; /** - * A pino-like logger: it hears a refused send at `info` (`ratelimit.limited`) and a send that - * waits at `debug` (`ratelimit.waiting`), with `waitMs`. + * A pino-like logger: it hears a refused send at `info` + * (`ratelimit.limited`), a send that waits at `debug` (`ratelimit.waiting`, + * with `waitMs`, and `ratelimit.waited`, with `waitedMs`, once it has its + * tokens), and buckets dropped before they were full at `warn` + * (`ratelimit.evicted`). */ readonly logger?: Logger; /** * The buckets the processes share, in place of the ones of this process: - * a send waits, while `maxWaitMs` allows, for the time the store says, and - * the store, not this process, keeps the order. + * a send waits, while `maxWaitMs` allows, for the time the store says (or, + * when it says none, the time the rate gives the tokens), and the store, + * not this process, keeps the order. */ readonly store?: RateLimitStore; } @@ -108,6 +120,7 @@ export interface ResolvedRateLimitConfig { readonly key?: (message: Message | RawMessage) => string; readonly cost?: (message: Message | RawMessage) => number; readonly maxKeys: number; + readonly maxQueue: number; readonly now: () => number; } diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index 66d1b6b..b34fe33 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -8,13 +8,15 @@ // would hold it longer, where unemail started counting only at its turn; a // cost over the capacity is refused, where unemail cut it to the capacity and // sent more than the limit; the buckets of the keys are held to `maxKeys`, the -// least used dropped first; and disposal stops the waits. +// least used full one dropped first; the waiters of a bucket are held to +// `maxQueue`; and disposal stops the waits. const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport } = require('../core/transport.js'); const { refuseRaw } = require('../internal/capabilities.js'); const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); +const { warnLimited } = require('../internal/warn.js'); const { createLog } = require('../internal/log.js'); const { abortReason, sleep } = require('../internal/sleep.js'); const { createRateLimitConfig } = require('./config.js'); @@ -116,6 +118,9 @@ class RateLimitTransport { // or the receipt of a refusal. #admit(message, signal) { if (this.#disposed) return Promise.reject(disposedError()); + // A signal that aborted between two messages of a batch: the message + // would take its tokens and go, though the caller stopped. + if (signal?.aborted) return Promise.reject(abortReason(signal)); const { cost: costOf, key: keyOf } = this.config; const cost = costOf === undefined ? 1 : costOf(message); if (typeof cost !== 'number' || !(cost >= 0) || cost === Infinity) { @@ -151,8 +156,16 @@ class RateLimitTransport { { category: 'rate-limit', retryable: true, retryAfterMilliseconds: waitMs }, ); } + // However long the waits may be, the queue of a bucket is bounded. + if (bucket.waiters.size >= this.config.maxQueue) { + this.#log.write('info', () => ({ code: 'ratelimit.limited', waitMs }), 'A send was refused.'); + return this.#refused( + `The local rate limit of ${this.id} holds ${this.config.maxQueue} messages waiting already.`, + { category: 'rate-limit', retryable: true, retryAfterMilliseconds: waitMs }, + ); + } this.#log.write('debug', () => ({ code: 'ratelimit.waiting', waitMs }), 'A send waits.'); - return this.#wait(bucket, cost, signal); + return this.#waited(this.#wait(bucket, cost, signal)); } // Takes the tokens from the store the processes share, waiting while it @@ -164,8 +177,17 @@ class RateLimitTransport { let waited = 0; for (;;) { const answer = await this.config.store.take(key, cost, { limit, intervalMs, burst }); - if (answer?.taken === true) return undefined; - const waitMs = Math.max(1, Math.ceil(Number(answer?.waitMs) || 1)); + if (answer?.taken === true) { + if (waited > 0) this.#logWaited(waited); + return undefined; + } + // A store that does not say how long is asked again after the time the + // rate gives the tokens, not polled at every millisecond. + const told = Number(answer?.waitMs); + const waitMs = Math.max( + 1, + Math.ceil(told > 0 && Number.isFinite(told) ? told : cost / this.#refillPerMs), + ); if (this.config.mode === 'reject' || waited + waitMs > this.config.maxWaitMs) { this.#log.write( 'info', @@ -184,6 +206,18 @@ class RateLimitTransport { } } + // Tells the time a send waited for its tokens, once it has them. + async #waited(waiting) { + const started = this.config.now(); + await waiting; + this.#logWaited(this.config.now() - started); + return undefined; + } + + #logWaited(waitedMs) { + this.#log.write('debug', () => ({ code: 'ratelimit.waited', waitedMs }), 'A send waited.'); + } + #refused(text, options) { return Promise.resolve( createFailedReceipt(text, { @@ -270,14 +304,37 @@ class RateLimitTransport { return bucket; } - // Makes room for a key: drops the least used buckets that have no waiter, - // until there is room. One that is not full starts full when its key comes - // back, the price of holding no more than `maxKeys` keys; a bucket with - // waiters is never dropped, so they alone can hold more. + // Makes room for a key: drops the least used buckets that are full, which + // lose nothing, as a key that comes back starts full; then, if no full one + // makes room, the least used without a waiter, whose key comes back with + // more tokens than it had, which is told. A bucket with waiters is never + // dropped, so they alone can hold more than `maxKeys`. #evict() { for (const [key, bucket] of this.#buckets) { if (this.#buckets.size < this.config.maxKeys) return; - if (bucket.waiters.size === 0) this.#buckets.delete(key); + if (bucket.waiters.size > 0) continue; + this.#refill(bucket); + if (bucket.tokens >= this.config.burst) this.#buckets.delete(key); + } + let dropped = 0; + for (const [key, bucket] of this.#buckets) { + if (this.#buckets.size < this.config.maxKeys) break; + if (bucket.waiters.size === 0) { + this.#buckets.delete(key); + dropped++; + } + } + if (dropped > 0) this.#tellEvicted(dropped); + } + + #tellEvicted(dropped) { + const text = + `The rate limit of ${this.id} dropped ${dropped} bucket(s) not yet full to hold ` + + `${this.config.maxKeys} keys: their keys start full again. Raise maxKeys, or use a store.`; + if (this.#log.hasLogger) { + this.#log.write('warn', () => ({ code: 'ratelimit.evicted', dropped }), text); + } else { + warnLimited('MAILER_RATELIMIT_EVICTED', text); } } } diff --git a/tests/unit/ratelimit.test.js b/tests/unit/ratelimit.test.js index 760b9b8..e1ece64 100644 --- a/tests/unit/ratelimit.test.js +++ b/tests/unit/ratelimit.test.js @@ -315,3 +315,114 @@ test('RateLimit: processes that share a store share one limit', async (t) => { /take method/, ); }); + +test('RateLimit: a full bucket is dropped before one that is not, which is told', async (t) => { + // Before: the least used bucket with no waiter was dropped, full or not, so + // a limited key rotated out came back full, silently. + const clock = timeline(t); + const warned = []; + t.mock.method(process, 'emitWarning', (_text, options) => warned.push(options?.code)); + const limiter = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 1000, + mode: 'reject', + maxKeys: 2, + key: (m) => m.recipients[0].address, + now: clock.now, + }); + const to = (address) => limiter.send(message('x', address)); + assert.strictEqual((await to('a@example.com')).successful, true); + await clock.advance(1000); + assert.strictEqual((await to('b@example.com')).successful, true); + // a has refilled and is the least used; b is not full: a goes, b stays. + assert.strictEqual((await to('c@example.com')).successful, true); + assert.strictEqual((await to('b@example.com')).successful, false); + assert.deepStrictEqual(warned, []); + // With no full bucket, the least used goes, and a warning says so. + assert.strictEqual((await to('d@example.com')).successful, true); + assert.deepStrictEqual(warned, ['MAILER_RATELIMIT_EVICTED']); +}); + +test('RateLimit: an abort between two messages stops the next though tokens are there', async () => { + // Before: only a message that had to wait heard the aborted signal; one + // whose tokens were there took them and went. + const inner = fakeTransport(); + const limiter = new RateLimitTransport(inner, { limit: 10 }); + const controller = new AbortController(); + const receipts = limiter.sendMany([message('a'), message('b')], { signal: controller.signal }); + const iterator = receipts[Symbol.asyncIterator](); + assert.strictEqual((await iterator.next()).value.successful, true); + controller.abort(new Error('stop')); + await assert.rejects(iterator.next(), /stop/); + assert.strictEqual(inner.calls.length, 1); +}); + +test('RateLimit: maxQueue bounds the waiters, whatever maxWaitMs allows', async (t) => { + // Before: with maxWaitMs Infinity, any number of messages waited. + const clock = timeline(t); + const limiter = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 1000, + maxWaitMs: Infinity, + maxQueue: 1, + now: clock.now, + }); + assert.strictEqual((await limiter.send(message('first'))).successful, true); + const waiting = limiter.send(message('second')); + const refused = await limiter.send(message('third')); + assert.strictEqual(refused.successful, false); + assert.strictEqual(refused.errors[0].category, 'rate-limit'); + assert.match(refused.errorMessages[0], /holds 1 messages waiting/); + await clock.advance(1000); + assert.strictEqual((await waiting).successful, true); + assert.throws( + () => new RateLimitTransport(fakeTransport(), { limit: 1, maxQueue: 0 }), + /maxQueue/, + ); +}); + +test('RateLimit: a store that gives no wait is asked again after the rate, and waits are told', async (t) => { + // Before: a store answering { taken: false } alone was asked again every + // millisecond, a thousand round trips a second for each waiting message. + const clock = timeline(t); + let takes = 0; + const records = []; + const logger = { debug: (record) => records.push(record), info() {}, warn() {} }; + const limiter = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 1000, + now: clock.now, + logger, + store: { take: () => ({ taken: ++takes > 2 }) }, + }); + const sending = limiter.send(message()); + // Lets the answer of the store come, and its wait start, before the clock moves. + await clock.advance(0); + await clock.advance(999); + assert.strictEqual(takes, 1); + await clock.advance(1); + await clock.advance(1000); + assert.strictEqual((await sending).successful, true); + assert.strictEqual(takes, 3); + const waited = records.find((record) => record.code === 'ratelimit.waited'); + assert.strictEqual(waited.waitedMs, 2000); + // A local wait is told once its tokens are there. + const local = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 1000, + now: clock.now, + logger, + }); + records.length = 0; + await local.send(message()); + const second = local.send(message()); + await clock.advance(1000); + await second; + assert.deepStrictEqual( + records.map((record) => [record.code, record.waitedMs ?? record.waitMs]), + [ + ['ratelimit.waiting', 1000], + ['ratelimit.waited', 1000], + ], + ); +}); From edac8b816b700e3ff7a06d21abbcdb51e45d6188 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:25:38 +0300 Subject: [PATCH 108/143] fix(sendmail): a process that ignores the SIGINT of a broken message is killed A message that could not be read sent SIGINT alone, so with timeout 0 or Infinity a process that ignored it never ended and held its place among maxConcurrent. It gets SIGKILL after the grace, as an abort or a timeout does after SIGTERM. Co-Authored-By: Claude Opus 5.5 --- src/sendmail/sendmail-transport.js | 10 +++--- tests/unit/sendmail.test.js | 51 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index fe0530f..942ea42 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -141,8 +141,8 @@ const collectStderr = (stream) => { }; // Asks the process to stop, and makes it if it does not within the grace. -const terminate = (child) => { - child.kill('SIGTERM'); +const terminate = (child, signal = 'SIGTERM') => { + child.kill(signal); const timer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS); timer.unref?.(); child.once('close', () => clearTimeout(timer)); @@ -188,9 +188,11 @@ const run = (child, chunks, newline, abort, timeout) => return written; }, (error) => { - // The message could not be read: sendmail must not deliver a part. + // The message could not be read: sendmail must not deliver a part, and + // one that does not stop on SIGINT is killed after the grace, so that + // it holds no place among maxConcurrent with no timeout to end it. outcome.contentError = error; - child.kill('SIGINT'); + terminate(child, 'SIGINT'); }, ); }); diff --git a/tests/unit/sendmail.test.js b/tests/unit/sendmail.test.js index 8b6653b..607b667 100644 --- a/tests/unit/sendmail.test.js +++ b/tests/unit/sendmail.test.js @@ -430,6 +430,57 @@ test('Sendmail: a process that ignores SIGTERM gets SIGKILL after a grace', asyn assert.deepStrictEqual(children[0].killed, ['SIGTERM', 'SIGKILL']); }); +test('Sendmail: a process that ignores the SIGINT of a broken message gets SIGKILL', async (t) => { + // Before: a message that could not be read sent SIGINT alone, so with no + // timeout a process that ignored it held its place for good. + t.mock.timers.enable({ apis: ['setTimeout'] }); + t.after(() => t.mock.timers.reset()); + let child; + const spawn = () => { + child = new EventEmitter(); + child.stdin = new PassThrough(); + child.stderr = new PassThrough(); + child.stdin.resume(); + child.killed = []; + child.kill = (signal) => { + child.killed.push(signal); + if (signal === 'SIGKILL') process.nextTick(() => child.emit('close', null, signal)); + return true; + }; + return child; + }; + const broken = createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 'Broken', + content: { text: 'x' }, + attachments: { + filename: 'a.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: async function* () { + yield new Uint8Array(10); + throw new Error('disk gone'); + }, + }, + }); + const sending = new SendmailTransport({ spawn, timeout: 0 }).send(broken); + // The composition yields with setTimeout(0), which the mocked clock runs; + // setImmediate is not waited for, as a second test of a file that mocks + // the timers does not get it on Node 24. + for (let i = 0; i < 10_000 && child?.killed.length !== 1; i++) { + t.mock.timers.tick(0); + await new Promise((resolve) => process.nextTick(resolve)); + } + assert.deepStrictEqual(child.killed, ['SIGINT']); + t.mock.timers.tick(5000); + const receipt = await sending; + assert.deepStrictEqual(child.killed, ['SIGINT', 'SIGKILL']); + assert.strictEqual(receipt.successful, false); + assert.match(receipt.errorMessages[0], /disk gone/); +}); + test('Sendmail: an exit with 0 before the whole message was read is not a success', async () => { // Before: code 0 was success whatever had been written. const spawn = () => { From 28710ecd6d2b1216a534f4a0856e065d6acf89f9 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:26:32 +0300 Subject: [PATCH 109/143] fix(tee): the mirrors start once the primary has answered The copies started before the primary's send, so a mirror could run ahead of the provider in use, which the plan of the review called out. They start once the primary has answered, whatever it answered; a primary that throws is not mirrored. Not awaited by default, they add nothing to the caller's wait. Co-Authored-By: Claude Opus 5.5 --- docs/transports/tee.md | 12 +++++++----- src/tee/index.d.ts | 5 +++-- src/tee/tee-transport.js | 21 +++++++++------------ tests/unit/tee.test.js | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/docs/transports/tee.md b/docs/transports/tee.md index a2d8416..cc525c4 100644 --- a/docs/transports/tee.md +++ b/docs/transports/tee.md @@ -30,10 +30,12 @@ timeout, and follow the caller's signal. Disposing of the tee waits for the copies in flight before it disposes of the mirrors. The primary and the mirrors send one Message-ID, so a copy reads as the message it is. -The copies start with the primary's send, not after it: a mirror receives the -message whether or not the primary takes it, so a copy may go out for a -message the primary refused. A shadow that should see only what the primary -sent belongs after the send, in `afterSend` of [`./hooks`](/transports/hooks). +The copies start once the primary has answered, so a mirror never runs ahead +of the provider in use, and, as they are not awaited by default, the caller +waits no longer for them. A message is mirrored whatever the primary answered, +a refusal included, which a shadow comparing providers wants; a primary that +throws is not mirrored. A copy only of what the primary sent belongs in +`afterSend` of [`./hooks`](/transports/hooks), which sees the receipt. `onMirror` hears the outcome of every copy: its `receipt`, the `error` its send threw, or why it was `skipped`. Without it, a failed or skipped copy becomes a @@ -53,6 +55,6 @@ were left out. | `logger` | | A pino-like logger: without `onMirror`, a failed copy at `warn`, in place of a process warning. | `sendMany()` goes through the `sendMany()` of the primary, each message mirrored -as it is handed to it. `sendRaw()` mirrors to the mirrors that send raw +once the primary has answered it. `sendRaw()` mirrors to the mirrors that send raw messages and reports the others as skipped. `verify()` checks the primary alone, and disposal disposes of the primary and every mirror once. diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts index 1546809..2908b17 100644 --- a/src/tee/index.d.ts +++ b/src/tee/index.d.ts @@ -107,8 +107,9 @@ export declare class TeeTransport */ constructor(primary: Transport, config: TeeConfig); /** - * Sends a message through the primary and mirrors it; the copies start - * with the primary's send, whatever the primary answers. + * Sends a message through the primary, then mirrors it once the primary + * has answered, whatever it answered; a primary that throws is not + * mirrored. */ send(message: Message, options?: TransportOptions): Promise>; /** diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 4d7aded..8c3f06c 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -67,7 +67,9 @@ class TeeTransport { } /** - * Sends a message through the primary and mirrors it. + * Sends a message through the primary, then mirrors it once the primary + * has answered, whatever it answered; a primary that throws is not + * mirrored. * * @returns The receipt of the primary. * @throws What the primary throws. @@ -76,9 +78,8 @@ class TeeTransport { options?.signal?.throwIfAborted(); // The copies are of one message, so they carry one Message-ID. const pinned = pinMessageId(message); - const mirrored = this.#mirror(pinned, options?.signal, 'send'); const receipt = await this.primary.send(pinned, options); - await mirrored; + await this.#mirror(pinned, options?.signal, 'send'); return receipt; } @@ -90,24 +91,20 @@ class TeeTransport { async sendRaw(message, options) { options?.signal?.throwIfAborted(); if (!isRawTransport(this.primary)) return refuseRaw(this.primary); - const mirrored = this.#mirror(message, options?.signal, 'sendRaw'); const receipt = await this.primary.sendRaw(message, options); - await mirrored; + await this.#mirror(message, options?.signal, 'sendRaw'); return receipt; } /** - * Sends messages through the `sendMany()` of the primary, each mirrored as - * it is handed to it. + * Sends messages through the `sendMany()` of the primary, each mirrored + * once the primary has answered it. */ sendMany(messages, options) { return gatedSendMany(this.primary, messages, options, { - admit: (message) => { - const pinned = pinMessageId(message); - return { message: pinned, mirrored: this.#mirror(pinned, options?.signal, 'send') }; - }, + admit: (message) => ({ message: pinMessageId(message) }), settle: async (receipt, admitted) => { - await admitted.mirrored; + await this.#mirror(admitted.message, options?.signal, 'send'); return receipt; }, }); diff --git a/tests/unit/tee.test.js b/tests/unit/tee.test.js index 7265e0f..eb82ba0 100644 --- a/tests/unit/tee.test.js +++ b/tests/unit/tee.test.js @@ -284,3 +284,39 @@ test('Tee: a failed copy is a warning with no address, and a flood of them is cu assert.strictEqual(warnings.length, 11); assert.match(warnings[10][0], /2 more warnings of MAILER_TEE_MIRROR_FAILED were left out/); }); + +test('Tee: the mirrors start once the primary has answered, and not when it throws', async () => { + // Before: the copies started before the primary's send, so a mirror could + // run ahead of the provider in use. + const order = []; + const primary = fakeTransport(async (m) => { + order.push(`primary ${m.subject}`); + await settle(); + order.push(`answered ${m.subject}`); + return sent('p'); + }); + const mirror = fakeTransport((m) => { + order.push(`mirror ${m.subject}`); + return sent('m'); + }); + const tee = new TeeTransport(primary, { mirrors: [mirror], awaitMirrors: true }); + await tee.send(message('one')); + await collect(tee.sendMany([message('two')])); + assert.deepStrictEqual(order, [ + 'primary one', + 'answered one', + 'mirror one', + 'primary two', + 'answered two', + 'mirror two', + ]); + const throwing = new TeeTransport( + fakeTransport(() => { + throw new Error('bug'); + }), + { mirrors: [mirror], awaitMirrors: true }, + ); + order.length = 0; + await assert.rejects(throwing.send(message('three')), /bug/); + assert.deepStrictEqual(order, []); +}); From fa38834e8feda41a3a730d52e636faca521f67b6 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:27:52 +0300 Subject: [PATCH 110/143] fix(hooks): a message a beforeSend returns is checked as createMessage checks it A copy of the message with fields changed ({ ...message, recipients }) kept the shape the hook checked, so a line break in a display name or an address, or in the content type of an attachment, reached the transports that rely on createMessage refusing it. checkMessage of src/core/message.js runs the checks of createMessage on a message already made, and the hooks refuse one that fails them. Co-Authored-By: Claude Opus 5.5 --- src/core/message.js | 41 +++++++++++++++++++++++++++++++++++- src/hooks/hooks-transport.js | 14 ++++++++++++ tests/unit/hooks.test.js | 36 +++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/core/message.js b/src/core/message.js index b6f0098..e50d69e 100644 --- a/src/core/message.js +++ b/src/core/message.js @@ -294,4 +294,43 @@ const ensureArray = (value) => { return value == null ? [] : [value]; }; -module.exports = { checkAttachment, createMessage }; +// The lists of addresses of a message and how its errors name them. +const ADDRESS_LISTS = [ + ['recipients', 'recipient'], + ['ccRecipients', 'CC'], + ['bccRecipients', 'BCC'], + ['replyRecipients', 'reply-to'], +]; + +/** + * Checks again what `createMessage` checks of a message already made, such + * as one a hook of `./hooks` gives in place of another: a copy with fields + * changed (`{ ...message, to }`) skips `createMessage`, and the transports + * that rely on it would write what it refuses. + * + * @param message The message. + * @throws {TypeError} If a field is one `createMessage` refuses. + * @internal + */ +const checkMessage = (message) => { + checkAddress('sender', message.sender); + for (const [field, label] of ADDRESS_LISTS) { + if (!Array.isArray(message[field])) throwTypeError(`Invalid ${field}: it must be an array.`); + for (const address of message[field]) checkAddress(label, address); + } + if (!Array.isArray(message.attachments)) throwTypeError('Invalid attachments: not an array.'); + for (const attachment of message.attachments) { + if (!isAttachment(attachment)) throwTypeError('Invalid attachment.'); + checkAttachment(attachment); + } + if (!(message.headers instanceof Headers)) throwTypeError('Invalid headers: not a Headers.'); + if (message.messageId != null) checkMessageId('', message.messageId); + checkMessageIds('in-reply-to ', message.inReplyTo); + checkMessageIds('references ', message.references); + if (message.date !== undefined) checkDate(message.date); + checkScheduledAt(message.scheduledAt); + checkTracking(message.tracking); + checkMetadata(message.metadata); +}; + +module.exports = { checkAttachment, checkMessage, createMessage }; diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js index e7126d5..5d1b18b 100644 --- a/src/hooks/hooks-transport.js +++ b/src/hooks/hooks-transport.js @@ -12,6 +12,7 @@ // production and dropped it in production; and `beforeSend` may replace the // message or refuse it. +const { checkMessage } = require('../core/message.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); @@ -161,6 +162,19 @@ class HooksTransport { receipt: this.#refusal(`The ${name} hook returned neither a message nor nothing.`), }; } + // A copy of the message with fields changed did not go through + // createMessage, which the transports rely on: its checks run again. + if (name !== 'beforeSendRaw') { + try { + checkMessage(replaced); + } catch (error) { + return { + receipt: this.#refusal( + `The ${name} hook returned an invalid message: ${messageOf(error)}`, + ), + }; + } + } return { message: replaced }; } diff --git a/tests/unit/hooks.test.js b/tests/unit/hooks.test.js index b2c21cc..9ede26a 100644 --- a/tests/unit/hooks.test.js +++ b/tests/unit/hooks.test.js @@ -262,6 +262,42 @@ test('Hooks: a beforeSend that returns something else than a message refuses it' assert.strictEqual(inner.calls.length, 0); }); +test('Hooks: a copy a beforeSend returns is checked again as createMessage checks it', async () => { + // Before: a copy with fields changed kept the shape of a message, so a + // line break in a name or an address, or in the content type of an + // attachment, reached transports that rely on createMessage refusing it. + const inner = fakeTransport(); + const copies = [ + (m) => ({ ...m, sender: { name: 'Eve\r\nBcc: x@evil.example', address: 'a@example.com' } }), + (m) => ({ ...m, recipients: [{ address: 'b@example.com\r\nBcc: x@evil.example' }] }), + (m) => ({ + ...m, + attachments: [ + { + filename: 'a.txt', + contentType: 'text/plain\r\nX-Evil: 1', + content: new Uint8Array(1), + inline: false, + contentId: '', + }, + ], + }), + (m) => ({ ...m, headers: { 'X-Plain': 'object' } }), + (m) => ({ ...m, metadata: { 'bad key': 'x' } }), + ]; + for (const copy of copies) { + const hooks = new HooksTransport(inner, { beforeSend: copy }); + const receipt = await hooks.send(message()); + assert.strictEqual(receipt.errors[0].code, 'hooks.refused', String(copy)); + assert.match(receipt.errorMessages[0], /returned an invalid message/); + } + assert.strictEqual(inner.calls.length, 0); + // A sound copy goes. + const fine = new HooksTransport(inner, { beforeSend: (m) => ({ ...m, subject: 'Changed' }) }); + assert.strictEqual((await fine.send(message())).successful, true); + assert.strictEqual(inner.calls[0].message.subject, 'Changed'); +}); + test('Hooks: an onRetry hook whose promise rejects is a warning', async (t) => { // Before: the rejection of an async onRetry hook went unhandled. const warnings = []; From cdadd4a83533ee32fb8465f3aea59cbb855ff13d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:29:58 +0300 Subject: [PATCH 111/143] fix(webhooks): SNS certificates cached by their one spelling, once verified The cache of signing certificates was keyed by the URL as written, so SNS in capitals, :443, an empty query or fragment, or a tab made a new read of one certificate each; and a certificate read for a forged message was kept though its signature failed, so forged messages naming other certificates pushed the one in use out. A certificate URL is taken only as the URL parser writes it, with no ? or #, and a certificate is kept only once it has verified a signature; those in flight are still read once. Co-Authored-By: Claude Opus 5.5 --- src/webhooks/sns.js | 75 +++++++++++++++++++-------------- tests/unit/webhooks-ses.test.js | 46 ++++++++++++++++++++ 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index a43690a..0c965f2 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -111,39 +111,43 @@ const loadCertificate = async (fetchOption, url, timeoutMs) => { } }; -// The certificates by URL: each read once while it is cached, the oldest -// dropped past MAX_CERTIFICATES, one that failed to load not kept, and no -// more than MAX_LOADING read at once. +// The certificates by URL. Only one that has verified a signature is kept, +// the oldest dropped past MAX_CERTIFICATES, so that forged messages naming +// other certificates of the service cannot push the one in use out; one is +// read at most once while it loads, and no more than MAX_LOADING at once. const createCertificateCache = (fetchOption, trust, timeoutMs) => { - const certificates = new Map(); - let loading = 0; - return (url) => { - let pending = certificates.get(url); - if (pending !== undefined) return pending; - if (loading >= MAX_LOADING) { - return Promise.reject( - certificateError('Too many signing certificates are being read.', undefined, UNREAD), - ); - } - loading++; - pending = loadCertificate(fetchOption, url, timeoutMs) - .then((certificate) => { - trust(certificate); - return certificate; - }) - .finally(() => { - loading--; - }); - pending.catch(() => certificates.delete(url)); - certificates.set(url, pending); - if (certificates.size > MAX_CERTIFICATES) { - certificates.delete(certificates.keys().next().value); - } - return pending; + const verified = new Map(); + const loading = new Map(); + return { + get(url) { + const known = verified.get(url) ?? loading.get(url); + if (known !== undefined) return known; + if (loading.size >= MAX_LOADING) { + return Promise.reject( + certificateError('Too many signing certificates are being read.', undefined, UNREAD), + ); + } + const pending = loadCertificate(fetchOption, url, timeoutMs) + .then((certificate) => { + trust(certificate); + return certificate; + }) + .finally(() => { + loading.delete(url); + }); + loading.set(url, pending); + return pending; + }, + // Keeps a certificate whose signature of a message verified. + keep(url, certificate) { + if (verified.has(url)) return; + verified.set(url, Promise.resolve(certificate)); + if (verified.size > MAX_CERTIFICATES) verified.delete(verified.keys().next().value); + }, }; }; -const checkSignature = async (message, certificateOf, service, algorithms, common) => { +const checkSignature = async (message, certificates, service, algorithms, common) => { const fields = SIGNED_FIELDS.get(message.Type); if (fields === undefined) { throw new WebhookVerificationError( @@ -162,8 +166,14 @@ const checkSignature = async (message, certificateOf, service, algorithms, commo if (typeof url !== 'string' || !service.isCertificateUrl(url)) { throw certificateError('The signing certificate is not on a host of the service.'); } + // The URL is the key of the cache, so it is taken only as the URL parser + // writes it: another spelling of it (`SNS.` in capitals, `:443`, an empty + // `?` or `#`, a tab) would read the certificate again, and push others out. + if (new URL(url).href !== url || url.includes('?') || url.includes('#')) { + throw certificateError('The signing certificate URL is not written as a URL is.'); + } const text = stringToSign(message, fields); - const certificate = await certificateOf(message.SigningCertURL); + const certificate = await certificates.get(url); const now = common.now(); if (now < Date.parse(certificate.validFrom) || now > Date.parse(certificate.validTo)) { throw certificateError('The signing certificate is not valid now.'); @@ -172,6 +182,7 @@ const checkSignature = async (message, certificateOf, service, algorithms, commo if (!verifySignature(algorithm, Buffer.from(text, 'utf8'), certificate.publicKey, signature)) { throw new WebhookVerificationError('signature', 'The SNS signature does not match.'); } + certificates.keep(url, certificate); }; // The algorithms of the signature versions taken: both by default, as a @@ -222,7 +233,7 @@ const createSnsVerifier = (options, service) => { const topics = topicsOf(options.topicArns); const common = resolveCommonOptions(options, service.defaultToleranceMs); const algorithms = algorithmsOf(options.signatureVersions); - const certificateOf = createCertificateCache( + const certificates = createCertificateCache( checkOptionalFunction(options.fetch, 'fetch'), service.trust ?? (() => {}), checkPositiveNumber( @@ -238,7 +249,7 @@ const createSnsVerifier = (options, service) => { if (!topics.has(message.TopicArn)) { throw new WebhookVerificationError('topic', `The topic ${message.TopicArn} is not taken.`); } - await checkSignature(message, certificateOf, service, algorithms, common); + await checkSignature(message, certificates, service, algorithms, common); checkTime(Date.parse(message.Timestamp), common); const key = `${service.provider}:${message.MessageId}`; if (message.Type !== 'Notification') { diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js index 2d43023..75ca67d 100644 --- a/tests/unit/webhooks-ses.test.js +++ b/tests/unit/webhooks-ses.test.js @@ -399,6 +399,52 @@ test('Webhooks SES: a certificate URL off the path of SNS, or with a query, is n assert.strictEqual(requests.length, 0); }); +test('Webhooks SES: another spelling of the certificate URL is not read', async () => { + // Before: the cache was keyed by the URL as written, so SNS in capitals, + // :443, an empty query or a tab made a new read of one certificate each. + const { verify, requests } = verifier(); + for (const url of [ + 'https://SNS.eu-west-1.amazonaws.com/SimpleNotificationService-abc.pem', + 'https://sns.eu-west-1.amazonaws.com:443/SimpleNotificationService-abc.pem', + `${CERT_URL}?`, + `${CERT_URL}#`, + 'https://sns.eu-west-1.amazon\taws.com/SimpleNotificationService-abc.pem', + 'https:sns.eu-west-1.amazonaws.com/SimpleNotificationService-abc.pem', + 'https://sns.eu-west-1.amazonaws.com/./SimpleNotificationService-abc.pem', + ]) { + const message = notification({ eventType: 'Send', mail: mail() }, { SigningCertURL: url }); + await rejects(verify(request(message)), 'certificate'); + } + assert.strictEqual(requests.length, 0); +}); + +test('Webhooks SES: a certificate is kept only once it has verified a signature', async () => { + // Before: a certificate read for a forged message was cached though its + // signature failed, so forged messages naming other certificates of SNS + // pushed the one in use out of the cache, and each made a read. + const { verify, requests } = verifier(); + const good = notification({ eventType: 'Send', mail: mail() }); + await verify(request(good)); + assert.strictEqual(requests.length, 1); + for (let i = 0; i < 40; i++) { + const forged = { + ...notification( + { eventType: 'Send', mail: mail() }, + { + MessageId: `forged-${i}`, + SigningCertURL: `https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-f${i}.pem`, + }, + ), + Signature: Buffer.from('forged').toString('base64'), + }; + await rejects(verify(request(forged)), 'signature'); + } + const again = notification({ eventType: 'Send', mail: mail() }, { MessageId: 'sns-2' }); + await verify(request(again)); + // The forged ones made their reads; the real certificate was not read again. + assert.strictEqual(requests.filter((entry) => entry.url === CERT_URL).length, 1); +}); + test('Webhooks SES: a few certificates are read at a time, the next one is asked again', async () => { const waiting = []; const fetch = () => From 880a9aa2665b756d4a136eb7881390fbbecf741a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:32:55 +0300 Subject: [PATCH 112/143] fix(webhooks): Scaleway anchors are CAs valid when trusted; the rule of mapping - Scaleway: any certificate of the pinned trust chain issued, a CA or not, valid or not, so a pinned leaf let what it signed through from any Scaleway host, an Object Storage bucket included. A chain with a certificate that is no CA is refused when the verifier is made, and an authority issues only while it is valid; trust() of the SNS service is given the time. Scaleway documents no form of its certificate URLs, so the host stays what it was. - SparkPost: the bounce classes 25 and 26, SparkPost's own refusals, were soft bounces and are rejections; 60, 80 and 100, no failure, are unknown; 90 is an unsubscribe, not a hard bounce. - Maileroo: a failure was hard though Maileroo does not say it is permanent; it is undetermined. Maileroo and Scaleway: a deferral carries a soft bounce only when it gives its code or reason. The fixtures of the expired CA and its leaf are made with OpenSSL 3.6, as their README says. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 4 +- src/webhooks/index.d.ts | 3 +- src/webhooks/maileroo.js | 10 +++-- src/webhooks/scaleway.js | 40 ++++++++++++----- src/webhooks/sns.js | 7 +-- src/webhooks/sparkpost.js | 43 ++++++++++++++----- tests/fixtures/webhooks/README.md | 21 +++++++++ .../webhooks/scaleway-cert-of-expired-ca.pem | 19 ++++++++ .../fixtures/webhooks/scaleway-expired-ca.pem | 20 +++++++++ tests/unit/webhooks-mapping.test.js | 33 +++++++++----- tests/unit/webhooks-others.test.js | 41 +++++++++++++++++- 11 files changed, 201 insertions(+), 40 deletions(-) create mode 100644 tests/fixtures/webhooks/scaleway-cert-of-expired-ca.pem create mode 100644 tests/fixtures/webhooks/scaleway-expired-ca.pem diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index d7d4194..74f8ca5 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -392,7 +392,9 @@ messages, verified as those of SES are, from a listed topic, within an hour and once, but with a certificate on a Scaleway host that must have been issued by Scaleway's SNS trust chain. Download that chain once, from `https://messaging.s3.fr-par.scw.cloud/fr-par/sns/sns-trust-chain.pem`, and pass -it as `trustChain`: the verifier trusts no certificate the chain did not issue. +it as `trustChain`: the verifier trusts no certificate the chain did not issue, +refuses a chain that holds a certificate that is no CA, and takes an authority +of it only while it is valid. A subscription confirmation is handed back, as for SES. ## Providers without a verifier diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index b0d1513..da0cb89 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -470,7 +470,8 @@ export interface ScalewayWebhookVerifierOptions extends SnsWebhookVerifierOption readonly topicArns: readonly string[]; /** * The PEM of Scaleway's SNS trust chain, which issued the certificate a - * message is signed with. + * message is signed with: each certificate of it a CA, or the verifier is + * not made, and one valid when it is trusted. */ readonly trustChain: string; /** diff --git a/src/webhooks/maileroo.js b/src/webhooks/maileroo.js index 5cb3157..aa4c83c 100644 --- a/src/webhooks/maileroo.js +++ b/src/webhooks/maileroo.js @@ -36,11 +36,13 @@ const eventsOf = (payload) => { const data = isRecord(payload.event_data) ? payload.event_data : {}; const type = TYPES.get(payload.event_type) ?? 'unknown'; const reason = stringOf(data.reason) ?? stringOf(data.reject_reason); + // Maileroo says neither whether a failure is permanent nor whether a + // deferral is a bounce: a failure is of a kind not said, and a deferral + // carries a soft bounce only when it gives its reason. let bounce; - if (type === 'bounced' || type === 'deferred') { - bounce = { kind: type === 'bounced' ? 'hard' : 'soft' }; - if (reason !== undefined) bounce.diagnostic = reason; - } + if (type === 'bounced') bounce = { kind: 'undetermined' }; + else if (type === 'deferred' && reason !== undefined) bounce = { kind: 'soft' }; + if (bounce !== undefined && reason !== undefined) bounce.diagnostic = reason; const { tags, metadata } = splitNamedTags(payload.tags); const recipients = stringListOf(data.to); const list = recipients.length > 0 ? recipients : [undefined]; diff --git a/src/webhooks/scaleway.js b/src/webhooks/scaleway.js index faa80d1..9ab0379 100644 --- a/src/webhooks/scaleway.js +++ b/src/webhooks/scaleway.js @@ -35,21 +35,33 @@ const trustChainOf = (pem) => { } const authorities = []; for (const match of pem.matchAll(PEM_CERTIFICATE)) { - authorities.push(new X509Certificate(match[0])); + const authority = new X509Certificate(match[0]); + // A certificate that is no authority issues none: pinned, it would let + // any certificate it signed, one served from any Scaleway host, through. + if (!authority.ca) { + throw new TypeError(`trustChain holds a certificate that is no CA: ${authority.subject}`); + } + authorities.push(authority); } if (authorities.length === 0) throw new TypeError('trustChain holds no certificate.'); return authorities; }; +const isValidAt = (certificate, now) => + now >= Date.parse(certificate.validFrom) && now <= Date.parse(certificate.validTo); + const eventOf = (record, snsMessageId) => { const type = TYPES.get(record.type) ?? 'unknown'; + // A deferral carries a soft bounce only when it gives its code or reason. + const code = record.email_response_code ?? undefined; + const diagnostic = stringOf(record.email_response_message) ?? stringOf(record.email_error); let bounce; - if (type === 'bounced' || type === 'deferred') { - bounce = { kind: type === 'bounced' ? 'hard' : 'soft' }; - if (record.email_response_code !== undefined && record.email_response_code !== null) { - bounce.code = String(record.email_response_code); - } - const diagnostic = stringOf(record.email_response_message) ?? stringOf(record.email_error); + if (type === 'bounced') bounce = { kind: 'hard' }; + else if (type === 'deferred' && (code !== undefined || diagnostic !== undefined)) { + bounce = { kind: 'soft' }; + } + if (bounce !== undefined) { + if (code !== undefined) bounce.code = String(code); if (diagnostic !== undefined) bounce.diagnostic = diagnostic; } return createEvent({ @@ -89,11 +101,19 @@ const createScalewayWebhookVerifier = (options = {}) => { // not the host, is what a certificate is taken by. isCertificateUrl: isScalewayUrl, isSubscribeUrl: isScalewayUrl, - trust: (certificate) => { + trust: (certificate, now) => { for (const authority of authorities) { - if (certificate.checkIssued(authority) && certificate.verify(authority.publicKey)) return; + if ( + isValidAt(authority, now) && + certificate.checkIssued(authority) && + certificate.verify(authority.publicKey) + ) { + return; + } } - throw certificateError('The signing certificate was not issued by the trust chain.'); + throw certificateError( + 'The signing certificate was not issued by an authority of the trust chain valid now.', + ); }, eventsOf: (message, record) => [ record === undefined diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index 0c965f2..7e32856 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -224,8 +224,8 @@ const topicsOf = (topicArns) => { * `certificateTimeoutMs`, and the options every verifier takes. * @param service `provider`, `defaultToleranceMs`, `isCertificateUrl` and * `isSubscribeUrl` (where its certificates and subscriptions are), - * `trust(certificate)`, which throws for a certificate the service did not - * issue, and `eventsOf(message, record)`, the events of a notification, + * `trust(certificate, now)`, which throws for a certificate the service did + * not issue, and `eventsOf(message, record)`, the events of a notification, * `record` its parsed message when it is a JSON object. * @internal */ @@ -233,9 +233,10 @@ const createSnsVerifier = (options, service) => { const topics = topicsOf(options.topicArns); const common = resolveCommonOptions(options, service.defaultToleranceMs); const algorithms = algorithmsOf(options.signatureVersions); + const trust = service.trust; const certificates = createCertificateCache( checkOptionalFunction(options.fetch, 'fetch'), - service.trust ?? (() => {}), + trust === undefined ? () => {} : (certificate) => trust(certificate, common.now()), checkPositiveNumber( options.certificateTimeoutMs ?? DEFAULT_CERTIFICATE_TIMEOUT_MS, 'certificateTimeoutMs', diff --git a/src/webhooks/sparkpost.js b/src/webhooks/sparkpost.js index 44206fd..8c40168 100644 --- a/src/webhooks/sparkpost.js +++ b/src/webhooks/sparkpost.js @@ -38,15 +38,32 @@ const TYPES = new Map([ ['click', 'clicked'], ['amp_click', 'clicked'], ]); -// SparkPost's bounce classes: 10 Invalid Recipient, 30 Generic Bounce (No -// RCPT) and 90 Unsubscribe are hard; 1 Undetermined; the others soft. -const HARD_CLASSES = new Set(['10', '30', '90']); +// SparkPost's bounce classes, read by the rule of the guide: 10 Invalid +// Recipient and 30 Generic Bounce (No RCPT) are hard; 25 Admin Failure and 26 +// Smart Send Suppression, which SparkPost failed by its own policy, nothing +// sent, are rejections; 60 Auto-Reply, 80 Subscribe and 100 +// Challenge-Response are no failure; 90 Unsubscribe is one; 1 Undetermined +// says nothing; the others (soft, blocks, transient) are soft. +const BOUNCE_CLASSES = new Map([ + ['1', { type: 'bounced', kind: 'undetermined' }], + ['10', { type: 'bounced', kind: 'hard' }], + ['30', { type: 'bounced', kind: 'hard' }], + ['25', { type: 'rejected' }], + ['26', { type: 'rejected' }], + ['60', { type: 'unknown' }], + ['80', { type: 'unknown' }], + ['100', { type: 'unknown' }], + ['90', { type: 'unsubscribed' }], +]); +const SOFT = { type: 'bounced', kind: 'soft' }; +const UNDETERMINED = { type: 'bounced', kind: 'undetermined' }; + +const classOf = (event) => + event.bounce_class === undefined + ? UNDETERMINED + : (BOUNCE_CLASSES.get(String(event.bounce_class)) ?? SOFT); -const bounceOf = (event) => { - const bounceClass = event.bounce_class === undefined ? undefined : String(event.bounce_class); - let kind = 'soft'; - if (bounceClass === undefined || bounceClass === '1') kind = 'undetermined'; - else if (HARD_CLASSES.has(bounceClass)) kind = 'hard'; +const bounceOf = (event, kind) => { const bounce = { kind }; if (stringOf(event.error_code) !== undefined) bounce.code = event.error_code; const diagnostic = stringOf(event.raw_reason) ?? stringOf(event.reason); @@ -66,7 +83,13 @@ const innerOf = (record) => { }; const eventOf = (inner, index) => { - const type = TYPES.get(inner.type) ?? 'unknown'; + let type = TYPES.get(inner.type) ?? 'unknown'; + let bounce; + if (type === 'bounced') { + const classed = classOf(inner); + type = classed.type; + if (classed.kind !== undefined) bounce = bounceOf(inner, classed.kind); + } return createEvent({ provider: 'sparkpost', type, @@ -75,7 +98,7 @@ const eventOf = (inner, index) => { messageId: stringOf(inner.transmission_id), recipient: stringOf(inner.rcpt_to) ?? stringOf(inner.raw_rcpt_to), occurredAt: dateOf(inner.timestamp), - bounce: type === 'bounced' ? bounceOf(inner) : undefined, + bounce, reason: stringOf(inner.reason) ?? stringOf(inner.raw_reason), url: stringOf(inner.target_link_url), tags: stringsOf(inner.rcpt_tags), diff --git a/tests/fixtures/webhooks/README.md b/tests/fixtures/webhooks/README.md index cd8f5fb..e43e593 100644 --- a/tests/fixtures/webhooks/README.md +++ b/tests/fixtures/webhooks/README.md @@ -12,3 +12,24 @@ without AWS. They were made with The private key is intentionally committed and must never be used outside the tests. + +Scaleway test certificates +========================== + +`scaleway-ca.pem` stands for Scaleway's SNS trust chain, and +`scaleway-cert.pem` with `scaleway-key.pem` for a certificate it issued. +`scaleway-expired-ca.pem` is a CA valid only in 2020, and +`scaleway-cert-of-expired-ca.pem` a certificate it issued for the same key, +valid until 2120, so that a test can tell a CA no longer valid from its leaf. +They were made with OpenSSL 3.6: + + openssl req -x509 -newkey rsa:2048 -nodes -keyout ca-key.pem \ + -out scaleway-expired-ca.pem -subj "/CN=Expired Test Scaleway CA" \ + -not_before 20200101000000Z -not_after 20210101000000Z \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign" + openssl req -new -key scaleway-key.pem -subj "/CN=sns.mnq.fr-par.scaleway.com" -out leaf.csr + openssl x509 -req -in leaf.csr -CA scaleway-expired-ca.pem -CAkey ca-key.pem \ + -set_serial 7 -not_before 20200101000000Z -not_after 21200101000000Z \ + -out scaleway-cert-of-expired-ca.pem + +The key of the expired CA was not kept. diff --git a/tests/fixtures/webhooks/scaleway-cert-of-expired-ca.pem b/tests/fixtures/webhooks/scaleway-cert-of-expired-ca.pem new file mode 100644 index 0000000..5936fb0 --- /dev/null +++ b/tests/fixtures/webhooks/scaleway-cert-of-expired-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCDCCAfCgAwIBAgIBBzANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhFeHBp +cmVkIFRlc3QgU2NhbGV3YXkgQ0EwIBcNMjAwMTAxMDAwMDAwWhgPMjEyMDAxMDEw +MDAwMDBaMCYxJDAiBgNVBAMMG3Nucy5tbnEuZnItcGFyLnNjYWxld2F5LmNvbTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJJ17C4CKK3IIPqlzWf5ZkRR +kR7f6zmVDBl/nXk2QNKdoTS6iC9698iby4+fm5s83Bgj3Y857rKGvSxokulDhoqH +S3XRO1j3y9y4JDS4QOTTNl8ihWyZi3uTulHtoMpYGYZWMq0qrC7Pgytp2U/eEl0d +dsIOqs/QXLfi8gYXJLx+UeVBCEvUMmDDc6dBoY+AWrosC6aFnKGHB6DfFwj3KJ4N +jB1YivVK99vvD7aiBlV/EWC6A9B8KtKyU4mzzskTm8DA2z5Xv5ScZ2fMM2eEthFQ +gRr2A66yeuewuFO9LMYgPM6qhASUtScd7lQJACwcRlF/73DJlsYXGbaKvMO4i/MC +AwEAAaNCMEAwHQYDVR0OBBYEFFCR+zMGwtzj6MZsRXxHpxsElPHUMB8GA1UdIwQY +MBaAFF9iuWveghrmau5r5Zx2JIzcCjmTMA0GCSqGSIb3DQEBCwUAA4IBAQCxX8Ri +yExxWgyKuQOZaRfL9xkIN/5HueaLzXChrej8tePnMzcPJaPrwitXJzPhR3PpOwsa +YKIFMxsDfEIHbMm36jhhOGIIJd6ElxCL+yivd8Y7gJ+LaG7nc7E3XSy3owR4oGBj +6ucpPhcKGRLdK52n/79vbeBX2YYhMS37dGmzWsZ/qYhpqIJ9qwErm0Put69Lqny2 +bJQYNNkCMXnAYBNv1UPupDdkl3ohURZ8VjdVFcptAqjnHfhHBm+g0R3DZwXQ+Su5 +TeItJ3IsZTWm0k7AeyDyWTd7hiMO0iLYxYQRZfzzY+db8/qagU65D7HhDI3xkhSL +mP9QuEbQYwy+QSzj +-----END CERTIFICATE----- diff --git a/tests/fixtures/webhooks/scaleway-expired-ca.pem b/tests/fixtures/webhooks/scaleway-expired-ca.pem new file mode 100644 index 0000000..cc860a9 --- /dev/null +++ b/tests/fixtures/webhooks/scaleway-expired-ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIUegd1KkK8IO9wv5EpyARVGLJw6qMwDQYJKoZIhvcNAQEL +BQAwIzEhMB8GA1UEAwwYRXhwaXJlZCBUZXN0IFNjYWxld2F5IENBMB4XDTIwMDEw +MTAwMDAwMFoXDTIxMDEwMTAwMDAwMFowIzEhMB8GA1UEAwwYRXhwaXJlZCBUZXN0 +IFNjYWxld2F5IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuwLD +wdIDaZaaOo9e70ZAW+GlBvOVr/cnp0Tr5ZAjrr5BRJa8G7resASznndbm+99ER9B +ctm94I6hSe+Cczx+NwjZaGe3Yx2yDGWgXqhFTYCynXSqZMZHrw710+FRVpTsGuVj +g1lELT3Ppj0l5HOTNcoozXHFie8qmJhhMBTrOHiIlVcqpkgO+HvnpOZrvJx9ttAS +7hppH5Noewhii6PdT3LV85BdJcENObqxSwbEN9HHuAAo+go5hKBwbSvHvsJzQWNn +FHtyXvxLtV2IO40cw81CD5b/kMCwOAE00nNRJtuDLT1UxSSyO4zj8nGnn0YnZyO6 +uVnjVZykwhu4vXOHIQIDAQABo2MwYTAdBgNVHQ4EFgQUX2K5a96CGuZq7mvlnHYk +jNwKOZMwHwYDVR0jBBgwFoAUX2K5a96CGuZq7mvlnHYkjNwKOZMwDwYDVR0TAQH/ +BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggEBAINK2RXs +naMRfkEnAbDVMkTtMre2pDuOcEIvGJhtSPSKilPLeNJojAbOyK+xl/1ljEIDUhDT +vLnrvr/g+1cNvGOgXyyRyo4FoAyasrU5aH8GUcFttyWc2Ra12TaN6878NtGUfCxA +H9ZYpCbukxdfTcYSYTGIegHDSPaY2BE/woIY0QUCULhjVQUGnOm+at+JXinpf/Fu +kN4M9Cgi6KCxrzoXbScI+e+i8J00nHw7KKJ4j3y4cVOvr/yDM5/1gtOh7QMkxoxP +R1jC+GGJexbZ659BrhJgtn5bNu+WoqMeX+SD4rKNXd7zj0Ed++T08wB+8ipZZmte +AhbtpA0kr7kwvn4= +-----END CERTIFICATE----- diff --git a/tests/unit/webhooks-mapping.test.js b/tests/unit/webhooks-mapping.test.js index 4ca64be..3543d70 100644 --- a/tests/unit/webhooks-mapping.test.js +++ b/tests/unit/webhooks-mapping.test.js @@ -251,12 +251,15 @@ test('Webhooks mapping: Mailgun, a failure by its severity and reason', async () assert.deepStrictEqual(actual, expectedOf(rows)); }); -test('Webhooks mapping: Maileroo, by event, a failure hard and a deferral soft', async () => { +test('Webhooks mapping: Maileroo, by event, a failure of a kind not said', async () => { + // Changed after the review: a failure was hard and a deferral soft though + // Maileroo says neither; a deferral with its reason carries a soft bounce. const rows = [ ['accepted', 'accepted'], ['delivered', 'delivered'], - ['deferred', 'deferred', 'soft'], - ['failed', 'bounced', 'hard'], + ['deferred', 'deferred'], + [{ event_type: 'deferred', reason: '421 try later' }, 'deferred', 'soft'], + ['failed', 'bounced', 'undetermined'], ['rejected', 'rejected'], ['complained', 'complained'], ['opened', 'opened'], @@ -264,13 +267,14 @@ test('Webhooks mapping: Maileroo, by event, a failure hard and a deferral soft', ['no_such_event', 'unknown'], ]; const verifier = createMailerooWebhookVerifier({ secret: 'mr-secret', replayStore: null }); - const actual = await eachOf(rows, (type, index) => { + const actual = await eachOf(rows, (input, index) => { + const fields = typeof input === 'string' ? { event_type: input } : input; const body = JSON.stringify({ event_id: `mr-${index}`, - event_type: type, + event_type: fields.event_type, event_time: 1_800_000_000, message_reference_id: 'ref-1', - event_data: { to: 'bob@example.com' }, + event_data: { to: 'bob@example.com', reason: fields.reason }, }); return verifier.verify({ body, @@ -487,7 +491,9 @@ test('Webhooks mapping: Scaleway, by type, a drop hard and a deferral soft', asy const rows = [ ['email_queued', 'accepted'], ['email_delivered', 'delivered'], - ['email_deferred', 'deferred', 'soft'], + // Changed after the review: a deferral carries a soft bounce only with its + // code or reason. + ['email_deferred', 'deferred'], ['email_dropped', 'bounced', 'hard'], ['email_mailbox_not_found', 'bounced', 'hard'], ['email_blocklisted', 'rejected'], @@ -628,11 +634,18 @@ test('Webhooks mapping: SparkPost, a bounce by its bounce class', async () => { [classed('delay', '20'), 'deferred'], [classed('bounce', '10'), 'bounced', 'hard'], [classed('bounce', 30), 'bounced', 'hard'], - [classed('bounce', '90'), 'bounced', 'hard'], + // Changed after the review: 90 was a hard bounce, 25 and 26 soft ones, 60, + // 80 and 100 soft ones, though they are an unsubscribe, refusals by + // SparkPost's policy, and no failure. + [classed('bounce', '90'), 'unsubscribed'], + [classed('bounce', '25'), 'rejected'], + [classed('bounce', '26'), 'rejected'], + [classed('bounce', '60'), 'unknown'], + [classed('bounce', '80'), 'unknown'], + [classed('bounce', '100'), 'unknown'], [classed('bounce', '20'), 'bounced', 'soft'], - [classed('bounce', '25'), 'bounced', 'soft'], [classed('bounce', '50'), 'bounced', 'soft'], - [classed('bounce', '100'), 'bounced', 'soft'], + [classed('bounce', '70'), 'bounced', 'soft'], [classed('bounce', '1'), 'bounced', 'undetermined'], ['bounce', 'bounced', 'undetermined'], [classed('out_of_band', '10'), 'bounced', 'hard'], diff --git a/tests/unit/webhooks-others.test.js b/tests/unit/webhooks-others.test.js index 9837cae..94f7429 100644 --- a/tests/unit/webhooks-others.test.js +++ b/tests/unit/webhooks-others.test.js @@ -113,6 +113,41 @@ test('Webhooks Scaleway: an SNS message signed by a certificate the trust chain assert.strictEqual(served.requests.length, 1); }); +test('Webhooks Scaleway: a pinned certificate must be a CA, valid when it is trusted', async () => { + // Before: any certificate of the trust chain issued, a CA or not, valid or + // not, so a pinned leaf let what it signed through from any Scaleway host. + const fixtures = require('node:path').join(__dirname, '..', 'fixtures', 'webhooks'); + const read = (name) => require('node:fs').readFileSync(`${fixtures}/${name}`, 'utf8'); + assert.throws( + () => + createScalewayWebhookVerifier({ + topicArns: [SCW_TOPIC], + trustChain: scaleway.certificate, + replayStore: null, + }), + /no CA/, + ); + // A CA valid in 2020, which issued a leaf valid until 2120. + const expired = (now) => + createScalewayWebhookVerifier({ + topicArns: [SCW_TOPIC], + trustChain: read('scaleway-expired-ca.pem'), + fetch: certificateFetch(read('scaleway-cert-of-expired-ca.pem')).fetch, + now: () => Date.parse(now), + replayStore: null, + }); + const at = (timestamp) => + JSON.stringify(scwMessage({ type: 'email_spam' }, { Timestamp: timestamp })); + await rejects( + expired('2027-01-01T12:00:30Z').verify({ body: at('2027-01-01T12:00:00.000Z') }), + 'certificate', + ); + const { events } = await expired('2020-06-01T12:00:30Z').verify({ + body: at('2020-06-01T12:00:00.000Z'), + }); + assert.strictEqual(events[0].type, 'complained'); +}); + test('Webhooks Scaleway: a certificate the trust chain did not issue, or off Scaleway, is refused', async () => { // The SES test certificate is self-signed, not issued by the test CA. const foreign = createScalewayWebhookVerifier({ @@ -303,7 +338,11 @@ test('Webhooks Maileroo: the HMAC of the body, an event per recipient, once', as }), ), ); - assert.deepStrictEqual(failed.events[0].bounce, { kind: 'hard', diagnostic: 'no user' }); + // Changed after the review: Maileroo does not say a failure is permanent. + assert.deepStrictEqual(failed.events[0].bounce, { + kind: 'undetermined', + diagnostic: 'no user', + }); const nobody = await verifier.verify( sign(JSON.stringify({ event_id: 'n', event_type: 'opened' })), ); From e04fd1d175e11600553258d37ec37101092aa7ae Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:33:49 +0300 Subject: [PATCH 113/143] fix(webhooks): no store where no id is kept, no password in an Azure URL - Postmark, Brevo, Mailjet, SparkPost and SMTP2GO took a replayStore and a leaseMs and never read them, so a caller believed requests were taken once. Both throw a TypeError, as toleranceMs and now do. - Azure: a validationUrl with a password and no user name passed. - The lease was said shorter than SNS's first retry of 20 seconds, which 30 seconds is not: a delivery that comes back while an id is held is refused as pending, and SNS's second retry takes it. The example key of the store is base64url, as the code writes it. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 4 +-- src/webhooks/azure.js | 7 ++++- src/webhooks/index.d.ts | 5 ++-- src/webhooks/request.js | 21 +++++++++++---- tests/unit/webhooks-request.test.js | 41 +++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 10 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 74f8ca5..fdb4313 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -130,7 +130,7 @@ are committed. function that makes its verifier for each request, a store they share; `null` for an application that deduplicates the events by their `id` itself. A shared store implements three methods. The ids it is given are the provider and a -SHA-256 of the provider's id (`mailtrap:3q2+7w…`), as some ids name a +SHA-256 of the provider's id in base64url (`mailtrap:3q2-7w…`), as some ids name a recipient. The ids of a batch go to it up to 64 at a time, so a store must take calls that run at once, as a Redis client does by pipelining them: @@ -155,7 +155,7 @@ const replayStore = { | Option | Default | | | --- | --- | --- | | `replayStore` | required | The ids taken, or `null`. | -| `leaseMs` | 30 seconds | How long an id stays reserved while it is handled: longer than a handler takes, shorter than the provider's first retry (20 seconds for SNS). | +| `leaseMs` | 30 seconds | How long an id stays reserved while it is handled: longer than a handler takes. A delivery that comes back while it is held is refused as `pending` (503), and the next one is taken: SNS retries every 20 seconds, so a request whose handler died is taken on its second retry. The verifiers of providers that sign no id (Postmark, Brevo, Mailjet, SparkPost, SMTP2GO) refuse it, and `replayStore`. | | `toleranceMs` | 5 minutes (an hour for SES) | How far from now a request may be signed. A verifier whose provider signs no time (Postmark, Brevo, Mailjet, SparkPost, SMTP2GO, MailerSend, Mailtrap, Maileroo, Azure) refuses it, and `now`. | | `maxBodyBytes` | 5 MiB | The longest body taken. | | `now` | `Date.now` | The clock of `toleranceMs`. | diff --git a/src/webhooks/azure.js b/src/webhooks/azure.js index fef5a60..5f22258 100644 --- a/src/webhooks/azure.js +++ b/src/webhooks/azure.js @@ -100,7 +100,12 @@ const isEventGridUrl = (value) => { } catch { return false; } - return url.protocol === 'https:' && EVENT_GRID_HOST.test(url.hostname) && url.username === ''; + return ( + url.protocol === 'https:' && + EVENT_GRID_HOST.test(url.hostname) && + url.username === '' && + url.password === '' + ); }; const validationOf = (payload) => { diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index da0cb89..8b22ced 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -149,8 +149,9 @@ export interface WebhookVerifierOptions { readonly replayStore: ReplayStore | null; /** * How long an id stays reserved while its request is handled, in - * milliseconds: longer than a handler takes, shorter than the provider's - * first retry. + * milliseconds: longer than a handler takes. A delivery that comes back + * while it is held is refused as `pending`, and the next one is taken (SNS + * retries every 20 seconds: its second retry). * @default 30000 */ readonly leaseMs?: number; diff --git a/src/webhooks/request.js b/src/webhooks/request.js index fc05f6c..b353b6e 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -15,9 +15,10 @@ const { } = require('../internal/options.js'); const DEFAULT_MAX_BODY_BYTES = 5 * 1024 * 1024; -// Longer than an application takes to handle a request, shorter than the -// first retry of a provider (20 s for SNS), so that a request whose handler -// died is taken again. +// Longer than an application takes to handle a request. A delivery that +// comes back while it is held is refused as pending, and the provider's next +// one after it is taken: SNS retries every 20 s, so a request whose handler +// died is taken on its second retry. const DEFAULT_LEASE_MS = 30 * 1000; /** * How far from now a signed request is taken unless a verifier says @@ -220,8 +221,18 @@ const resolveUntimedOptions = (options) => { return resolveCommonOptions(options, DAY_REPLAY_TTL_MS); }; -const resolveCredentialOptions = (options) => - resolveUntimedOptions({ ...options, replayStore: null }); +// A provider that signs no id keeps none, so an option of a store would be +// taken and never read. +const STORELESS = ['replayStore', 'leaseMs']; + +const resolveCredentialOptions = (options) => { + for (const name of STORELESS) { + if (options[name] !== undefined) { + throw new TypeError(`${name} is not read: the provider signs no id to keep.`); + } + } + return resolveUntimedOptions({ ...options, replayStore: null }); +}; // The key of an id in the replay store: its provider, and a digest of the id, // which may name a recipient (an event of Mailtrap without an event_id). diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 35d67f3..f329348 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -386,6 +386,47 @@ test('Webhooks: a verifier of a provider that signs no time refuses a window of } }); +test('Webhooks: a verifier of a provider that signs no id refuses a replay store', () => { + // Before: Postmark, Brevo, Mailjet, SparkPost and SMTP2GO took a replayStore + // and a leaseMs and never read them, so a caller believed requests were + // taken once. + const { + createMailjetWebhookVerifier, + createSparkPostWebhookVerifier, + } = require('../../src/webhooks/index.js'); + const makers = [ + (more) => createPostmarkWebhookVerifier({ username: 'u', password: 'p', ...more }), + (more) => createBrevoWebhookVerifier({ token: 't', ...more }), + (more) => createSmtp2goWebhookVerifier({ username: 'u', password: 'p', ...more }), + (more) => createMailjetWebhookVerifier({ username: 'u', password: 'p', ...more }), + (more) => createSparkPostWebhookVerifier({ username: 'u', password: 'p', ...more }), + ]; + for (const make of makers) { + assert.throws( + () => make({ replayStore: createMemoryReplayStore() }), + /replayStore is not read/, + ); + assert.throws(() => make({ leaseMs: 1000 }), /leaseMs is not read/); + assert.throws(() => make({ toleranceMs: 1000 }), /toleranceMs is not read/); + } +}); + +test('Webhooks: the validation URL of Azure carries no credentials', async () => { + // Before: a password without a user name passed the check of the URL. + const azure = createAzureWebhookVerifier({ token: 't', replayStore: null }); + const body = JSON.stringify([ + { + id: 'v-1', + eventType: 'Microsoft.EventGrid.SubscriptionValidationEvent', + data: { validationCode: 'c', validationUrl: 'https://:pw@rp-eu.eventgrid.azure.net/x' }, + }, + ]); + await assert.rejects( + azure.verify({ body, headers: { 'aeg-event-type': 'SubscriptionValidation' } }), + refusal('payload', 400), + ); +}); + test('Webhooks: a refusal says what to answer, a certificate not read asks for a retry', async () => { assert.strictEqual(new WebhookVerificationError('signature', 'x').status, 401); assert.strictEqual(new WebhookVerificationError('payload', 'x').status, 400); From f0be2ed48058c2e43c534c8622f8270f3b950e4b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:35:45 +0300 Subject: [PATCH 114/143] fix(webhooks): the replay store fences a reservation by the token of its request commit and release took the id alone, so a request whose lease ran out freed the reservation of the delivery that held the id now, and a third one was handled at the same time; the same gap the token of ./idempotency closed. A delivery reserves its ids under a token of its own, which its commit and release give: the memory store ignores a commit or a release under another token while the id is held, and the guide's Redis store does the same with two scripts. A store that ignores the token works as before. The memory store also warns (MAILER_REPLAY_EVICTED) when it drops committed ids before their time, as a request sent again then may be taken twice. ./webhooks goes from 15 to 15.5 KB. Co-Authored-By: Claude Opus 5.5 --- docs/guide/webhooks.md | 27 +++++++++----- scripts/size-budgets.json | 2 +- src/webhooks/index.d.ts | 27 ++++++++++---- src/webhooks/request.js | 58 +++++++++++++++++++++-------- tests/types/webhooks.test-d.ts | 3 ++ tests/unit/webhooks-request.test.js | 58 +++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 34 deletions(-) diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index fdb4313..04650bb 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -134,20 +134,29 @@ SHA-256 of the provider's id in base64url (`mailtrap:3q2-7w…`), as some ids na recipient. The ids of a batch go to it up to 64 at a time, so a store must take calls that run at once, as a Redis client does by pipelining them: +A reservation is made under the `token` of its request, which its commit and +release give, so that a request whose lease ran out neither frees nor commits +the reservation of the delivery that holds the id now: + ```js -// Redis, with node-redis: an atomic reservation, a commit, and a release -// that frees a reservation only, never a committed id. -const RELEASE = "if redis.call('get', KEYS[1]) == 'pending' then return redis.call('del', KEYS[1]) end"; +// Redis, with node-redis: an atomic reservation under a token, a commit +// unless another request holds the id, and a release of one's own +// reservation only, never a committed id. +const COMMIT = `local v = redis.call('get', KEYS[1]) +if v == false or v == 'done' or v == 'pending:' .. ARGV[1] then + return redis.call('set', KEYS[1], 'done', 'PX', ARGV[2]) end`; +const RELEASE = `if redis.call('get', KEYS[1]) == 'pending:' .. ARGV[1] then + return redis.call('del', KEYS[1]) end`; const replayStore = { - async reserve(id, leaseMs) { - if (await redis.set(id, 'pending', { NX: true, PX: leaseMs })) return 'reserved'; + async reserve(id, leaseMs, token) { + if (await redis.set(id, `pending:${token}`, { NX: true, PX: leaseMs })) return 'reserved'; return (await redis.get(id)) === 'done' ? 'done' : 'pending'; }, - async commit(id, ttlMs) { - await redis.set(id, 'done', { PX: ttlMs }); + async commit(id, ttlMs, token) { + await redis.eval(COMMIT, { keys: [id], arguments: [token, String(ttlMs)] }); }, - async release(id) { - await redis.eval(RELEASE, { keys: [id] }); + async release(id, token) { + await redis.eval(RELEASE, { keys: [id], arguments: [token] }); }, }; ``` diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index edccd1f..4840878 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -3,7 +3,7 @@ "./mime": 7.5, "./attachments": 4.5, "./render": 3, - "./webhooks": 15, + "./webhooks": 15.5, "./smtp": 23, "./sendmail": 11.5, "./jmap": 13, diff --git a/src/webhooks/index.d.ts b/src/webhooks/index.d.ts index 8b22ced..8099f4d 100644 --- a/src/webhooks/index.d.ts +++ b/src/webhooks/index.d.ts @@ -66,18 +66,29 @@ export type ReplayState = 'reserved' | 'pending' | 'done'; * processes, on Redis or a database, makes `reserve` one atomic operation * (`SET NX PX`, an insert under a unique index). The ids of a batch are * passed up to 64 at a time, so its methods are called while others run. + * A reservation is made under the `token` of its request, which the commit + * and the release give: a store that keeps it fences a request whose lease + * ran out, so that it neither frees nor commits the reservation of the + * request that holds the id now. */ export interface ReplayStore { /** - * Reserves an id for `leaseMs`, unless it is held: `reserved` when it is - * new, its lease ran out or it was released; `pending` while another - * request holds it; `done` once one was committed and until its time ends. + * Reserves an id for `leaseMs` under `token`, unless it is held: + * `reserved` when it is new, its lease ran out or it was released; + * `pending` while another request holds it; `done` once one was committed + * and until its time ends. */ - reserve(id: string, leaseMs: number): ReplayState | Promise; - /** Keeps a reserved id as taken for `ttlMs`. */ - commit(id: string, ttlMs: number): void | Promise; - /** Frees a reserved id that is not committed; a committed one stays. */ - release(id: string): void | Promise; + reserve(id: string, leaseMs: number, token?: string): ReplayState | Promise; + /** + * Keeps a reserved id as taken for `ttlMs`, unless another request holds + * it now under another token. + */ + commit(id: string, ttlMs: number, token?: string): void | Promise; + /** + * Frees an id reserved under `token` that is not committed; a committed + * one, or one another request holds, stays. + */ + release(id: string, token?: string): void | Promise; } /** diff --git a/src/webhooks/request.js b/src/webhooks/request.js index b353b6e..e7a4880 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -6,8 +6,9 @@ // verified, and an id is kept as taken only once the application has handled // its events. -const { createHash, createHmac, randomBytes, timingSafeEqual } = require('node:crypto'); +const { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } = require('node:crypto'); const { warnThrown } = require('../internal/callback.js'); +const { warnLimited } = require('../internal/warn.js'); const { checkOptionalFunction, checkPositiveInteger, @@ -70,13 +71,17 @@ class WebhookVerificationError extends Error { /** * Makes a store of the ids of the requests taken, in this process: an id is - * `pending` for the lease of the request that reserved it, and `done` for - * its time once that request is committed. It holds at most `maxEntries` - * ids, the oldest done ones dropped first; a pending id is never dropped - * before its lease ends. + * `pending` for the lease of the request that reserved it, under its token, + * and `done` for its time once that request is committed; a commit or a + * release under another token, from a request whose lease ran out and whose + * id another holds now, is ignored. It holds at most `maxEntries` ids, the + * oldest done ones dropped first, which a warning tells while they are kept + * (`MAILER_REPLAY_EVICTED`); a pending id is never dropped before its lease + * ends. * * @param options `maxEntries` (10000) and `now`. - * @returns `{ reserve(id, leaseMs), commit(id, ttlMs), release(id) }`. + * @returns `{ reserve(id, leaseMs, token), commit(id, ttlMs, token), + * release(id, token) }`. * @throws {TypeError} If an option is invalid. */ const createMemoryReplayStore = (options = {}) => { @@ -88,11 +93,22 @@ const createMemoryReplayStore = (options = {}) => { // A Map iterates in the order of insertion, and an entry is written anew // when it changes, so the first ones are the oldest. const trim = (at) => { + let dropped = 0; for (const [id, entry] of entries) { const live = entry.expiresAt > at; if (live && entries.size <= maxEntries) break; if (live && !entry.done) continue; entries.delete(id); + if (live) dropped++; + } + // A committed id dropped before its time can be taken again: the store + // is too small for the rate of the requests. + if (dropped > 0) { + warnLimited( + 'MAILER_REPLAY_EVICTED', + `The memory replay store dropped ${dropped} id(s) before their time to hold ` + + `${maxEntries}: a request sent again may be taken twice. Raise maxEntries.`, + ); } }; @@ -103,19 +119,25 @@ const createMemoryReplayStore = (options = {}) => { }; return { - reserve(id, leaseMs) { + reserve(id, leaseMs, token) { const at = now(); const entry = entries.get(id); if (entry !== undefined && entry.expiresAt > at) return entry.done ? 'done' : 'pending'; - write(id, { done: false, expiresAt: at + leaseMs }, at); + write(id, { done: false, expiresAt: at + leaseMs, token }, at); return 'reserved'; }, - commit(id, ttlMs) { + commit(id, ttlMs, token) { const at = now(); + const entry = entries.get(id); + // Another request holds the id now: its handler decides. + if (entry !== undefined && !entry.done && entry.expiresAt > at && entry.token !== token) { + return; + } write(id, { done: true, expiresAt: at + ttlMs }, at); }, - release(id) { - if (entries.get(id)?.done === false) entries.delete(id); + release(id, token) { + const entry = entries.get(id); + if (entry?.done === false && entry.token === token) entries.delete(id); }, }; }; @@ -281,16 +303,19 @@ class WebhookDelivery { events = []; #store; #ttlMs; + // The token the ids were reserved under, which a commit and a release give. + #token; // The reserved ids, each with the number of its events not committed. #left = new Map(); #keyOf = new WeakMap(); - constructor(fields, store, ttlMs) { + constructor(fields, store, ttlMs, token) { for (const name in fields) { if (Object.hasOwn(fields, name)) this[name] = fields[name]; } this.#store = store; this.#ttlMs = ttlMs; + this.#token = token; } add(key, events) { @@ -327,7 +352,7 @@ class WebhookDelivery { else keys.push(key); } } - await this.#settle(keys, (key) => this.#store.commit(key, this.#ttlMs)); + await this.#settle(keys, (key) => this.#store.commit(key, this.#ttlMs, this.#token)); } /** @@ -338,7 +363,7 @@ class WebhookDelivery { if (this.#store === null) return; const keys = []; for (const key of this.#left.keys()) keys.push(key); - await this.#settle(keys, (key) => this.#store.release(key)); + await this.#settle(keys, (key) => this.#store.release(key, this.#token)); } // A key whose call failed stays, to be committed or released again. @@ -368,7 +393,8 @@ class WebhookDelivery { */ const claimDelivery = async (common, entries, fields = {}, ttlMs = common.toleranceMs * 2) => { const store = common.replayStore; - const delivery = new WebhookDelivery(fields, store, ttlMs); + const token = store === null ? undefined : randomUUID(); + const delivery = new WebhookDelivery(fields, store, ttlMs, token); let done = 0; let pending = 0; if (store === null) { @@ -379,7 +405,7 @@ const claimDelivery = async (common, entries, fields = {}, ttlMs = common.tolera for (const entry of entries) keys.push(storeKeyOf(entry.key)); const states = []; const { failure } = await callEach(keys, async (key, index) => { - states[index] = await store.reserve(key, common.leaseMs); + states[index] = await store.reserve(key, common.leaseMs, token); }); // The events keep the order of the request, whatever order the store // answered in. diff --git a/tests/types/webhooks.test-d.ts b/tests/types/webhooks.test-d.ts index 8ca135a..735cd8b 100644 --- a/tests/types/webhooks.test-d.ts +++ b/tests/types/webhooks.test-d.ts @@ -48,6 +48,9 @@ expectType>(result.commit()); expectType>(result.commit([result.events[0]])); expectType>(result.release()); expectType>(store.reserve('id', 30_000)); +expectType>(store.reserve('id', 30_000, 'token')); +expectType>(store.commit('id', 60_000, 'token')); +expectType>(store.release('id', 'token')); const shared: ReplayStore = { reserve: async () => 'reserved' as const, commit: async () => {}, diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index f329348..3f822d9 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -538,3 +538,61 @@ test('Webhooks: onVerify hears each request taken or refused, and why', async (t /onVerify must be a function/, ); }); + +test('Webhooks: a late commit or release under another token leaves the id alone', async () => { + // Before: commit and release took the id alone, so a request whose lease + // ran out freed the reservation of the delivery that held the id now, and + // a third one was handled at the same time. + let time = 0; + const store = createMemoryReplayStore({ now: () => time }); + assert.strictEqual(store.reserve('a', 1000, 'first'), 'reserved'); + time = 1500; + assert.strictEqual(store.reserve('a', 1000, 'second'), 'reserved'); + store.release('a', 'first'); + assert.strictEqual(store.reserve('a', 1000, 'third'), 'pending'); + store.commit('a', 5000, 'first'); + assert.strictEqual(store.reserve('a', 1000, 'third'), 'pending'); + store.commit('a', 5000, 'second'); + assert.strictEqual(store.reserve('a', 1000, 'third'), 'done'); + // A commit after the lease, with nobody holding the id, still keeps it. + assert.strictEqual(store.reserve('b', 1000, 'x'), 'reserved'); + time = 3000; + store.commit('b', 5000, 'x'); + assert.strictEqual(store.reserve('b', 1000, 'y'), 'done'); + // The verifier passes one token per delivery to each call. + const calls = []; + const watching = { + reserve: async (id, leaseMs, token) => calls.push(['reserve', token]) && 'reserved', + commit: async (id, ttlMs, token) => { + calls.push(['commit', token]); + }, + release: async (id, token) => { + calls.push(['release', token]); + }, + }; + const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', replayStore: watching }); + const delivery = await mailtrap.verify(signMailtrap(mailtrapBatch('ev-1', 'ev-2'))); + await delivery.commit([delivery.events[0]]); + await delivery.release(); + const tokens = new Set(calls.map((call) => call[1])); + assert.strictEqual(tokens.size, 1); + assert.match([...tokens][0], /^[0-9a-f-]{36}$/); + assert.deepStrictEqual( + calls.map((call) => call[0]), + ['reserve', 'reserve', 'commit', 'release'], + ); +}); + +test('Webhooks: the memory replay store tells of a committed id dropped before its time', (t) => { + // Before: past maxEntries, live committed ids were dropped silently, so a + // request sent again in their time was taken twice. + const warned = []; + t.mock.method(process, 'emitWarning', (_text, options) => warned.push(options?.code)); + const store = createMemoryReplayStore({ maxEntries: 2, now: () => 0 }); + for (const id of ['a', 'b', 'c']) { + store.reserve(id, 1000, id); + store.commit(id, 60_000, id); + } + assert.deepStrictEqual(warned, ['MAILER_REPLAY_EVICTED']); + assert.strictEqual(store.reserve('a', 1000, 'again'), 'reserved'); +}); From c1e5aea65ad4faba9b8353164d120313610e8fb7 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:37:31 +0300 Subject: [PATCH 115/143] fix(webhooks): find a message of unknown delivery by the correlationId of its metadata The guide and the d.ts sent a receipt of unknown delivery to rfcMessageId, through a messageId pinned by ./pool or ./retry; but the HTTP APIs write a Message-ID of their own (SES, SendGrid and Mailgun do not take one), so the events carried another, and matched nothing. An event now carries the correlationId of the message's metadata, which every provider but Azure gives back with its events, and the guide says to find them by it; rfcMessageId is described as the Message-ID the provider reports. The rfcMessageId of SendGrid, MailerSend, SMTP2GO and Scaleway, untested, is pinned with it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- docs/guide/webhooks.md | 18 ++++-- index.d.ts | 17 +++-- src/webhooks/request.js | 4 ++ tests/unit/webhooks-mapping.test.js | 98 +++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adadd70..f6b1909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,8 +131,9 @@ additive but the defaults listed below. it meanwhile is refused as `pending`. `replayStore` is required where a verifier keeps ids (`createMemoryReplayStore()` in one process, a shared store of `reserve`, `commit` and `release`, or `null`), and a refusal carries the `status` to answer the provider with. - `onVerify` hears each request taken or refused, and an event carries the `rfcMessageId` it was - sent with, where the provider tells it, to find the events of a message of unknown delivery. + `onVerify` hears each request taken or refused, and an event carries the `correlationId` of the + message's metadata, given back by every provider but Azure, to find the events of a message of + unknown delivery, and the `rfcMessageId` the provider reports. - **`./pool` `verifyAll()`:** how each transport did, `verified`, `failed` or `unverifiable`, where `verify()` passes when one does. - **A failure of unknown delivery stands out:** `./otel` sets `mailer.delivery_unknown` on its diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 04650bb..fbc8d16 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -70,7 +70,8 @@ as many events. | `type` | `accepted`, `delivered`, `deferred`, `bounced`, `rejected`, `complained`, `unsubscribed`, `opened`, `clicked` or `unknown`. | | `id` | The id of the event, the same each time the provider sends it again. | | `messageId` | The provider's id of the message, the `messageId` of its receipt. | -| `rfcMessageId` | The Message-ID the message was sent with, when the event carries it (SES, SendGrid, Mailgun, and the providers whose events carry the header fields of the email). | +| `rfcMessageId` | The Message-ID as the provider reports it, when the event carries it (SES, SendGrid, Mailgun, MailerSend, SMTP2GO, Scaleway): one the provider wrote, for an HTTP API. | +| `correlationId` | The `correlationId` of the message's metadata, given back by every provider but Azure: what a message of unknown delivery is found by. | | `recipient` | The recipient, when the event names one, its domain in lower case. | | `occurredAt` | When it happened. | | `bounce` | For a bounce: `kind` (`hard`, `soft` or `undetermined`), `code` and `diagnostic`. | @@ -97,9 +98,18 @@ for the application to read. `tests/unit/webhooks-mapping.test.js` holds the mapping of every provider's events to this table. A receipt of unknown delivery (`deliveryUnknown`) has no `messageId` of the -provider to find its events by. Give the message a `messageId` of its own, or -send it through `./pool` or `./retry`, which pin one, and find its events by -`rfcMessageId`, where the provider tells it. +provider to find its events by, and `rfcMessageId` does not help: the HTTP +APIs write a Message-ID of their own, not the one the message was given. Give +the message `metadata: { correlationId }`, which every provider but Azure keeps +and gives back with its events, and find them by the event's `correlationId`: + +```js +const message = createMessage({ ...fields, metadata: { correlationId: order.id } }); +// Later, in the webhook handler: +for (const event of delivery.events) { + await deliveries.record(event.correlationId ?? event.messageId, event); +} +``` ## Time and replays diff --git a/index.d.ts b/index.d.ts index 1d9b6df..41aa9ac 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1350,13 +1350,20 @@ export interface DeliveryEvent { */ readonly messageId?: string; /** - * The Message-ID of the message as it was sent, when the event carries it - * (SES, SendGrid, Mailgun, and the providers whose events carry the header - * fields of the email). A receipt of unknown delivery has no `messageId` - * of the provider: give the message a `messageId`, or pass it through - * `./pool` or `./retry`, which pin one, and find its events by this. + * The Message-ID of the message as the provider reports it, when the event + * carries it (SES, SendGrid, Mailgun, MailerSend, SMTP2GO, Scaleway). The + * HTTP APIs write a Message-ID of their own, so it is not the one a + * message was given; to find the events of a message, use + * `correlationId`. */ readonly rfcMessageId?: string; + /** + * The `correlationId` of the message's `metadata`, which the providers + * that keep metadata (all but Azure) give back with its events: what a + * receipt of unknown delivery, which has no `messageId` of the provider, + * finds its events by. + */ + readonly correlationId?: string; /** * The recipient, when the event names one, its domain in lower case so * that an address reads the same in the events of every provider. diff --git a/src/webhooks/request.js b/src/webhooks/request.js index e7a4880..9724d55 100644 --- a/src/webhooks/request.js +++ b/src/webhooks/request.js @@ -634,6 +634,10 @@ const createEvent = (fields) => { } event.tags = fields.tags ?? []; event.metadata = fields.metadata ?? Object.create(null); + // What the application gave the message to find its events by, which a + // receipt of unknown delivery lacks the provider's id for. + const correlationId = stringOf(event.metadata.correlationId); + if (correlationId !== undefined) event.correlationId = correlationId; event.raw = fields.raw; return event; }; diff --git a/tests/unit/webhooks-mapping.test.js b/tests/unit/webhooks-mapping.test.js index 3543d70..52561de 100644 --- a/tests/unit/webhooks-mapping.test.js +++ b/tests/unit/webhooks-mapping.test.js @@ -681,3 +681,101 @@ test('Webhooks mapping: SparkPost, a bounce by its bounce class', async () => { const { events } = await verifier.verify({ body, headers: { Authorization: basic('sp', 'pw') } }); assert.deepStrictEqual(batchOf(rows, events), expectedOf(rows)); }); + +test('Webhooks mapping: the Message-ID a provider reports, and the correlationId of metadata', async () => { + // Before: an event carried no correlationId, and the guide sent a message + // of unknown delivery to rfcMessageId, which the HTTP APIs write themselves. + const { publicKey, privateKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); + const sendgrid = createSendGridWebhookVerifier({ + publicKey: publicKey.export({ format: 'der', type: 'spki' }).toString('base64'), + now: () => NOW, + replayStore: null, + }); + const sgBody = JSON.stringify([ + { + email: 'bob@example.com', + timestamp: 1_799_999_990, + sg_event_id: 'sg-c', + event: 'delivered', + 'smtp-id': '', + correlationId: 'order-1', + }, + ]); + const sgSignature = sign('sha256', Buffer.from(`1800000000${sgBody}`), privateKey).toString( + 'base64', + ); + const fromSendGrid = await sendgrid.verify({ + body: sgBody, + headers: { + 'X-Twilio-Email-Event-Webhook-Signature': sgSignature, + 'X-Twilio-Email-Event-Webhook-Timestamp': '1800000000', + }, + }); + const mailersend = createMailerSendWebhookVerifier({ secret: 'ms-secret', replayStore: null }); + const msBody = JSON.stringify({ + type: 'activity.delivered', + created_at: '2027-01-15T08:00:00.000000Z', + data: { + id: 'ms-c', + email: { + headers: [ + { name: 'Message-ID', value: '' }, + { name: 'X-Metadata-correlationId', value: 'order-2' }, + ], + }, + }, + }); + const fromMailerSend = await mailersend.verify({ + body: msBody, + headers: { signature: hmac('ms-secret', msBody) }, + }); + const smtp2go = createSmtp2goWebhookVerifier({ username: 'hook', password: 'pw' }); + const fromSmtp2go = await smtp2go.verify({ + body: JSON.stringify({ + event: 'delivered', + email_id: '1abc', + rcpt: 'bob@example.com', + 'Message-Id': '', + 'X-Metadata-correlationId': 'order-3', + }), + headers: { authorization: basic('hook', 'pw') }, + }); + const topic = 'arn:scw:sns:fr-par:project-1:tem-events'; + const scw = createScalewayWebhookVerifier({ + topicArns: [topic], + trustChain: scaleway.trustChain, + fetch: certificateFetch(scaleway.certificate).fetch, + now: () => SNS_NOW, + replayStore: null, + }); + const fromScaleway = await scw.verify({ + body: JSON.stringify( + signSns( + { + Type: 'Notification', + MessageId: 'scw-c', + TopicArn: topic, + Message: JSON.stringify({ + id: 'scw-event-c', + type: 'email_delivered', + email_headers: [ + { key: 'Message-Id', value: '' }, + { key: 'X-Metadata-correlationId', value: 'order-4' }, + ], + }), + Timestamp: '2027-01-01T12:00:00.000Z', + SigningCertURL: 'https://messaging.s3.fr-par.scw.cloud/fr-par/sns/cert-1.pem', + }, + '2', + scaleway.privateKey, + ), + ), + }); + const read = (delivery) => [delivery.events[0].rfcMessageId, delivery.events[0].correlationId]; + assert.deepStrictEqual([fromSendGrid, fromMailerSend, fromSmtp2go, fromScaleway].map(read), [ + ['', 'order-1'], + ['', 'order-2'], + ['', 'order-3'], + ['', 'order-4'], + ]); +}); From 85749fde92c87700a5ebb18b1b4e29f8d5ebd90c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:38:15 +0300 Subject: [PATCH 116/143] test(webhooks): the guide's handler on node:http, and the window of the store The example of the guide, on a node:http server: a delivery whose handling failed answers 500 and is taken again on the provider's next one, which a third delivery then finds a replay. And a batch of 150 ids goes to the store 64 at a time, no more. Co-Authored-By: Claude Opus 5.5 --- tests/unit/webhooks-request.test.js | 78 +++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/unit/webhooks-request.test.js b/tests/unit/webhooks-request.test.js index 3f822d9..cf1b53f 100644 --- a/tests/unit/webhooks-request.test.js +++ b/tests/unit/webhooks-request.test.js @@ -596,3 +596,81 @@ test('Webhooks: the memory replay store tells of a committed id dropped before i assert.deepStrictEqual(warned, ['MAILER_REPLAY_EVICTED']); assert.strictEqual(store.reserve('a', 1000, 'again'), 'reserved'); }); + +test('Webhooks: the handler of the guide takes again a delivery whose handling failed', async (t) => { + // The example of docs/guide/webhooks.md on node:http: before the two steps + // of the replay store, the first delivery's id was taken inside verify(), + // so the provider's redelivery was a replay, answered 200, and lost. + const http = require('node:http'); + const replayStore = createMemoryReplayStore(); + const resend = createResendWebhookVerifier({ secret: SECRET, now: () => NOW, replayStore }); + const recorded = []; + let failures = 1; + const deliveries = { + record: async (event) => { + if (failures-- > 0) throw new Error('database down'); + recorded.push(event.id); + }, + }; + const server = http.createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + let delivery; + try { + delivery = await resend.verify({ body: Buffer.concat(chunks), headers: req.headers }); + } catch (error) { + if (!(error instanceof WebhookVerificationError)) throw error; + res.statusCode = error.status; + res.end(); + return; + } + try { + for (const event of delivery.events) await deliveries.record(event); + await delivery.commit(); + res.statusCode = 200; + } catch { + await delivery.release(); + res.statusCode = 500; + } + res.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => server.close()); + const url = `http://127.0.0.1:${server.address().port}/webhooks/resend`; + const body = JSON.stringify({ + type: 'email.delivered', + created_at: '2027-01-15T08:00:00.000Z', + data: { email_id: 'e-1', to: ['bob@example.com'] }, + }); + const signed = signSvix(body, 'msg_guide'); + const post = async () => + (await fetch(url, { method: 'POST', body, headers: signed.headers })).status; + assert.strictEqual(await post(), 500); + assert.strictEqual(await post(), 200); + assert.strictEqual(await post(), 200); + assert.deepStrictEqual(recorded, ['msg_guide:0']); +}); + +test('Webhooks: a batch goes to the store 64 ids at a time, no more', async () => { + // The window that sends the ids of a batch together is bounded, so a batch + // of thousands does not open thousands of calls to a store at once. + let inFlight = 0; + let most = 0; + const store = { + reserve: async () => { + inFlight++; + most = Math.max(most, inFlight); + await new Promise((resolve) => setImmediate(resolve)); + inFlight--; + return 'reserved'; + }, + commit: async () => {}, + release: async () => {}, + }; + const ids = []; + for (let i = 0; i < 150; i++) ids.push(`ev-${i}`); + const mailtrap = createMailtrapWebhookVerifier({ secret: 'mt', replayStore: store }); + const delivery = await mailtrap.verify(signMailtrap(mailtrapBatch(...ids))); + assert.strictEqual(delivery.events.length, 150); + assert.strictEqual(most, 64); +}); From 4909985fc35142017660d77ca033359ae4d5bbf0 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:41:24 +0300 Subject: [PATCH 117/143] fix: a timeout past what a timer keeps is a long one, not one of 1 ms A timeout over 2^31 - 1 ms (some 24.8 days) went to setTimeout as it was, which fires such a delay at once, so a provider with timeout: 3e9 timed out every request, an SMTP socket and an HTTP proxy failed at once, and AbortSignal.timeout threw past 2^32 - 1, which made every SNS message of a large certificateTimeoutMs answer 503. timerMs of internal/sleep.js cuts a delay to the longest a timer keeps, in effect none, at every timer that takes one from the options. ./smtp goes to 23.5 KB, ./maileroo to 7.5 and ./tee to 3.5. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 6 ++-- src/attachments/url.js | 3 +- src/internal/http.js | 4 +-- src/internal/sleep.js | 19 ++++++++++-- src/jmap/blob-uploader.js | 3 +- src/jmap/jmap-transport.js | 3 +- src/jmap/raw-upload.js | 3 +- src/pool/pool-transport.js | 3 +- src/ratelimit/ratelimit-transport.js | 4 +-- src/sendmail/sendmail-transport.js | 4 +-- src/smtp/data-stream.js | 8 +++-- src/smtp/http-proxy.js | 5 +++- src/smtp/smtp-connection.js | 17 ++++++----- src/tee/tee-transport.js | 5 ++-- src/webhooks/sns.js | 6 +++- tests/unit/internal.test.js | 45 ++++++++++++++++++++++++++++ 16 files changed, 108 insertions(+), 30 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 4840878..d5bc58e 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -4,7 +4,7 @@ "./attachments": 4.5, "./render": 3, "./webhooks": 15.5, - "./smtp": 23, + "./smtp": 23.5, "./sendmail": 11.5, "./jmap": 13, "./ses": 8, @@ -12,7 +12,7 @@ "./sendgrid": 7.5, "./mailgun": 7.5, "./mailtrap": 7.5, - "./maileroo": 7, + "./maileroo": 7.5, "./lettermint": 7, "./plunk": 7, "./postmark": 7.5, @@ -34,5 +34,5 @@ "./ratelimit": 3.5, "./idempotency": 4, "./hooks": 2.5, - "./tee": 3 + "./tee": 3.5 } diff --git a/src/attachments/url.js b/src/attachments/url.js index fa4f26f..02383c2 100644 --- a/src/attachments/url.js +++ b/src/attachments/url.js @@ -14,6 +14,7 @@ const net = require('node:net'); const { readBytes } = require('../internal/body.js'); const { checkMaxBytes, createAttachment, mediaTypeOf } = require('./attachment.js'); const { DEFAULT_TYPE, extensionOfType, typeOfFilename } = require('./mime-types.js'); +const { timerMs } = require('../internal/sleep.js'); // The addresses no attachment of a sender's message should come from: this // network, the private ranges, CGNAT, loopback, link-local (where cloud @@ -130,7 +131,7 @@ const signalOf = (timeout, signal) => { const timer = timeout === 0 || timeout === Number.POSITIVE_INFINITY ? undefined - : AbortSignal.timeout(timeout); + : AbortSignal.timeout(timerMs(timeout)); if (timer === undefined) return signal; return signal == null ? timer : AbortSignal.any([timer, signal]); }; diff --git a/src/internal/http.js b/src/internal/http.js index 5b92a64..cf13ecf 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -6,7 +6,7 @@ const { combineSignals } = require('../core/abort-signal.js'); const { isCertificateFailure } = require('../core/receipt.js'); const { markDeliveryUnknown } = require('./delivery-unknown.js'); const { isUnsentRequestError } = require('./unsent.js'); -const { abortReason, sleep } = require('./sleep.js'); +const { abortReason, sleep, timerMs } = require('./sleep.js'); const { redactAddresses } = require('./redact-addresses.js'); const isAbortError = (error) => error instanceof Error && error.name === 'AbortError'; @@ -59,7 +59,7 @@ const startDeadline = (timeoutMs, signal) => { const controller = new AbortController(); const timer = timeoutMs > 0 && Number.isFinite(timeoutMs) - ? setTimeout(() => controller.abort(), timeoutMs) + ? setTimeout(() => controller.abort(), timerMs(timeoutMs)) : undefined; const combined = combineSignals(controller.signal, signal); return { diff --git a/src/internal/sleep.js b/src/internal/sleep.js index ebc46fa..3fbaf41 100644 --- a/src/internal/sleep.js +++ b/src/internal/sleep.js @@ -33,8 +33,23 @@ const sleep = (milliseconds, signal) => const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(); - }, milliseconds); + }, timerMs(milliseconds)); signal?.addEventListener('abort', onAbort, { once: true }); }); -module.exports = { abortReason, sleep }; +// The longest delay a timer keeps on Node, Bun and Deno: a longer one fires at +// once (Node and Bun) or throws (AbortSignal.timeout past 2^32 - 1). +const MAX_TIMER_MS = 2_147_483_647; + +/** + * A delay a timer can take: one longer than the longest a timer keeps, some + * 24.8 days, is cut to it, where a timer would fire at once. A timeout that + * long is in effect none. + * + * @param milliseconds The delay asked for. + * @returns The delay to give the timer. + * @internal + */ +const timerMs = (milliseconds) => (milliseconds > MAX_TIMER_MS ? MAX_TIMER_MS : milliseconds); + +module.exports = { abortReason, sleep, timerMs }; diff --git a/src/jmap/blob-uploader.js b/src/jmap/blob-uploader.js index c064b83..d2651e2 100644 --- a/src/jmap/blob-uploader.js +++ b/src/jmap/blob-uploader.js @@ -5,6 +5,7 @@ const { combineSignals } = require('../core/abort-signal.js'); const { JmapApiError } = require('./errors.js'); const { readJson, readText } = require('../internal/body.js'); const { Buffer } = require('node:buffer'); +const { timerMs } = require('../internal/sleep.js'); /** * Upload a blob to the JMAP server. @@ -43,7 +44,7 @@ const uploadBlob = async (config, uploadUrl, accountId, blob, signal) => { } const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), config.timeout); + const timeoutId = setTimeout(() => controller.abort(), timerMs(config.timeout)); // Combine signals if an external signal is provided const combinedSignal = combineSignals(controller.signal, signal); diff --git a/src/jmap/jmap-transport.js b/src/jmap/jmap-transport.js index 2b02878..907fa20 100644 --- a/src/jmap/jmap-transport.js +++ b/src/jmap/jmap-transport.js @@ -20,6 +20,7 @@ const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact. const { chunks, readItems } = require('../internal/batch.js'); const { schedulingRefusal } = require('../internal/schedule.js'); const { certificateFailure } = require('../internal/refusal.js'); +const { timerMs } = require('../internal/sleep.js'); /** * JMAP capabilities required for email sending. @@ -1060,7 +1061,7 @@ const verificationOperation = async (timeout, operation, signal) => { let onAbort; const timer = setTimeout( () => controller.abort(new JmapApiError('JMAP verification timed out.')), - timeout, + timerMs(timeout), ); try { combined.signal.throwIfAborted(); diff --git a/src/jmap/raw-upload.js b/src/jmap/raw-upload.js index 9fc85e1..91e1fa3 100644 --- a/src/jmap/raw-upload.js +++ b/src/jmap/raw-upload.js @@ -7,6 +7,7 @@ const { parseRetryAfter } = require('../core/receipt.js'); const { JmapApiError } = require('./errors.js'); const { readJson, readText } = require('../internal/body.js'); const { Buffer } = require('node:buffer'); +const { timerMs } = require('../internal/sleep.js'); /** Runs a raw operation with a progress-resettable deadline through response parsing. @internal */ const rawOperation = async (timeout, operation, signal) => { @@ -19,7 +20,7 @@ const rawOperation = async (timeout, operation, signal) => { clearTimeout(timer); timer = setTimeout( () => controller.abort(new JmapApiError('Raw JMAP operation timed out.')), - timeout, + timerMs(timeout), ); }; let abort; diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 9b8f332..0c517d3 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -12,6 +12,7 @@ const { RoundRobinStrategy } = require('./strategies/round-robin-strategy.js'); const { WeightedStrategy } = require('./strategies/weighted-strategy.js'); const { PriorityStrategy } = require('./strategies/priority-strategy.js'); const { SelectorStrategy } = require('./strategies/selector-strategy.js'); +const { timerMs } = require('../internal/sleep.js'); /** * Pool transport that combines multiple transports with various load balancing @@ -394,7 +395,7 @@ class PoolTransport { const timeoutId = setTimeout(() => { abortSource ??= 'timeout'; timeoutController.abort(createAbortError()); - }, this.config.timeout); + }, timerMs(this.config.timeout)); let cleanupAbortSource = noop; let signal = timeoutController.signal; let cleanupCombinedSignal = noop; diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index b34fe33..e6e10e4 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -18,7 +18,7 @@ const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); const { warnLimited } = require('../internal/warn.js'); const { createLog } = require('../internal/log.js'); -const { abortReason, sleep } = require('../internal/sleep.js'); +const { abortReason, sleep, timerMs } = require('../internal/sleep.js'); const { createRateLimitConfig } = require('./config.js'); /** @@ -266,7 +266,7 @@ class RateLimitTransport { } if (head.cost > bucket.tokens) { const delay = Math.max(1, Math.ceil((head.cost - bucket.tokens) / this.#refillPerMs)); - bucket.timer = setTimeout(() => this.#pump(bucket), delay); + bucket.timer = setTimeout(() => this.#pump(bucket), timerMs(delay)); return; } bucket.waiters.shift(); diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index 942ea42..9bf96c8 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -30,7 +30,7 @@ const { createLog } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); -const { abortReason } = require('../internal/sleep.js'); +const { abortReason, timerMs } = require('../internal/sleep.js'); const { Fifo } = require('../internal/gate.js'); const { composeMessage } = require('../mime/index.js'); const { createSendmailConfig } = require('./config.js'); @@ -169,7 +169,7 @@ const run = (child, chunks, newline, abort, timeout) => : setTimeout(() => { outcome.timedOut = true; terminate(child); - }, timeout); + }, timerMs(timeout)); const done = () => { clearTimeout(timer); abort?.removeEventListener('abort', onAbort); diff --git a/src/smtp/data-stream.js b/src/smtp/data-stream.js index f720617..2bb45e9 100644 --- a/src/smtp/data-stream.js +++ b/src/smtp/data-stream.js @@ -3,6 +3,7 @@ const { combineSignals } = require('../core/abort-signal.js'); const { Buffer } = require('node:buffer'); +const { timerMs } = require('../internal/sleep.js'); /** Races preparation against source inactivity and remote socket termination. */ /** @@ -69,7 +70,10 @@ const prepareOnSocket = async (socket, timeoutMs, prepare, signal) => { let timer; const progress = () => { clearTimeout(timer); - timer = setTimeout(() => fail(new TypeError('SMTP message preparation timeout.')), timeoutMs); + timer = setTimeout( + () => fail(new TypeError('SMTP message preparation timeout.')), + timerMs(timeoutMs), + ); }; socket.on('error', fail); socket.on('close', closed); @@ -257,7 +261,7 @@ const writeMessageData = async (socket, source, timeoutMs, checkSize, signal) => let timer; const progress = () => { clearTimeout(timer); - timer = setTimeout(() => fail(new TypeError('SMTP DATA timeout.')), timeoutMs); + timer = setTimeout(() => fail(new TypeError('SMTP DATA timeout.')), timerMs(timeoutMs)); }; const data = createDataReplyReader( () => terminated, diff --git a/src/smtp/http-proxy.js b/src/smtp/http-proxy.js index 229dbbc..42fd375 100644 --- a/src/smtp/http-proxy.js +++ b/src/smtp/http-proxy.js @@ -16,6 +16,7 @@ const { Buffer } = require('node:buffer'); const net = require('node:net'); const tls = require('node:tls'); const { isLoopbackHost } = require('../internal/loopback.js'); +const { timerMs } = require('../internal/sleep.js'); // The most bytes of the reply to CONNECT before its blank line, so a proxy // that never ends its header fields cannot fill the memory of the sender. @@ -220,7 +221,9 @@ const tunnel = (socket, isHttps, request, target, timeout, signal) => socket.once('close', onClose); // The whole exchange, not each wait for a byte: a proxy that answers a // byte at a time would hold a socket timeout off forever. - if (timeout > 0 && timeout !== Number.POSITIVE_INFINITY) timer = setTimeout(onTimeout, timeout); + if (timeout > 0 && timeout !== Number.POSITIVE_INFINITY) { + timer = setTimeout(onTimeout, timerMs(timeout)); + } signal?.addEventListener('abort', onAbort, { once: true }); }); diff --git a/src/smtp/smtp-connection.js b/src/smtp/smtp-connection.js index 4bbdc5e..98c00d3 100644 --- a/src/smtp/smtp-connection.js +++ b/src/smtp/smtp-connection.js @@ -27,6 +27,7 @@ const { const { prepareMessageStream } = require('./message-stream.js'); const { prepareOnSocket, writeMessageData } = require('./data-stream.js'); const { parseEnhancedSmtpStatusCode } = require('./smtp-status-code.js'); +const { timerMs } = require('../internal/sleep.js'); // RFC 5321 §4.5.3.1.5 limits a reply line to 512 octets, but servers send // longer text; these bound the memory a broken or hostile server can take @@ -485,7 +486,7 @@ class SmtpConnection { const timer = setTimeout(() => { onError(new TypeError('Connection timeout.')); this.socket?.destroy(); - }, this.config.connectionTimeout); + }, timerMs(this.config.connectionTimeout)); const cleanup = () => { clearTimeout(timer); @@ -526,7 +527,7 @@ class SmtpConnection { try { this.socket = this.openSocket(); - this.socket.setTimeout(this.config.socketTimeout); + this.socket.setTimeout(timerMs(this.config.socketTimeout)); this.observeSocket(this.socket); this.socket.once(establishedEvent, onConnect); this.socket.once('error', onError); @@ -578,7 +579,7 @@ class SmtpConnection { const timeout = new AbortController(); const timer = setTimeout( () => timeout.abort(new TypeError('Connection timeout.')), - this.config.connectionTimeout, + timerMs(this.config.connectionTimeout), ); const combined = signal == null ? timeout.signal : AbortSignal.any([timeout.signal, signal]); let tunnel; @@ -599,7 +600,7 @@ class SmtpConnection { combined.throwIfAborted(); this.tunneled = true; this.socket = this.config.secure ? this.wrapInTls(tunnel) : tunnel; - this.socket.setTimeout(this.config.socketTimeout); + this.socket.setTimeout(timerMs(this.config.socketTimeout)); this.observeSocket(this.socket); if (this.config.secure) await secureConnectOf(this.socket, combined); this.log.write( @@ -749,7 +750,7 @@ class SmtpConnection { const collector = createReplyCollector(); const timer = setTimeout(() => { onError(new Error('Greeting timeout.')); - }, this.config.socketTimeout); + }, timerMs(this.config.socketTimeout)); // Only complete lines count: a final line still waiting for its CRLF // could be cut short. @@ -899,7 +900,7 @@ class SmtpConnection { return new Promise((resolve, reject) => { const timer = setTimeout( () => fail(new Error('STARTTLS upgrade timeout.')), - this.config.connectionTimeout, + timerMs(this.config.connectionTimeout), ); const cleanup = () => { clearTimeout(timer); @@ -920,7 +921,7 @@ class SmtpConnection { const onClose = () => fail(new TypeError('SMTP connection closed during STARTTLS.')); const onSecureConnect = () => { cleanup(); - tlsSocket.setTimeout(this.config.socketTimeout); + tlsSocket.setTimeout(timerMs(this.config.socketTimeout)); this.log.write('debug', () => ({ protocol: tlsSocket.getProtocol() }), 'SMTP upgraded'); resolve(); }; @@ -1145,7 +1146,7 @@ class SmtpConnection { return await this.sendPreparedMessage(message, signal); } finally { this.active = false; - if (this.usable) this.socket?.setTimeout(this.config.socketTimeout); + if (this.usable) this.socket?.setTimeout(timerMs(this.config.socketTimeout)); } } diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 8c3f06c..5af7317 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -19,6 +19,7 @@ const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createTeeConfig } = require('./config.js'); +const { timerMs } = require('../internal/sleep.js'); const describe = (outcome) => { if (outcome.skipped !== undefined) return outcome.skipped; @@ -33,7 +34,7 @@ const withDeadline = (sending, timeout, mirror) => { const deadline = new Promise((resolve, reject) => { timer = setTimeout( () => reject(new Error(`The mirror ${mirror} did not answer within ${timeout} ms.`)), - timeout, + timerMs(timeout), ); // A detached send does not keep the process alive; disposal waits for it. timer.unref?.(); @@ -163,7 +164,7 @@ class TeeTransport { return; } // A detached mirror outlives the caller's signal; an awaited one follows it. - const timeout = AbortSignal.timeout(this.config.timeout); + const timeout = AbortSignal.timeout(timerMs(this.config.timeout)); const mirrorSignal = detached || signal === undefined ? timeout : AbortSignal.any([signal, timeout]); if (detached) this.#pending++; diff --git a/src/webhooks/sns.js b/src/webhooks/sns.js index 7e32856..514170b 100644 --- a/src/webhooks/sns.js +++ b/src/webhooks/sns.js @@ -17,6 +17,7 @@ const { parseJsonOf, resolveCommonOptions, } = require('./request.js'); +const { timerMs } = require('../internal/sleep.js'); const MAX_CERTIFICATE_BYTES = 64 * 1024; const MAX_CERTIFICATES = 32; @@ -93,7 +94,10 @@ const loadCertificate = async (fetchOption, url, timeoutMs) => { const fetch = fetchOption ?? globalThis.fetch; let response; try { - response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(timeoutMs) }); + response = await fetch(url, { + redirect: 'error', + signal: AbortSignal.timeout(timerMs(timeoutMs)), + }); } catch (error) { throw certificateError('The signing certificate could not be read.', error, UNREAD); } diff --git a/tests/unit/internal.test.js b/tests/unit/internal.test.js index c828803..d93583d 100644 --- a/tests/unit/internal.test.js +++ b/tests/unit/internal.test.js @@ -575,3 +575,48 @@ test('Decorators: pool and retry keep their loops to themselves', () => { ); assert.strictEqual('withRetries' in new RetryTransport(inner), false); }); + +test('Internal: a timeout past what a timer keeps is a long one, not one of 1 ms', async (t) => { + // Before: a timeout over 2^31 - 1 ms went to setTimeout as it was, which + // fires such a delay at once, so every request of a provider with + // timeout: 3e9 timed out, and AbortSignal.timeout past 2^32 - 1 threw. + const { timerMs } = require('../../src/internal/sleep.js'); + assert.strictEqual(timerMs(1000), 1000); + assert.strictEqual(timerMs(0), 0); + assert.strictEqual(timerMs(3e9), 2_147_483_647); + assert.strictEqual(timerMs(Number.POSITIVE_INFINITY), 2_147_483_647); + const { createMessage } = require('../../index.js'); + const { PostmarkTransport } = require('../../src/postmark/index.js'); + t.mock.method(globalThis, 'fetch', async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return Response.json({ MessageID: 'pm-1', ErrorCode: 0 }); + }); + const postmark = new PostmarkTransport({ serverToken: 'token', timeout: 3e9, retries: 0 }); + const receipt = await postmark.send( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 'x' }, + }), + ); + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + const { TeeTransport } = require('../../src/tee/index.js'); + const { fakeTransport, sent } = require('../helpers/decorators/fake.js'); + const outcomes = []; + const tee = new TeeTransport(fakeTransport(), { + mirrors: [fakeTransport(async () => sent('m'))], + timeout: 5e9, + awaitMirrors: true, + onMirror: (outcome) => outcomes.push(outcome.receipt?.successful ?? outcome.error?.message), + }); + await tee.send( + createMessage({ + from: 'a@example.com', + to: 'b@example.com', + subject: 's', + content: { text: 'x' }, + }), + ); + assert.deepStrictEqual(outcomes, [true]); +}); From 8867886081ea7e38db98ac9f9290e5742df091bc Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:43:38 +0300 Subject: [PATCH 118/143] fix: a logger that throws is told once, and keys reach the logs redacted A logger that threw on every record made a process warning of each, with its error text as it was, which can name the recipient of the record. The log of a component tells its logger's first failure, by the name and code of the error alone, and not the later ones; ./logger holds its own to ten a minute, addresses redacted. ./idempotency wrote the key in its records as it was, though a key built of a recipient names one; the addresses in it are redacted. ./sendmail goes to 12 KB, ./sendgrid, ./postmark and ./mailersend to 8, ./breaker to 3.5 and ./ratelimit to 4, for the log of every component. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 12 ++++++------ src/idempotency/idempotency-transport.js | 19 ++++++++++++++++--- src/internal/log.js | 19 ++++++++++++++++--- src/logger/logger-transport.js | 8 +++++--- tests/unit/idempotency.test.js | 18 ++++++++++++++++++ tests/unit/internal.test.js | 21 +++++++++++++++++++++ 6 files changed, 82 insertions(+), 15 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index d5bc58e..53332f0 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -5,20 +5,20 @@ "./render": 3, "./webhooks": 15.5, "./smtp": 23.5, - "./sendmail": 11.5, + "./sendmail": 12, "./jmap": 13, "./ses": 8, "./resend": 7.5, - "./sendgrid": 7.5, + "./sendgrid": 8, "./mailgun": 7.5, "./mailtrap": 7.5, "./maileroo": 7.5, "./lettermint": 7, "./plunk": 7, - "./postmark": 7.5, + "./postmark": 8, "./brevo": 7.5, "./mailjet": 8.5, - "./mailersend": 7.5, + "./mailersend": 8, "./sparkpost": 8, "./zeptomail": 7.5, "./smtp2go": 8, @@ -30,8 +30,8 @@ "./mock": 2, "./otel": 5, "./logger": 2.5, - "./breaker": 3, - "./ratelimit": 3.5, + "./breaker": 3.5, + "./ratelimit": 4, "./idempotency": 4, "./hooks": 2.5, "./tee": 3.5 diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index f17a351..51b9765 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -18,6 +18,7 @@ const { describeThrown, warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); const { disposeOf, isAbortOf, messageOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); +const { redactAddresses } = require('../internal/redact-addresses.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createIdempotencyConfig } = require('./config.js'); const { fingerprintOf } = require('./fingerprint.js'); @@ -159,7 +160,11 @@ class IdempotencyTransport { #replyOf(record, fingerprint, key) { if (record.fingerprint !== fingerprint) { - this.#log.write('warn', { code: 'idempotency.key_reused', key }, 'A key was used again.'); + this.#log.write( + 'warn', + () => ({ code: 'idempotency.key_reused', key: redactAddresses(key) }), + 'A key was used again.', + ); return createFailedReceipt( `The idempotency key ${key} was used for another message; this one was not sent.`, { @@ -171,10 +176,18 @@ class IdempotencyTransport { ); } if (record.state === 'done' && record.receipt !== undefined) { - this.#log.write('debug', { code: 'idempotency.hit', key }, 'A send was answered as before.'); + this.#log.write( + 'debug', + () => ({ code: 'idempotency.hit', key: redactAddresses(key) }), + 'A send was answered as before.', + ); return record.receipt; } - this.#log.write('info', { code: 'idempotency.in_flight', key }, 'A send of the key is out.'); + this.#log.write( + 'info', + () => ({ code: 'idempotency.in_flight', key: redactAddresses(key) }), + 'A send of the key is out.', + ); // The category is given, as the words of the text, which hold the key, // would otherwise choose one (a key `order-timeout-1` made it a timeout, // which a circuit breaker counts against the provider). diff --git a/src/internal/log.js b/src/internal/log.js index 5f07b72..5764e89 100644 --- a/src/internal/log.js +++ b/src/internal/log.js @@ -2,6 +2,13 @@ // the pino-like logger the application passes in. Without one a transport // writes nothing, except the warnings of warnDegradation. +// What a logger's failure is, by its name and code alone: its text may name +// a recipient of the record it failed to write. +const failureOf = (error) => { + const name = typeof error?.name === 'string' ? error.name : typeof error; + return typeof error?.code === 'string' ? `${name} (${error.code})` : name; +}; + /** * Makes the log of a component: the logger bound to `bindings` through * `child()` when it has one, or with them merged into each record otherwise. @@ -18,6 +25,9 @@ const createLog = (logger, bindings) => { if (logger == null) return SILENT; const bound = typeof logger.child === 'function' ? logger.child(bindings) : logger; const merge = bound === logger; + // A logger that throws for every record would fill the output: its first + // failure is told, the others not. + let failed = false; const isEnabled = (level) => typeof bound[level] === 'function' && (bound.isLevelEnabled?.(level) ?? true); return { @@ -33,9 +43,12 @@ const createLog = (logger, bindings) => { try { bound[level](merge ? { ...bindings, ...built } : built, message); } catch (error) { - process.emitWarning(`A logger passed to the mailer threw: ${error?.message ?? error}`, { - code: 'MAILER_LOGGER_FAILED', - }); + if (failed) return; + failed = true; + process.emitWarning( + `A logger passed to the mailer threw ${failureOf(error)}; its later failures are not told.`, + { code: 'MAILER_LOGGER_FAILED' }, + ); } }, }; diff --git a/src/logger/logger-transport.js b/src/logger/logger-transport.js index 71827eb..d16af9d 100644 --- a/src/logger/logger-transport.js +++ b/src/logger/logger-transport.js @@ -5,6 +5,7 @@ const { inspect } = require('node:util'); const { generateId } = require('../core/id.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); +const { warnLimited } = require('../internal/warn.js'); const { resolveLoggerTransportOptions } = require('./config.js'); const { PendingQueue } = require('./pending-queue.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); @@ -369,9 +370,10 @@ class LoggerTransport { try { this.logger[level](properties, message); } catch (error) { - process.emitWarning(`The logger of LoggerTransport threw: ${describe(error)}`, { - code: 'MAILER_LOGGER_FAILED', - }); + warnLimited( + 'MAILER_LOGGER_FAILED', + `The logger of LoggerTransport threw: ${redactAddresses(describe(error))}`, + ); } } } diff --git a/tests/unit/idempotency.test.js b/tests/unit/idempotency.test.js index cc71537..976d01f 100644 --- a/tests/unit/idempotency.test.js +++ b/tests/unit/idempotency.test.js @@ -520,3 +520,21 @@ test('Idempotency: a key the wrapped sendMany never took is free again', async ( assert.strictEqual(receipts[0].successful, true); assert.strictEqual(inner.calls.length, 1); }); + +test('Idempotency: the logger hears a key with the addresses in it redacted', async () => { + // Before: the records of a hit, a send in flight and a key used again + // carried the key as it was, though a key built of a recipient names one. + const records = []; + const logger = { + debug: (record) => records.push(record), + info: (record) => records.push(record), + warn: (record) => records.push(record), + }; + const once = new IdempotencyTransport(fakeTransport(), { logger }); + const keyed = message({ idempotencyKey: 'welcome:bob@example.com' }); + await once.send(keyed); + await once.send(keyed); + const hit = records.find((record) => record.code === 'idempotency.hit'); + assert.ok(hit); + assert.doesNotMatch(hit.key, /bob@example\.com/); +}); diff --git a/tests/unit/internal.test.js b/tests/unit/internal.test.js index d93583d..a77bbf5 100644 --- a/tests/unit/internal.test.js +++ b/tests/unit/internal.test.js @@ -620,3 +620,24 @@ test('Internal: a timeout past what a timer keeps is a long one, not one of 1 ms ); assert.deepStrictEqual(outcomes, [true]); }); + +test('Internal: a logger that throws is told once, by its name, not its text', (t) => { + // Before: every record a broken logger threw on was a process warning, with + // its error text as it was, such as the recipient it failed to write. + const { createLog } = require('../../src/internal/log.js'); + const warnings = []; + t.mock.method(process, 'emitWarning', (text, options) => warnings.push([options.code, text])); + const log = createLog( + { + info: () => { + throw new Error('cannot write bob@example.com'); + }, + }, + {}, + ); + for (let i = 0; i < 25; i++) log.write('info', {}, 'x'); + assert.strictEqual(warnings.length, 1); + assert.strictEqual(warnings[0][0], 'MAILER_LOGGER_FAILED'); + assert.match(warnings[0][1], /threw Error; its later failures are not told/); + assert.doesNotMatch(warnings[0][1], /bob@example\.com/); +}); From 7e6011756acd801e941019a8e3a472691bed2727 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:46:13 +0300 Subject: [PATCH 119/143] fix: every failed receipt of the library names its category Receipts of the library's own conditions had none and were classified by the words of their text: an answer without a message id (the shared sentWith, Mailjet, Postmark, SMTP2GO, SparkPost), a repeat Brevo or Azure took before, a pool with no error to tell, and a thrown error in the shared providerFailure, sendmail and SES. The first are unknown, as a delivery of unknown delivery is; a thrown error is classified by classifyReceiptError, the codes along its causes first, then its words. A test fails when a call of createFailedReceipt in src/ gives no category, no status to tell it by, no options that carry one and no errors of other transports. ./sparkpost goes to 8.5 KB. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/azure/azure-transport.js | 1 + src/brevo/brevo-transport.js | 7 ++++- src/internal/api-client.js | 19 ++++++++++- src/internal/api.js | 2 +- src/mailjet/mailjet-transport.js | 6 +++- src/pool/pool-transport.js | 3 ++ src/postmark/postmark-transport.js | 1 + src/sendmail/sendmail-transport.js | 6 ++-- src/ses/ses-transport.js | 6 +++- src/smtp2go/smtp2go-transport.js | 1 + src/sparkpost/sparkpost-transport.js | 1 + tests/unit/internal.test.js | 47 ++++++++++++++++++++++++++++ 13 files changed, 94 insertions(+), 8 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 53332f0..a3f6425 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -19,7 +19,7 @@ "./brevo": 7.5, "./mailjet": 8.5, "./mailersend": 8, - "./sparkpost": 8, + "./sparkpost": 8.5, "./zeptomail": 7.5, "./smtp2go": 8, "./azure": 9, diff --git a/src/azure/azure-transport.js b/src/azure/azure-transport.js index e261e3d..008f286 100644 --- a/src/azure/azure-transport.js +++ b/src/azure/azure-transport.js @@ -28,6 +28,7 @@ const failureOf = (error) => { // Azure took this request before: it is running, or went then. return createFailedReceipt(`${error.message} (a repeat of a request Azure took before)`, { provider: 'azure', + category: 'unknown', deliveryUnknown: true, statusCode: error.statusCode, attempts: error.attempts, diff --git a/src/brevo/brevo-transport.js b/src/brevo/brevo-transport.js index 75a676b..5559c13 100644 --- a/src/brevo/brevo-transport.js +++ b/src/brevo/brevo-transport.js @@ -24,7 +24,12 @@ const failureOf = (error) => { // Brevo took a request with this key before: the message went then. return createFailedReceipt( 'Brevo already took a message with this idempotency key; it is not sent again.', - { provider: 'brevo', deliveryUnknown: true, code: 'brevo.duplicate_parameter' }, + { + provider: 'brevo', + category: 'unknown', + deliveryUnknown: true, + code: 'brevo.duplicate_parameter', + }, ); } return providerFailure('brevo', error); diff --git a/src/internal/api-client.js b/src/internal/api-client.js index 570078e..f7275a7 100644 --- a/src/internal/api-client.js +++ b/src/internal/api-client.js @@ -3,7 +3,11 @@ // and messages through its hooks: the error of their API, an authenticated // request through `requestWithRetry`, and the receipt of a failure. -const { createFailedReceipt, parseRetryAfter } = require('../core/receipt.js'); +const { + classifyReceiptError, + createFailedReceipt, + parseRetryAfter, +} = require('../core/receipt.js'); const { readText } = require('./body.js'); const { isDeliveryUnknown } = require('./delivery-unknown.js'); const { requestWithRetry } = require('./http.js'); @@ -220,10 +224,22 @@ const providerFailure = (provider, error) => { provider, deliveryUnknown: isDeliveryUnknown(error), attempts: typeof error?.attempts === 'number' ? error.attempts : undefined, + // By the codes along its causes first, then by its words. + ...categoryOf(error), }) ); }; +/** + * The category and retryability of a thrown error, by `classifyReceiptError`. + * + * @internal + */ +const categoryOf = (error) => { + const { category, retryable } = classifyReceiptError(error); + return { category, retryable }; +}; + /** * Parses the JSON of a 2xx, or throws what `invalid(error)` makes of a body * that is not JSON. @@ -257,6 +273,7 @@ const isCallerAbort = (error, signal) => module.exports = { ProviderApiError, + categoryOf, createApiClient, isCallerAbort, parseAnswer, diff --git a/src/internal/api.js b/src/internal/api.js index 65cc365..be61948 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -170,7 +170,7 @@ const sendConverted = async (message, options, steps) => { const sentWith = (provider, messageId, missing) => typeof messageId === 'string' && messageId !== '' ? { successful: true, messageId, provider } - : createFailedReceipt(missing, { provider, deliveryUnknown: true }); + : createFailedReceipt(missing, { provider, category: 'unknown', deliveryUnknown: true }); module.exports = { ProviderApiError, diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index e43c9c3..f48efb1 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -71,9 +71,12 @@ class MailjetTransport { if (entry?.Status !== 'success') { const errors = Array.isArray(entry?.Errors) ? entry.Errors : []; const text = describeErrors(errors) || `message status "${entry?.Status ?? 'unknown'}"`; + const statusCode = errors[0]?.StatusCode; return createFailedReceipt(`Mailjet: ${text}`, { provider: 'mailjet', - statusCode: errors[0]?.StatusCode, + // By the status of its error; with none, Mailjet says nothing of it. + ...(statusCode === undefined && { category: 'unknown' }), + statusCode, code: errors[0]?.ErrorCode == null ? undefined : `mailjet.${errors[0].ErrorCode}`, deliveryUnknown: entry == null, providerDetails: entry ?? undefined, @@ -89,6 +92,7 @@ class MailjetTransport { if (messageId === '') { return createFailedReceipt('Mailjet answered without a MessageUUID.', { provider: 'mailjet', + category: 'unknown', deliveryUnknown: true, }); } diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 0c517d3..80116ce 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -252,6 +252,9 @@ class PoolTransport { errorMessages.length > 0 ? errorMessages : ['All transports failed to send the message.'], { provider: 'pool', + // The errors of the transports carry their categories; with none, + // the pool says nothing of what failed. + category: errors.length > 0 ? undefined : 'unknown', errors: errors.length > 0 ? errors : undefined, attempts: attemptedIndices.size, deliveryUnknown: failure.deliveryUnknown, diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index cacba07..9950d31 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -36,6 +36,7 @@ const receiptOf = (entry) => { if (entry == null || typeof entry !== 'object') { return createFailedReceipt('Postmark gave no answer for the message.', { provider: 'postmark', + category: 'unknown', deliveryUnknown: true, }); } diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index 9bf96c8..e2a4432 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -24,7 +24,7 @@ const { iterateRawMessage, RawMessageValidationError, } = require('../core/raw-message.js'); -const { createFailedReceipt } = require('../core/receipt.js'); +const { classifyReceiptError, createFailedReceipt } = require('../core/receipt.js'); const { refusalFailure } = require('../internal/refusal.js'); const { createLog } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); @@ -66,7 +66,9 @@ const failureOf = (error) => { if (error instanceof RawMessageValidationError) { return createFailedReceipt(message, { provider: 'sendmail', category: 'validation' }); } - return createFailedReceipt(message, { provider: 'sendmail' }); + // By the codes along its causes first, then by its words. + const { category, retryable } = classifyReceiptError(error); + return createFailedReceipt(message, { provider: 'sendmail', category, retryable }); }; // An address that would read as an option, as nodemailer refuses it; `--` diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 9e828cf..951b17a 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -2,7 +2,7 @@ // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee const { generateId } = require('../core/id.js'); -const { createFailedReceipt } = require('../core/receipt.js'); +const { classifyReceiptError, createFailedReceipt } = require('../core/receipt.js'); const { createSesConfig } = require('./config.js'); const { SesApiError, SesHttpClient } = require('./http-client.js'); const { convertMessage, convertRawMessage } = require('./message-converter.js'); @@ -292,9 +292,13 @@ const createSesFailure = (message, error) => { }); } + // By the codes along its causes first, then by its words. + const { category, retryable } = classifyReceiptError(error); return createFailedReceipt(message, { deliveryUnknown: isDeliveryUnknown(error), provider: 'ses', + category, + retryable, }); }; diff --git a/src/smtp2go/smtp2go-transport.js b/src/smtp2go/smtp2go-transport.js index d92e242..294f65f 100644 --- a/src/smtp2go/smtp2go-transport.js +++ b/src/smtp2go/smtp2go-transport.js @@ -42,6 +42,7 @@ const receiptOf = (json) => { if (typeof messageId !== 'string' || messageId === '') { return createFailedReceipt('SMTP2GO answered without an email_id.', { provider: 'smtp2go', + category: 'unknown', deliveryUnknown: true, }); } diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js index e7af1fb..dec8f2f 100644 --- a/src/sparkpost/sparkpost-transport.js +++ b/src/sparkpost/sparkpost-transport.js @@ -34,6 +34,7 @@ const receiptOf = (json) => { if (typeof messageId !== 'string' || messageId === '') { return createFailedReceipt('SparkPost answered without a transmission id.', { provider: 'sparkpost', + category: 'unknown', deliveryUnknown: true, }); } diff --git a/tests/unit/internal.test.js b/tests/unit/internal.test.js index a77bbf5..b153eac 100644 --- a/tests/unit/internal.test.js +++ b/tests/unit/internal.test.js @@ -641,3 +641,50 @@ test('Internal: a logger that throws is told once, by its name, not its text', ( assert.match(warnings[0][1], /threw Error; its later failures are not told/); assert.doesNotMatch(warnings[0][1], /bob@example\.com/); }); + +test('Receipts: every failed receipt the library makes names its category, or its status', () => { + // Before: receipts of the library's own conditions (an answer without an + // id, a repeat, a failure of the decorators) had none, and were classified + // by the words of their text, which can hold the caller's data: the key + // `order-timeout-1` made a receipt of ./idempotency a timeout. + const fs = require('node:fs'); + const path = require('node:path'); + const root = path.join(__dirname, '../../src'); + const files = []; + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name.endsWith('.js')) files.push(full); + } + }; + walk(root); + const unnamed = []; + for (const file of files) { + if (file.endsWith(path.join('core', 'receipt.js'))) continue; + const text = fs.readFileSync(file, 'utf8'); + let at = text.indexOf('createFailedReceipt('); + while (at !== -1) { + // The text of the call, to its closing parenthesis. + let depth = 0; + let end = at + 'createFailedReceipt'.length; + for (; end < text.length; end++) { + if (text[end] === '(') depth++; + else if (text[end] === ')' && --depth === 0) break; + } + const call = text.slice(at, end + 1); + // A category, a status it is told by, options spread that carry one, or + // the structured errors of other transports, which carry theirs. + if ( + !/\bcategory\b|\bstatusCode\b|\.\.\.|\berrors\b|^createFailedReceipt\(receiptError/.test( + call, + ) + ) { + const line = text.slice(0, at).split('\n').length; + unnamed.push(`${path.relative(root, file)}:${line}`); + } + at = text.indexOf('createFailedReceipt(', end); + } + } + assert.deepStrictEqual(unnamed, []); +}); From d5340d80a61df262aba7ab6aed73d8ec08622f6b Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:46:53 +0300 Subject: [PATCH 120/143] test: no interval of a default clock moves with the time of day The monotonic default of the rate limit, of the memory store of ./idempotency and of the replay store of ./webhooks, which only the breaker's had a test of: a jump of Date.now refills no bucket and ends no lease. Co-Authored-By: Claude Opus 5.5 --- tests/unit/decorators.test.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/decorators.test.js b/tests/unit/decorators.test.js index bfe45fd..f56494a 100644 --- a/tests/unit/decorators.test.js +++ b/tests/unit/decorators.test.js @@ -186,3 +186,31 @@ test('Decorators: a logger hears the breaker, the rate limit, idempotency and th ['warn', 'MAILER_TEE_MIRROR_FAILED', 'tee'], ]); }); + +test('Decorators: a jump of the time of day moves no interval of a default clock', async (t) => { + // Before: ratelimit, the memory stores of idempotency and of the webhooks + // measured on Date.now, so a correction of the system time refilled a + // bucket, or ended a lease, early. + let shift = 0; + const realNow = Date.now.bind(Date); + t.mock.method(Date, 'now', () => realNow() + shift); + const { RateLimitTransport } = require('../../src/ratelimit/index.js'); + const { createMemoryIdempotencyStore } = require('../../src/idempotency/index.js'); + const { createMemoryReplayStore } = require('../../src/webhooks/index.js'); + const limiter = new RateLimitTransport(fakeTransport(), { + limit: 1, + intervalMs: 3_600_000, + mode: 'reject', + }); + const one = message(); + assert.strictEqual((await limiter.send(one)).successful, true); + const idempotency = createMemoryIdempotencyStore(); + const reserved = await idempotency.reserve('k', 'f', 60_000); + const replay = createMemoryReplayStore(); + assert.strictEqual(replay.reserve('r', 60_000, 't'), 'reserved'); + shift = 2 * 3_600_000; + assert.strictEqual((await limiter.send(one)).successful, false); + assert.ok((await idempotency.reserve('k', 'f', 60_000)).record !== undefined); + assert.ok(reserved.token !== undefined); + assert.strictEqual(replay.reserve('r', 60_000, 'u'), 'pending'); +}); From 069289a114504d85cc2aba7a8e1ca1d98281c0ac Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:48:38 +0300 Subject: [PATCH 121/143] fix(api): the retry loop takes what a provider classifies as retryable classify() of a provider reached the receipt alone, so a retryable 420 of SparkPost's spent limit was not retried within the send, while its receipt said it could be. The shared client passes isRetryable to requestWithRetry: the retryable of the provider's classification when it gives one, the status otherwise. ./mailgun and ./scaleway go to 8 KB. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 4 ++-- src/internal/api-client.js | 8 +++++++- src/internal/http.js | 1 + tests/unit/sparkpost.test.js | 25 +++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index a3f6425..b64dcec 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -10,7 +10,7 @@ "./ses": 8, "./resend": 7.5, "./sendgrid": 8, - "./mailgun": 7.5, + "./mailgun": 8, "./mailtrap": 7.5, "./maileroo": 7.5, "./lettermint": 7, @@ -23,7 +23,7 @@ "./zeptomail": 7.5, "./smtp2go": 8, "./azure": 9, - "./scaleway": 7.5, + "./scaleway": 8, "./cloudflare": 8, "./pool": 3.5, "./retry": 3, diff --git a/src/internal/api-client.js b/src/internal/api-client.js index f7275a7..0973ad7 100644 --- a/src/internal/api-client.js +++ b/src/internal/api-client.js @@ -10,7 +10,7 @@ const { } = require('../core/receipt.js'); const { readText } = require('./body.js'); const { isDeliveryUnknown } = require('./delivery-unknown.js'); -const { requestWithRetry } = require('./http.js'); +const { isRetryableFailure, requestWithRetry } = require('./http.js'); const { certificateFailure, refusalFailure } = require('./refusal.js'); const { createLog } = require('./log.js'); @@ -89,6 +89,9 @@ const defaultFinalError = (provider) => (error, attempts) => cause: error, }); +const retryableOf = (error) => + typeof error.retryable === 'boolean' ? error.retryable : isRetryableFailure(error); + // The text of an error answer, or undefined when it cannot be read: the // error then keeps the status, which says more than the failure to read. const readErrorText = async (response) => { @@ -183,6 +186,9 @@ const createApiClient = (config, provider) => { fetchWithAuth(options.url, { method: options.method ?? 'POST', headers, body }, signal), read: async (response, attempt) => read(response, await readText(response), attempt), reject: async (response, attempt) => reject(response, await readErrorText(response), attempt), + // What the provider says of an error is retried as it says, a 420 of + // SparkPost's spent limit among them; the status says the rest. + isRetryable: retryableOf, createTimeoutError: timeoutError, finalError, wrapAccepted: provider.acceptedError, diff --git a/src/internal/http.js b/src/internal/http.js index cf13ecf..b3766a0 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -188,6 +188,7 @@ module.exports = { abortReason, exponentialDelayMs, isAbortError, + isRetryableFailure, isRetryableStatus, jitteredDelayMs, requestWithRetry, diff --git a/tests/unit/sparkpost.test.js b/tests/unit/sparkpost.test.js index 4a23c9a..0eb2e96 100644 --- a/tests/unit/sparkpost.test.js +++ b/tests/unit/sparkpost.test.js @@ -257,3 +257,28 @@ test('SparkPost: verify reads the account, the options are checked, the key neve /stop/, ); }); + +test('SparkPost: a 420 is retried within the send, as its classification says', async (t) => { + // Before: the classification of an error reached the receipt alone, so a + // retryable 420 was not retried within the send. + t.mock.timers.enable({ apis: ['setTimeout'] }); + t.after(() => t.mock.timers.reset()); + let calls = 0; + stubFetch(t, () => { + calls++; + return calls < 3 + ? json({ errors: [{ message: 'Exceed Sending Limit', code: '2102' }] }, 420) + : json({ results: { id: 'tx-1', total_accepted_recipients: 1 } }); + }); + const state = { receipt: undefined }; + const sending = transport({ retries: 2 }).send(message()); + sending.then((answer) => (state.receipt = answer), assert.fail); + for (let i = 0; i < 100 && state.receipt === undefined; i++) { + t.mock.timers.tick(1000); + await new Promise((resolve) => process.nextTick(resolve)); + await Promise.resolve(); + } + const receipt = await sending; + assert.strictEqual(receipt.successful, true, receipt.errorMessages?.join()); + assert.strictEqual(calls, 3); +}); From e71d2d020f018ec76deaf4de655e39dda680a6c0 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:51:22 +0300 Subject: [PATCH 122/143] fix(verify): a 403 verifies only for a provider that refuses a sending key with it verify() took a 403 for a known key that may only send, for every provider, so a key Postmark, Brevo, Mailjet, SendGrid, SMTP2GO or Azure refused with it passed. acceptForbidden is set where a key that may only send is refused the read with a 403: SES (AccessDenied for GetAccount), Mailgun (a domain sending key), SparkPost, MailerSend and Scaleway. The flow no longer goes through an error marked verified: requestWithRetry takes an accepts(status) whose answers go to read without being taken for a delivery, and a status the check does not take fails with it, retried as it says. Co-Authored-By: Claude Opus 5.5 --- src/internal/http.js | 8 +++- src/internal/verify.js | 64 +++++++++++++------------- src/mailersend/mailersend-transport.js | 5 +- src/mailgun/mailgun-transport.js | 2 + src/scaleway/scaleway-transport.js | 5 +- src/ses/ses-transport.js | 2 + src/sparkpost/sparkpost-transport.js | 5 +- tests/unit/provider-verify.test.js | 26 +++++++++++ 8 files changed, 80 insertions(+), 37 deletions(-) diff --git a/src/internal/http.js b/src/internal/http.js index b3766a0..20d008e 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -90,7 +90,9 @@ const startDeadline = (timeoutMs, signal) => { * * @param request What the provider passes in: * - `send(signal, attempt)`: one fetch, with the signal of the attempt; - * - `read(response, attempt)`: the result of a 2xx response; + * - `read(response, attempt)`: the result of a 2xx response, or of one + * `accepts(status)` takes (a check that reads a refusal), which is not + * taken for a delivery: what `read` throws for it is retried by its status; * - `reject(response, attempt)`: the error of any other response; * - `retries`, `timeoutMs`, `signal`; * - `isRetryable(error)`: by default, a failure without a status or with a @@ -132,7 +134,9 @@ const requestWithRetry = async (request) => { const response = await request.send(deadline.signal, attempt); hasResponse = true; isAccepted = response.ok; - if (response.ok) return await request.read(response, attempt); + if (response.ok || request.accepts?.(response.status) === true) { + return await request.read(response, attempt); + } failure = await request.reject(response, attempt); } catch (error) { const isTimeout = deadline.isExpired && !signal?.aborted && isAbortError(error); diff --git a/src/internal/verify.js b/src/internal/verify.js index 484dd28..b07a93f 100644 --- a/src/internal/verify.js +++ b/src/internal/verify.js @@ -22,43 +22,41 @@ const failureOf = (provider, status, text) => { /** * Verifies the credentials of an HTTP provider with one read of its API. A - * 2xx verifies them; so does a 403, which says the key is known but may not - * read what was asked (a key that may only send); a 401, or any other - * status, fails. + * 2xx verifies them; so does a 403 with `acceptForbidden`, for a provider + * whose keys that may only send are refused that read with it; a 401, or + * any other status, fails, unless `check` takes it. * * @param request `provider`, the name in the error; `send(signal)`, the * authenticated request; `check(status, body)`, which may throw on a 2xx or - * return `true` to accept another status; and `retries`, `timeoutMs`, - * `signal` and `log` of the transport. + * return `true` to take another status; `acceptForbidden`; and `retries`, + * `timeoutMs`, `signal` and `log` of the transport. * @throws {Error} With the `statusCode` of the answer, if the check fails. * @internal */ -const verifyByRequest = async (request) => { - let answer; - try { - answer = await requestWithRetry({ - log: request.log, - signal: request.signal, - retries: request.retries, - timeoutMs: request.timeoutMs, - isIdempotent: true, - send: request.send, - read: async (response) => ({ status: response.status, body: await readText(response) }), - reject: async (response) => { - const text = await readText(response); - if (response.status === 403 || request.check?.(response.status, text) === true) { - // Not retried, as its status says, and caught below. - return Object.assign(new Error('Verified.'), { verified: true, statusCode: 403 }); - } - return failureOf(request.provider, response.status, text); - }, - }); - } catch (error) { - if (error?.verified === true) return; - throw error; - } - request.check?.(answer.status, answer.body); -}; +const verifyByRequest = (request) => + requestWithRetry({ + log: request.log, + signal: request.signal, + retries: request.retries, + timeoutMs: request.timeoutMs, + isIdempotent: true, + send: request.send, + // Every answer is read, as the check reads the body of a refusal; one it + // does not take fails with its status, retried as that says. + accepts: () => true, + read: async (response) => { + const { status } = response; + const body = await readText(response); + if (status >= 200 && status <= 299) { + request.check?.(status, body); + return; + } + if (status === 403 && request.acceptForbidden === true) return; + if (request.check?.(status, body) === true) return; + throw failureOf(request.provider, status, body); + }, + reject: async (response) => failureOf(request.provider, response.status, ''), + }); /** * Verifies the credentials of a transport with one authenticated read of @@ -68,7 +66,8 @@ const verifyByRequest = async (request) => { * @param label The provider's name, for the error. * @param url The URL read. * @param options The caller's options. - * @param request `init` of the request in place of a GET, and `check`. + * @param request `init` of the request in place of a GET, `check` and + * `acceptForbidden`. * @internal */ const verifyByGet = async (transport, label, url, options, request = {}) => { @@ -78,6 +77,7 @@ const verifyByGet = async (transport, label, url, options, request = {}) => { provider: label, send: (signal) => transport.httpClient.fetchWithAuth(url, init, signal), check: request.check, + acceptForbidden: request.acceptForbidden, retries: transport.config.retries, timeoutMs: transport.config.timeout, signal: options?.signal, diff --git a/src/mailersend/mailersend-transport.js b/src/mailersend/mailersend-transport.js index 5eb27ce..290689f 100644 --- a/src/mailersend/mailersend-transport.js +++ b/src/mailersend/mailersend-transport.js @@ -71,7 +71,10 @@ class MailerSendTransport { * @throws {Error} If MailerSend refuses the token or cannot be reached. */ verify(options) { - return verifyByGet(this, 'MailerSend', `${this.config.baseUrl}/v1/domains`, options); + // A token without the access of domains is refused it with a 403. + return verifyByGet(this, 'MailerSend', `${this.config.baseUrl}/v1/domains`, options, { + acceptForbidden: true, + }); } } diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index 74b0efa..952d575 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -84,6 +84,8 @@ class MailgunTransport { const url = `${this.config.baseUrl}/domains/${encodeURIComponent(this.config.domain)}`; await verifyByRequest({ provider: 'Mailgun', + // A 403 is a known key that may not read: a domain sending key is refused the read of its domain. + acceptForbidden: true, send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), retries: this.config.retries, timeoutMs: this.config.timeout, diff --git a/src/scaleway/scaleway-transport.js b/src/scaleway/scaleway-transport.js index 7c03978..56f20e5 100644 --- a/src/scaleway/scaleway-transport.js +++ b/src/scaleway/scaleway-transport.js @@ -69,7 +69,10 @@ class ScalewayTransport { */ verify(options) { const query = `project_id=${encodeURIComponent(this.config.projectId)}&page_size=1`; - return verifyByGet(this, 'Scaleway', `${this.httpClient.base}/domains?${query}`, options); + // A key whose policy may send but not read the domains is refused with a 403. + return verifyByGet(this, 'Scaleway', `${this.httpClient.base}/domains?${query}`, options, { + acceptForbidden: true, + }); } } diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 951b17a..b3f5934 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -96,6 +96,8 @@ class SesTransport { const url = `https://email.${this.config.region}.amazonaws.com/v2/email/account`; await verifyByRequest({ provider: 'SES', + // A 403 is a known key that may not read: an IAM key that may only send is refused GetAccount (AccessDenied). + acceptForbidden: true, send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), retries: this.config.retries, timeoutMs: this.config.timeout, diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js index dec8f2f..3dd587f 100644 --- a/src/sparkpost/sparkpost-transport.js +++ b/src/sparkpost/sparkpost-transport.js @@ -116,7 +116,10 @@ class SparkPostTransport { * @throws {Error} If SparkPost refuses the key or cannot be reached. */ verify(options) { - return verifyByGet(this, 'SparkPost', `${this.config.baseUrl}/api/v1/account`, options); + // An API key without the grant of the account is refused it with a 403. + return verifyByGet(this, 'SparkPost', `${this.config.baseUrl}/api/v1/account`, options, { + acceptForbidden: true, + }); } } diff --git a/tests/unit/provider-verify.test.js b/tests/unit/provider-verify.test.js index bda886c..7719ec7 100644 --- a/tests/unit/provider-verify.test.js +++ b/tests/unit/provider-verify.test.js @@ -152,3 +152,29 @@ test('Verify: an aborted signal rejects before any request', async (t) => { } assert.strictEqual(requests.length, 0); }); + +test('Verify: a 403 verifies only where a key that may only send is refused with it', async (t) => { + // Before: a 403 verified the credentials of every provider, so a key + // Postmark or Brevo refused passed verify(). + const { PostmarkTransport } = require('../../src/postmark/index.js'); + const { BrevoTransport } = require('../../src/brevo/index.js'); + const { SparkPostTransport } = require('../../src/sparkpost/index.js'); + const { MailerSendTransport } = require('../../src/mailersend/index.js'); + const { ScalewayTransport } = require('../../src/scaleway/index.js'); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ message: 'Forbidden' }, { status: 403 }), + ); + const refused = [ + new PostmarkTransport({ serverToken: 'token', retries: 0 }), + new BrevoTransport({ apiKey: 'key', retries: 0 }), + ]; + for (const transport of refused) { + await assert.rejects(transport.verify(), (error) => error.statusCode === 403, transport.id); + } + const taken = [ + new SparkPostTransport({ apiKey: 'key', retries: 0 }), + new MailerSendTransport({ apiKey: 'key', retries: 0 }), + new ScalewayTransport({ secretKey: 'key', projectId: 'p', retries: 0 }), + ]; + for (const transport of taken) await transport.verify(); +}); From 8f1d01dc434eb5237e942e1e315cf1abe3b1c2c2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:54:15 +0300 Subject: [PATCH 123/143] refactor(ses): the client on createApiClient, SigV4 in its authorize SES was the HTTP provider left with a client of its own. Its SigV4 signature is the authorize of the shared client, made anew for each attempt so that X-Amz-Date is the attempt's, the 200 SES answers is checked in the read, and SesApiError extends ProviderApiError, its errors the providerDetails. The receipts are those of before; the test of the clients now leaves JMAP alone out. ./ses goes to 9 KB for the shared client. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 2 +- src/mailgun/mailgun-transport.js | 3 +- src/ses/http-client.js | 128 ++++++++++------------------- src/ses/ses-transport.js | 3 +- tests/unit/api.test.js | 6 +- tests/unit/provider-verify.test.js | 5 +- 6 files changed, 52 insertions(+), 95 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index b64dcec..42aaeef 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -7,7 +7,7 @@ "./smtp": 23.5, "./sendmail": 12, "./jmap": 13, - "./ses": 8, + "./ses": 9, "./resend": 7.5, "./sendgrid": 8, "./mailgun": 8, diff --git a/src/mailgun/mailgun-transport.js b/src/mailgun/mailgun-transport.js index 952d575..d1e9f0d 100644 --- a/src/mailgun/mailgun-transport.js +++ b/src/mailgun/mailgun-transport.js @@ -84,7 +84,8 @@ class MailgunTransport { const url = `${this.config.baseUrl}/domains/${encodeURIComponent(this.config.domain)}`; await verifyByRequest({ provider: 'Mailgun', - // A 403 is a known key that may not read: a domain sending key is refused the read of its domain. + // A 403 is a known key that may not read: a domain sending key is refused + // the read of its domain. acceptForbidden: true, send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), retries: this.config.retries, diff --git a/src/ses/http-client.js b/src/ses/http-client.js index 4ff649d..d6d30b5 100644 --- a/src/ses/http-client.js +++ b/src/ses/http-client.js @@ -1,59 +1,35 @@ // Ported from upyo v0.7.0 (packages/ses/src/http-client.ts) // https://github.com/dahlia/upyo — MIT, Copyright 2025 Hong Minhee +// Changed after the port: the request goes through createApiClient of +// internal/api-client.js, as that of every HTTP provider does, its SigV4 +// signature made by the authorize of each attempt, and SesApiError extends +// its error; the messages and the errors of upstream stay. const { createHash, createHmac } = require('node:crypto'); const { parseRetryAfter } = require('../core/receipt.js'); -const { requestWithRetry } = require('../internal/http.js'); -const { createLog } = require('../internal/log.js'); -const { readText } = require('../internal/body.js'); +const { ProviderApiError, createApiClient } = require('../internal/api-client.js'); class SesHttpClient { config; #signingKey = null; + #request; constructor(config) { this.config = config; - this.log = createLog(config.logger, { transport: 'ses' }); - } - - sendMessage(messageData, signal, onRetry) { - const region = this.config.region; - const url = `https://email.${region}.amazonaws.com/v2/email/outbound-emails`; - - return this.makeRequest( - url, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(messageData), - signal, + const client = createApiClient(config, { + id: 'ses', + label: 'SES', + // Each attempt is signed anew: X-Amz-Date is the time of the attempt. + authorize: async (headers, request) => { + const credentials = await this.getCredentials(); + const signed = this.signRequest( + request.url, + { method: request.method, headers, body: request.body }, + credentials, + ); + for (const [name, value] of signed) headers.set(name, value); }, - onRetry, - ); - } - - /** - * Makes an HTTP request to the SES API with retry logic. - * - * @param url The URL to make the request to. - * @param options Fetch options. - * @returns Promise that resolves to the parsed response. - * @throws {DOMException} If the caller aborts the request. - * @throws {SesApiError} If SES returns a client error or all retry attempts - * are exhausted. - */ - makeRequest(url, options, onRetry) { - return requestWithRetry({ - log: this.log, - signal: options.signal, - onRetry, - retries: this.config.retries, - timeoutMs: this.config.timeout, - send: (signal) => this.fetchWithAuth(url, options, signal), - read: async (response, attempt) => { - const text = await readText(response); + read: (response, text, attempt) => { if (response.status !== 200) throw createSesApiError(response, text, attempt); return { statusCode: response.status, @@ -61,10 +37,8 @@ class SesHttpClient { headers: this.headersToRecord(response.headers), }; }, - reject: async (response, attempt) => - createSesApiError(response, await readText(response), attempt), - createTimeoutError: () => - new Error(`SES API request timed out after ${this.config.timeout} ms.`), + reject: (response, text, attempt) => createSesApiError(response, text ?? '', attempt), + timeoutError: () => new Error(`SES API request timed out after ${config.timeout} ms.`), finalError: (error, attempts) => error instanceof SesApiError ? error @@ -72,41 +46,25 @@ class SesHttpClient { cause: error, }), }); + this.log = client.log; + this.fetchWithAuth = client.fetchWithAuth; + this.#request = client.request; } /** - * Makes a signed fetch request to the SES API. + * Sends a message through the SES v2 API. * - * @param url The URL to make the request to. - * @param options Fetch options. - * @param signal The signal of the attempt, with its deadline. - * @returns Promise that resolves to the fetch response. + * @param messageData The JSON data of SendEmail. + * @param signal Optional AbortSignal for cancellation. + * @param onRetry Optional callback told of each retry. + * @returns Promise that resolves to the SES response. + * @throws {DOMException} If the caller aborts the request. + * @throws {SesApiError} If SES returns a client error or all retry attempts + * are exhausted. */ - async fetchWithAuth(url, options, signal) { - const credentials = await this.getCredentials(); - const headers = new Headers(options.headers); - - const signedHeaders = this.signRequest( - url, - { - ...options, - headers, - }, - credentials, - ); - - for (const [key, value] of Object.entries(this.config.headers)) { - signedHeaders.set(key, value); - } - - // A redirect comes back as a response rather than being followed, so the - // credentials in the headers never reach another origin. - return await (this.config.fetch ?? globalThis.fetch)(url, { - ...options, - headers: this.headersToRecord(signedHeaders), - signal, - redirect: 'manual', - }); + sendMessage(messageData, signal, onRetry) { + const url = `https://email.${this.config.region}.amazonaws.com/v2/email/outbound-emails`; + return this.#request({ url, body: messageData, signal, onRetry }); } async getCredentials() { @@ -225,19 +183,19 @@ const createSesApiError = (response, text, attempt) => { ); }; -class SesApiError extends Error { - statusCode; +class SesApiError extends ProviderApiError { errors; - retryAfterMilliseconds; - attempts; constructor(message, statusCode, errors, retryAfterMilliseconds, attempts, options) { - super(message, options); + super('ses', message, { + statusCode, + retryAfterMilliseconds, + attempts, + cause: options?.cause, + providerDetails: errors, + }); this.name = 'SesApiError'; - this.statusCode = statusCode; this.errors = errors; - this.retryAfterMilliseconds = retryAfterMilliseconds; - this.attempts = attempts; } } diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index b3f5934..28b0536 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -96,7 +96,8 @@ class SesTransport { const url = `https://email.${this.config.region}.amazonaws.com/v2/email/account`; await verifyByRequest({ provider: 'SES', - // A 403 is a known key that may not read: an IAM key that may only send is refused GetAccount (AccessDenied). + // A 403 is a known key that may not read: an IAM key that may only send + // is refused GetAccount (AccessDenied). acceptForbidden: true, send: (signal) => this.httpClient.fetchWithAuth(url, { method: 'GET' }, signal), retries: this.config.retries, diff --git a/tests/unit/api.test.js b/tests/unit/api.test.js index 39169f0..a9f0c9a 100644 --- a/tests/unit/api.test.js +++ b/tests/unit/api.test.js @@ -147,10 +147,10 @@ test('API: a refusal of the converter is a non-retryable unsupported failure', ( assert.notStrictEqual(range.errors[0].category, 'unsupported'); }); -test('API: every HTTP provider requests through createApiClient, but SES and JMAP', () => { +test('API: every HTTP provider requests through createApiClient, but JMAP', () => { // Before: the seven clients ported from upyo each carried their own fetch, // retry wiring, timeouts and truncation, and a fix of one (bug 29) missed - // another. SES signs each attempt with SigV4 and JMAP has its own session. + // another. SES signs each attempt in its authorize; JMAP has its own session. const fs = require('node:fs'); const path = require('node:path'); const src = path.join(__dirname, '../../src'); @@ -162,7 +162,7 @@ test('API: every HTTP provider requests through createApiClient, but SES and JMA if (text.includes('requestWithRetry')) direct.push(entry.name); else assert.match(text, /createApiClient\(/, entry.name); } - assert.deepStrictEqual(direct.sort(), ['jmap', 'ses']); + assert.deepStrictEqual(direct, ['jmap']); }); test('API: the errors of 1.0 keep their names, arguments and fields over the shared error', () => { diff --git a/tests/unit/provider-verify.test.js b/tests/unit/provider-verify.test.js index 7719ec7..c8fc20e 100644 --- a/tests/unit/provider-verify.test.js +++ b/tests/unit/provider-verify.test.js @@ -49,10 +49,7 @@ test('Verify: SES reads the account, signed, and takes AccessDenied for a known await ses().verify(); assert.strictEqual(requests[0].url, 'https://email.eu-west-1.amazonaws.com/v2/email/account'); assert.strictEqual(requests[0].method, 'GET'); - assert.match( - requests[0].headers.authorization ?? requests[0].headers.Authorization, - /^AWS4-HMAC-SHA256 /, - ); + assert.match(new Headers(requests[0].headers).get('authorization'), /^AWS4-HMAC-SHA256 /); await ses().verify(); await assert.rejects(ses().verify(), (error) => { assert.strictEqual(error.statusCode, 401); From bf963c5be8407ce81bab38caca0d2be983677905 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:56:49 +0300 Subject: [PATCH 124/143] refactor: one isCallerAbort, isAbortError and abortReason, in internal/abort.js The check of a caller's abort was written in the shared client, the decorators (isAbortOf), the pool, SES and sendmail, the test of an AbortError in http.js, the pool and SES, and the reason of an aborted signal in sleep.js and retry. Each is one function of internal/abort.js now, which sleep.js passes on. A module more is a wrapper more in a bundle: ./ses goes to 9.5 KB, ./resend to 8, ./lettermint to 7.5, ./retry to 3.5, ./idempotency to 4.5 and ./hooks to 3. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 12 +++++----- src/breaker/breaker-transport.js | 4 ++-- src/idempotency/idempotency-transport.js | 4 ++-- src/internal/abort.js | 30 ++++++++++++++++++++++++ src/internal/api-client.js | 9 +------ src/internal/decorator.js | 11 ++------- src/internal/http.js | 4 +--- src/internal/sleep.js | 12 ++-------- src/pool/pool-transport.js | 10 +------- src/retry/retry-transport.js | 9 +++---- src/sendmail/sendmail-transport.js | 5 +--- src/ses/ses-transport.js | 9 +------ 12 files changed, 52 insertions(+), 67 deletions(-) create mode 100644 src/internal/abort.js diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 42aaeef..358e620 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -7,13 +7,13 @@ "./smtp": 23.5, "./sendmail": 12, "./jmap": 13, - "./ses": 9, - "./resend": 7.5, + "./ses": 9.5, + "./resend": 8, "./sendgrid": 8, "./mailgun": 8, "./mailtrap": 7.5, "./maileroo": 7.5, - "./lettermint": 7, + "./lettermint": 7.5, "./plunk": 7, "./postmark": 8, "./brevo": 7.5, @@ -26,13 +26,13 @@ "./scaleway": 8, "./cloudflare": 8, "./pool": 3.5, - "./retry": 3, + "./retry": 3.5, "./mock": 2, "./otel": 5, "./logger": 2.5, "./breaker": 3.5, "./ratelimit": 4, - "./idempotency": 4, - "./hooks": 2.5, + "./idempotency": 4.5, + "./hooks": 3, "./tee": 3.5 } diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 5230c6e..7658029 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -16,7 +16,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, isAbortOf, verifyWrapped } = require('../internal/decorator.js'); +const { disposeOf, isCallerAbort, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createBreakerConfig } = require('./config.js'); @@ -239,7 +239,7 @@ class CircuitBreakerTransport { } #thrown(error, signal, entry) { - if (isAbortOf(error, signal)) { + if (isCallerAbort(error, signal)) { // The caller gave up: nothing is learned of the provider. if (entry.probe && entry.period === this.#period) this.#probing = false; return; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index 51b9765..57fbec0 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -16,7 +16,7 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isRawTransport } = require('../core/transport.js'); const { describeThrown, warnThrown } = require('../internal/callback.js'); const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, isAbortOf, messageOf, verifyWrapped } = require('../internal/decorator.js'); +const { disposeOf, isCallerAbort, messageOf, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { gatedSendMany } = require('../internal/gate.js'); @@ -256,7 +256,7 @@ class IdempotencyTransport { // frees it. async #thrown(entry, error, signal) { if (entry.key === undefined) return; - if (isAbortOf(error, signal)) { + if (isCallerAbort(error, signal)) { const receipt = createFailedReceipt( 'The send was aborted after it began; whether the message went is not known.', { diff --git a/src/internal/abort.js b/src/internal/abort.js new file mode 100644 index 0000000..75078ec --- /dev/null +++ b/src/internal/abort.js @@ -0,0 +1,30 @@ +// Not ported: the abort of a caller's signal, read the one way every +// transport and decorator reads it: its reason, and whether an error is it. + +/** + * The reason of an aborted signal, or an `AbortError` when it has none. + * + * @param signal The aborted signal. + * @returns The value to reject or throw with. + * @internal + */ +const abortReason = (signal) => + signal?.reason ?? new DOMException('The operation was aborted.', 'AbortError'); + +/** + * Whether a value is an `AbortError`. + * + * @internal + */ +const isAbortError = (error) => error instanceof Error && error.name === 'AbortError'; + +/** + * Whether an error is the abort of the caller's own signal: its reason, or + * an `AbortError` while it is aborted. + * + * @internal + */ +const isCallerAbort = (error, signal) => + signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); + +module.exports = { abortReason, isAbortError, isCallerAbort }; diff --git a/src/internal/api-client.js b/src/internal/api-client.js index 0973ad7..6d8624f 100644 --- a/src/internal/api-client.js +++ b/src/internal/api-client.js @@ -11,6 +11,7 @@ const { const { readText } = require('./body.js'); const { isDeliveryUnknown } = require('./delivery-unknown.js'); const { isRetryableFailure, requestWithRetry } = require('./http.js'); +const { isCallerAbort } = require('./abort.js'); const { certificateFailure, refusalFailure } = require('./refusal.js'); const { createLog } = require('./log.js'); @@ -269,14 +270,6 @@ const parseAnswer = (text, invalid) => { const truncateErrorText = (text) => text.length > MAX_ERROR_TEXT ? `${text.slice(0, MAX_ERROR_TEXT)}...` : text; -/** - * Whether an error is the abort of the caller's own signal. - * - * @internal - */ -const isCallerAbort = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); - module.exports = { ProviderApiError, categoryOf, diff --git a/src/internal/decorator.js b/src/internal/decorator.js index 1cdc871..d9aedc4 100644 --- a/src/internal/decorator.js +++ b/src/internal/decorator.js @@ -3,6 +3,7 @@ // and of what was thrown. const { isVerifiableTransport } = require('../core/transport.js'); +const { isCallerAbort } = require('./abort.js'); /** * Disposes of a transport that supports disposal, asynchronous or not. @@ -25,14 +26,6 @@ const verifyWrapped = async (transport, options) => { if (isVerifiableTransport(transport)) await transport.verify(options); }; -/** - * Whether an error is the abort of the caller's signal. - * - * @internal - */ -const isAbortOf = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); - /** * The text of what was thrown. * @@ -40,4 +33,4 @@ const isAbortOf = (error, signal) => */ const messageOf = (error) => (error instanceof Error ? error.message : String(error)); -module.exports = { disposeOf, isAbortOf, messageOf, verifyWrapped }; +module.exports = { disposeOf, isCallerAbort, messageOf, verifyWrapped }; diff --git a/src/internal/http.js b/src/internal/http.js index 20d008e..53e014d 100644 --- a/src/internal/http.js +++ b/src/internal/http.js @@ -6,11 +6,9 @@ const { combineSignals } = require('../core/abort-signal.js'); const { isCertificateFailure } = require('../core/receipt.js'); const { markDeliveryUnknown } = require('./delivery-unknown.js'); const { isUnsentRequestError } = require('./unsent.js'); -const { abortReason, sleep, timerMs } = require('./sleep.js'); +const { abortReason, isAbortError, sleep, timerMs } = require('./sleep.js'); const { redactAddresses } = require('./redact-addresses.js'); -const isAbortError = (error) => error instanceof Error && error.name === 'AbortError'; - /** * The delay before retry `attempt` (from 0): one second, doubled each time. * diff --git a/src/internal/sleep.js b/src/internal/sleep.js index 3fbaf41..78ed1cb 100644 --- a/src/internal/sleep.js +++ b/src/internal/sleep.js @@ -2,15 +2,7 @@ // five ways, some deaf to the AbortSignal or rejecting with an error of their // own instead of its reason. -/** - * The reason of an aborted signal, or an `AbortError` when it has none. - * - * @param signal The aborted signal. - * @returns The value to reject or throw with. - * @internal - */ -const abortReason = (signal) => - signal?.reason ?? new DOMException('The operation was aborted.', 'AbortError'); +const { abortReason, isAbortError, isCallerAbort } = require('./abort.js'); /** * Waits, and stops waiting when the signal aborts, rejecting with its reason. @@ -52,4 +44,4 @@ const MAX_TIMER_MS = 2_147_483_647; */ const timerMs = (milliseconds) => (milliseconds > MAX_TIMER_MS ? MAX_TIMER_MS : milliseconds); -module.exports = { abortReason, sleep, timerMs }; +module.exports = { abortReason, isAbortError, isCallerAbort, sleep, timerMs }; diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 80116ce..61b95f4 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -12,7 +12,7 @@ const { RoundRobinStrategy } = require('./strategies/round-robin-strategy.js'); const { WeightedStrategy } = require('./strategies/weighted-strategy.js'); const { PriorityStrategy } = require('./strategies/priority-strategy.js'); const { SelectorStrategy } = require('./strategies/selector-strategy.js'); -const { timerMs } = require('../internal/sleep.js'); +const { isAbortError, isCallerAbort, timerMs } = require('../internal/sleep.js'); /** * Pool transport that combines multiple transports with various load balancing @@ -449,10 +449,6 @@ const createAbortError = () => { return new DOMException('The operation was aborted.', 'AbortError'); }; -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - // Appends the structured errors of a failed receipt, or ones made from its // messages, each naming the transport. const appendReceiptErrors = (target, receipt, provider) => { @@ -547,10 +543,6 @@ const isFailedReceipt = (value) => { ); }; -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - // A rejection of the message itself, as opposed to one of the transport // (authentication, rate limit, outage) or of what it can express // (`unsupported`), which another transport may not share. diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index 1d74d68..563d2e1 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -12,6 +12,7 @@ const { isRawTransport, isVerifiableTransport } = require('../core/transport.js' const { refuseRaw } = require('../internal/capabilities.js'); const { createLog } = require('../internal/log.js'); const { createRetryConfig } = require('./config.js'); +const { abortReason } = require('../internal/sleep.js'); /** * Transport decorator that retries transient delivery failures. @@ -593,12 +594,12 @@ const toAsyncIterator = (values) => { }; const raceWithAbort = (promise, signal) => { - if (signal.aborted) return Promise.reject(getAbortReason(signal)); + if (signal.aborted) return Promise.reject(abortReason(signal)); return new Promise((resolve, reject) => { const abort = () => { cleanup(); - reject(getAbortReason(signal)); + reject(abortReason(signal)); }; const cleanup = () => { signal.removeEventListener('abort', abort); @@ -618,10 +619,6 @@ const raceWithAbort = (promise, signal) => { }); }; -const getAbortReason = (signal) => { - return signal.reason ?? new DOMException('The operation was aborted.', 'AbortError'); -}; - const toReceiptError = (value) => { return isReceiptError(value) ? value : undefined; }; diff --git a/src/sendmail/sendmail-transport.js b/src/sendmail/sendmail-transport.js index e2a4432..df67ff1 100644 --- a/src/sendmail/sendmail-transport.js +++ b/src/sendmail/sendmail-transport.js @@ -30,7 +30,7 @@ const { createLog } = require('../internal/log.js'); const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); -const { abortReason, timerMs } = require('../internal/sleep.js'); +const { abortReason, isCallerAbort, timerMs } = require('../internal/sleep.js'); const { Fifo } = require('../internal/gate.js'); const { composeMessage } = require('../mime/index.js'); const { createSendmailConfig } = require('./config.js'); @@ -56,9 +56,6 @@ const EXIT_STATUSES = new Map([ [127, { category: 'configuration', retryable: false }], ]); -const isCallerAbort = (error, signal) => - signal?.aborted === true && (error === signal.reason || error?.name === 'AbortError'); - const failureOf = (error) => { const refusal = refusalFailure('sendmail', error); if (refusal !== undefined) return refusal; diff --git a/src/ses/ses-transport.js b/src/ses/ses-transport.js index 28b0536..1450f25 100644 --- a/src/ses/ses-transport.js +++ b/src/ses/ses-transport.js @@ -12,6 +12,7 @@ const { isDeliveryUnknown } = require('../internal/delivery-unknown.js'); const { chunks } = require('../internal/batch.js'); const { certificateFailure, refusalFailure } = require('../internal/refusal.js'); const { verifyByRequest } = require('../internal/verify.js'); +const { isCallerAbort } = require('../internal/sleep.js'); /** * Amazon SES email transport implementation. @@ -305,12 +306,4 @@ const createSesFailure = (message, error) => { }); }; -const isCallerAbort = (error, signal) => { - return signal?.aborted === true && (isAbortError(error) || error === signal.reason); -}; - -const isAbortError = (error) => { - return error instanceof Error && error.name === 'AbortError'; -}; - module.exports = { SesTransport }; From fc4a5ba3d614762730a5d73dd1b2e1b05fe34283 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 12:59:11 +0300 Subject: [PATCH 125/143] refactor: the decorators start sendRaw, verify and dispose in one way The start of sendRaw() of a decorator (the caller's abort heard, then the refusal of a raw message its transport sends none of) was written seven times; it is refuseUnlessRaw of internal/decorator.js. Retry, OpenTelemetry and the logger verify and dispose through verifyWrapped and disposeOf, as the decorators of 1.1 do, in place of copies of their own; verifyWrapped takes no transport, as the logger may have none. ./otel goes to 5.5 KB and ./logger to 3 for the module they share. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 4 ++-- src/breaker/breaker-transport.js | 15 ++++++------ src/hooks/hooks-transport.js | 13 +++++++---- src/idempotency/idempotency-transport.js | 14 ++++++++---- src/internal/decorator.js | 22 ++++++++++++++---- src/logger/logger-transport.js | 29 ++++-------------------- src/otel/opentelemetry-transport.js | 26 +++++---------------- src/ratelimit/ratelimit-transport.js | 8 +++---- src/retry/retry-transport.js | 28 +++++------------------ src/tee/tee-transport.js | 7 +++--- 10 files changed, 68 insertions(+), 98 deletions(-) diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index 358e620..f11ebf6 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -28,8 +28,8 @@ "./pool": 3.5, "./retry": 3.5, "./mock": 2, - "./otel": 5, - "./logger": 2.5, + "./otel": 5.5, + "./logger": 3, "./breaker": 3.5, "./ratelimit": 4, "./idempotency": 4.5, diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 7658029..9db795f 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -13,10 +13,13 @@ // batch that throws counts once. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); -const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, isCallerAbort, verifyWrapped } = require('../internal/decorator.js'); +const { + disposeOf, + isCallerAbort, + refuseUnlessRaw, + verifyWrapped, +} = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createBreakerConfig } = require('./config.js'); @@ -103,10 +106,8 @@ class CircuitBreakerTransport { * `configuration`, which the circuit does not count. */ async sendRaw(message, options) { - if (!isRawTransport(this.wrappedTransport)) { - options?.signal?.throwIfAborted(); - return refuseRaw(this.wrappedTransport); - } + const refusal = refuseUnlessRaw(this.wrappedTransport, options); + if (refusal !== undefined) return refusal; return this.#guard(options, (sendOptions) => this.wrappedTransport.sendRaw(message, sendOptions), ); diff --git a/src/hooks/hooks-transport.js b/src/hooks/hooks-transport.js index 5d1b18b..eb8dcd0 100644 --- a/src/hooks/hooks-transport.js +++ b/src/hooks/hooks-transport.js @@ -14,10 +14,13 @@ const { checkMessage } = require('../core/message.js'); const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport } = require('../core/transport.js'); const { warnThrown } = require('../internal/callback.js'); -const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, messageOf, verifyWrapped } = require('../internal/decorator.js'); +const { + disposeOf, + messageOf, + refuseUnlessRaw, + verifyWrapped, +} = require('../internal/decorator.js'); const { gatedSendMany } = require('../internal/gate.js'); const { checkOptionalFunction } = require('../internal/options.js'); @@ -73,8 +76,8 @@ class HooksTransport { * `configuration`. */ async sendRaw(message, options) { - options?.signal?.throwIfAborted(); - if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const refusal = refuseUnlessRaw(this.wrappedTransport, options); + if (refusal !== undefined) return refusal; const context = this.#contextOf('sendRaw', options?.signal); const before = await this.#before('beforeSendRaw', message, context); if (before.receipt !== undefined) return before.receipt; diff --git a/src/idempotency/idempotency-transport.js b/src/idempotency/idempotency-transport.js index 57fbec0..5a5d8a3 100644 --- a/src/idempotency/idempotency-transport.js +++ b/src/idempotency/idempotency-transport.js @@ -13,10 +13,14 @@ // was asked about alone. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport } = require('../core/transport.js'); const { describeThrown, warnThrown } = require('../internal/callback.js'); -const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, isCallerAbort, messageOf, verifyWrapped } = require('../internal/decorator.js'); +const { + disposeOf, + isCallerAbort, + messageOf, + refuseUnlessRaw, + verifyWrapped, +} = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); const { gatedSendMany } = require('../internal/gate.js'); @@ -75,8 +79,8 @@ class IdempotencyTransport { * message has no `idempotencyKey`, so without `key` it goes as it is. */ async sendRaw(message, options) { - options?.signal?.throwIfAborted(); - if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const refusal = refuseUnlessRaw(this.wrappedTransport, options); + if (refusal !== undefined) return refusal; const entry = await this.#reserve(message); if (entry.receipt !== undefined) return entry.receipt; return this.#sendReserved(entry, options, (sendOptions) => diff --git a/src/internal/decorator.js b/src/internal/decorator.js index d9aedc4..cfe75ac 100644 --- a/src/internal/decorator.js +++ b/src/internal/decorator.js @@ -2,7 +2,8 @@ // wrap: its disposal, its verification, and the reading of a caller's abort // and of what was thrown. -const { isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { refuseRaw } = require('./capabilities.js'); const { isCallerAbort } = require('./abort.js'); /** @@ -17,13 +18,14 @@ const disposeOf = async (transport) => { }; /** - * Verifies a transport that can verify; one that cannot has nothing to check. + * Verifies a transport that can verify; one that cannot, or none, has nothing + * to check. * * @internal */ const verifyWrapped = async (transport, options) => { options?.signal?.throwIfAborted(); - if (isVerifiableTransport(transport)) await transport.verify(options); + if (transport != null && isVerifiableTransport(transport)) await transport.verify(options); }; /** @@ -33,4 +35,16 @@ const verifyWrapped = async (transport, options) => { */ const messageOf = (error) => (error instanceof Error ? error.message : String(error)); -module.exports = { disposeOf, isCallerAbort, messageOf, verifyWrapped }; +/** + * The start of `sendRaw()` of a decorator: the caller's abort heard, then the + * refusal of a raw message when the wrapped transport sends none, or + * `undefined` when it does. + * + * @internal + */ +const refuseUnlessRaw = (transport, options) => { + options?.signal?.throwIfAborted(); + return isRawTransport(transport) ? undefined : refuseRaw(transport); +}; + +module.exports = { disposeOf, isCallerAbort, messageOf, refuseUnlessRaw, verifyWrapped }; diff --git a/src/logger/logger-transport.js b/src/logger/logger-transport.js index d16af9d..c883f5b 100644 --- a/src/logger/logger-transport.js +++ b/src/logger/logger-transport.js @@ -8,8 +8,9 @@ const { redactAddresses } = require('../internal/redact-addresses.js'); const { warnLimited } = require('../internal/warn.js'); const { resolveLoggerTransportOptions } = require('./config.js'); const { PendingQueue } = require('./pending-queue.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); +const { isRawTransport } = require('../core/transport.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); // A raw message has its envelope where a message has its recipients. const isRawMessage = (message) => message.envelope != null && message.recipients == null; @@ -87,10 +88,8 @@ class LoggerTransport { * Verifies the wrapped transport, which has nothing to check when it is * absent or cannot verify. */ - async verify(options) { - options?.signal?.throwIfAborted(); - const transport = this.wrappedTransport; - if (transport != null && isVerifiableTransport(transport)) await transport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** @@ -148,17 +147,7 @@ class LoggerTransport { * management. */ async [Symbol.asyncDispose]() { - const transport = this.wrappedTransport; - if (transport == null) return; - - if (isAsyncDisposable(transport)) { - await transport[Symbol.asyncDispose](); - return; - } - - if (isDisposable(transport)) { - transport[Symbol.dispose](); - } + await disposeOf(this.wrappedTransport); } async sendOne( @@ -398,14 +387,6 @@ const redactFailure = (receipt) => { return { ...receipt, errorMessages, errors }; }; -const isAsyncDisposable = (value) => { - return Symbol.asyncDispose in value && typeof value[Symbol.asyncDispose] === 'function'; -}; - -const isDisposable = (value) => { - return Symbol.dispose in value && typeof value[Symbol.dispose] === 'function'; -}; - // What LogTape rendered for {error}: an Error as "TypeError: message", any // other value inspected. const describe = (value) => diff --git a/src/otel/opentelemetry-transport.js b/src/otel/opentelemetry-transport.js index 0c3aa70..f3759c1 100644 --- a/src/otel/opentelemetry-transport.js +++ b/src/otel/opentelemetry-transport.js @@ -5,8 +5,7 @@ const { createOpenTelemetryConfig, defaultErrorClassifier } = require('./config. const { MetricsCollector } = require('./metrics.js'); const { TracingCollector } = require('./tracing.js'); const { estimateMessageSize } = require('./message-size.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); -const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); // How many error messages of a failed batch its span keeps; the count says how // many there were in all. @@ -163,7 +162,8 @@ class OpenTelemetryTransport { * @returns The receipt of the wrapped transport. */ async sendRaw(message, options) { - if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const refusal = refuseUnlessRaw(this.wrappedTransport, options); + if (refusal !== undefined) return refusal; const startTime = performance.now(); const messageSize = message?.content instanceof Uint8Array ? message.content.byteLength : 0; const span = this.tracingCollector?.createRawSendSpan( @@ -183,9 +183,8 @@ class OpenTelemetryTransport { * Verifies the wrapped transport, which has nothing to check when it * cannot verify. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** @@ -356,20 +355,7 @@ class OpenTelemetryTransport { * Cleanup resources if the wrapped transport supports Disposable or AsyncDisposable. */ async [Symbol.asyncDispose]() { - if (!this.wrappedTransport) { - return; - } - - // Try AsyncDisposable first - if (typeof this.wrappedTransport[Symbol.asyncDispose] === 'function') { - await this.wrappedTransport[Symbol.asyncDispose](); - return; - } - - // Fall back to synchronous Disposable - if (typeof this.wrappedTransport[Symbol.dispose] === 'function') { - this.wrappedTransport[Symbol.dispose](); - } + await disposeOf(this.wrappedTransport); } extractTransportName(transport) { diff --git a/src/ratelimit/ratelimit-transport.js b/src/ratelimit/ratelimit-transport.js index e6e10e4..d0b708e 100644 --- a/src/ratelimit/ratelimit-transport.js +++ b/src/ratelimit/ratelimit-transport.js @@ -12,9 +12,7 @@ // `maxQueue`; and disposal stops the waits. const { createFailedReceipt } = require('../core/receipt.js'); -const { isRawTransport } = require('../core/transport.js'); -const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); +const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); const { Fifo, gatedSendMany } = require('../internal/gate.js'); const { warnLimited } = require('../internal/warn.js'); const { createLog } = require('../internal/log.js'); @@ -67,8 +65,8 @@ class RateLimitTransport { /** Sends a raw message once the bucket has its tokens. */ async sendRaw(message, options) { - options?.signal?.throwIfAborted(); - if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const unsent = refuseUnlessRaw(this.wrappedTransport, options); + if (unsent !== undefined) return unsent; const refusal = await this.#admit(message, options?.signal); if (refusal !== undefined) return refusal; return this.wrappedTransport.sendRaw(message, options); diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index 563d2e1..7fb6e43 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -8,11 +8,10 @@ const { } = require('../core/receipt.js'); const { combineSignals } = require('../core/abort-signal.js'); const { pinMessageId } = require('../core/message-id.js'); -const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); -const { refuseRaw } = require('../internal/capabilities.js'); const { createLog } = require('../internal/log.js'); const { createRetryConfig } = require('./config.js'); const { abortReason } = require('../internal/sleep.js'); +const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); /** * Transport decorator that retries transient delivery failures. @@ -83,8 +82,8 @@ class RetryTransport { * @throws {DOMException} If the operation is aborted. */ async sendRaw(message, options) { - options?.signal?.throwIfAborted(); - if (!isRawTransport(this.wrappedTransport)) return refuseRaw(this.wrappedTransport); + const refusal = refuseUnlessRaw(this.wrappedTransport, options); + if (refusal !== undefined) return refusal; return this.#withRetries( (sendOptions) => this.wrappedTransport.sendRaw(message, sendOptions), options, @@ -95,9 +94,8 @@ class RetryTransport { * Verifies the wrapped transport once, without retries; it has nothing to * check when it cannot verify. */ - async verify(options) { - options?.signal?.throwIfAborted(); - if (isVerifiableTransport(this.wrappedTransport)) await this.wrappedTransport.verify(options); + verify(options) { + return verifyWrapped(this.wrappedTransport, options); } /** @@ -227,21 +225,7 @@ class RetryTransport { * throws or rejects. */ async [Symbol.asyncDispose]() { - const wrappedTransport = Object(this.wrappedTransport); - const asyncDisposeSymbol = Symbol.asyncDispose; - if (typeof asyncDisposeSymbol === 'symbol') { - const asyncDispose = wrappedTransport[asyncDisposeSymbol]; - if (typeof asyncDispose === 'function') { - await asyncDispose.call(wrappedTransport); - return; - } - } - - const disposeSymbol = Symbol.dispose; - if (typeof disposeSymbol === 'symbol') { - const dispose = wrappedTransport[disposeSymbol]; - if (typeof dispose === 'function') dispose.call(wrappedTransport); - } + await disposeOf(this.wrappedTransport); } withSuccessMetadata(receipt, attempts) { diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 5af7317..c0d9351 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -14,8 +14,7 @@ const { pinMessageId } = require('../core/message-id.js'); const { isRawTransport } = require('../core/transport.js'); const { describeThrown, warnLimited, warnThrown } = require('../internal/callback.js'); const { redactAddresses } = require('../internal/redact-addresses.js'); -const { refuseRaw } = require('../internal/capabilities.js'); -const { disposeOf, verifyWrapped } = require('../internal/decorator.js'); +const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { gatedSendMany } = require('../internal/gate.js'); const { createTeeConfig } = require('./config.js'); @@ -90,8 +89,8 @@ class TeeTransport { * of the category `configuration`. */ async sendRaw(message, options) { - options?.signal?.throwIfAborted(); - if (!isRawTransport(this.primary)) return refuseRaw(this.primary); + const refusal = refuseUnlessRaw(this.primary, options); + if (refusal !== undefined) return refusal; const receipt = await this.primary.sendRaw(message, options); await this.#mirror(message, options?.signal, 'sendRaw'); return receipt; From 6708edb243b45764ccc4f3e54a24dfc2e424157c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:01:44 +0300 Subject: [PATCH 126/143] refactor: one way to read the input of a sendMany, and the text of a throw The input of a sendMany was read through three helpers, gate.js's iteratorOf, retry's toAsyncIterator and batch.js's readItems; they are iteratorOf and readItems of internal/iterate.js, the first the input's own iterator, whose return() reaches it while a next() is pending, as a generator would not let it. The pool disposes of its transports through disposeOf, and the pool, retry and OpenTelemetry tell what was thrown with messageOf. ./jmap goes to 13.5 KB, ./brevo to 8 and ./mailjet to 9 for the module. Co-Authored-By: Claude Opus 5.5 --- scripts/size-budgets.json | 6 +++--- src/internal/batch.js | 14 ++------------ src/internal/gate.js | 7 ++----- src/internal/iterate.js | 32 ++++++++++++++++++++++++++++++++ src/otel/tracing.js | 5 ++--- src/pool/pool-transport.js | 17 +++++------------ src/retry/retry-transport.js | 32 +++++++++----------------------- 7 files changed, 55 insertions(+), 58 deletions(-) create mode 100644 src/internal/iterate.js diff --git a/scripts/size-budgets.json b/scripts/size-budgets.json index f11ebf6..f144da6 100644 --- a/scripts/size-budgets.json +++ b/scripts/size-budgets.json @@ -6,7 +6,7 @@ "./webhooks": 15.5, "./smtp": 23.5, "./sendmail": 12, - "./jmap": 13, + "./jmap": 13.5, "./ses": 9.5, "./resend": 8, "./sendgrid": 8, @@ -16,8 +16,8 @@ "./lettermint": 7.5, "./plunk": 7, "./postmark": 8, - "./brevo": 7.5, - "./mailjet": 8.5, + "./brevo": 8, + "./mailjet": 9, "./mailersend": 8, "./sparkpost": 8.5, "./zeptomail": 7.5, diff --git a/src/internal/batch.js b/src/internal/batch.js index b7c36be..9342578 100644 --- a/src/internal/batch.js +++ b/src/internal/batch.js @@ -1,19 +1,9 @@ // Not ported: the messages of a sendMany() read as they are needed // rather than all at once, so that it holds a batch in memory, not its input. -const noop = () => {}; +const { readItems } = require('./iterate.js'); -/** - * The items of an iterable, synchronous or not, through one async iterator - * that can be read by hand and then iterated for the rest. - * - * @param iterable The items. - * @returns An async generator of them. - * @internal - */ -const readItems = async function* (iterable) { - for await (const item of iterable) yield item; -}; +const noop = () => {}; /** * Reads an iterable, synchronous or not, in arrays of at most `size` items, diff --git a/src/internal/gate.js b/src/internal/gate.js index 240c87f..8a1e833 100644 --- a/src/internal/gate.js +++ b/src/internal/gate.js @@ -3,6 +3,8 @@ // through that transport's own `sendMany()` so its batching is kept, with the // receipts in the order of the messages. +const { iteratorOf } = require('./iterate.js'); + /** * A first-in first-out queue whose `shift` takes constant time. * @@ -37,11 +39,6 @@ class Fifo { } } -const iteratorOf = (messages) => - typeof messages[Symbol.asyncIterator] === 'function' - ? messages[Symbol.asyncIterator]() - : messages[Symbol.iterator](); - // The most answers in place a run holds while the messages handed to the // wrapped transport before them wait for their receipts: past it, the run // ends, so that a circuit open over an endless input is not read ahead diff --git a/src/internal/iterate.js b/src/internal/iterate.js new file mode 100644 index 0000000..efb8eb6 --- /dev/null +++ b/src/internal/iterate.js @@ -0,0 +1,32 @@ +// Not ported: the messages of a sendMany() read as they are needed, whatever +// the input, synchronous or not: by hand through its own iterator (a gate, a +// retry), or as an async generator a batch iterates for the rest. + +/** + * The items of an iterable, synchronous or not, through one async iterator + * that can be read by hand and then iterated for the rest; its `return()` + * closes the iterable. + * + * @param iterable The items. + * @returns An async generator of them. + * @internal + */ +const readItems = async function* (iterable) { + for await (const item of iterable) yield item; +}; + +/** + * The iterator of an iterable, synchronous or not, as it is: one whose + * `return()` reaches the input at once, even while a `next()` is pending, + * which a generator would hold behind it. `await` takes either kind. + * + * @param values The items. + * @returns Their iterator. + * @internal + */ +const iteratorOf = (values) => + typeof values[Symbol.asyncIterator] === 'function' + ? values[Symbol.asyncIterator]() + : values[Symbol.iterator](); + +module.exports = { iteratorOf, readItems }; diff --git a/src/otel/tracing.js b/src/otel/tracing.js index c27b35c..ead84c7 100644 --- a/src/otel/tracing.js +++ b/src/otel/tracing.js @@ -6,6 +6,7 @@ const SPAN_KIND_CLIENT = 2; const SPAN_STATUS_OK = 1; const SPAN_STATUS_ERROR = 2; +const { messageOf } = require('../internal/decorator.js'); const { EmailAttributeExtractor } = require('./attributes.js'); const metadata = require('../version.js'); @@ -152,9 +153,7 @@ class LiveEmailSpan { const stacktrace = error instanceof Error ? describeStack(error, extractor) : undefined; this.span.addEvent('exception', { 'exception.type': error instanceof Error ? error.constructor.name : 'unknown', - 'exception.message': extractor.describeError( - error instanceof Error ? error.message : String(error), - ), + 'exception.message': extractor.describeError(messageOf(error)), ...(stacktrace && { 'exception.stacktrace': stacktrace }), }); } diff --git a/src/pool/pool-transport.js b/src/pool/pool-transport.js index 61b95f4..00567f6 100644 --- a/src/pool/pool-transport.js +++ b/src/pool/pool-transport.js @@ -5,6 +5,7 @@ const { combineSignals } = require('../core/abort-signal.js'); const { pinMessageId } = require('../core/message-id.js'); const { isRawTransport, isVerifiableTransport } = require('../core/transport.js'); const { refuseRaw } = require('../internal/capabilities.js'); +const { disposeOf, messageOf } = require('../internal/decorator.js'); const { createLog } = require('../internal/log.js'); const { createFailedReceipt, createReceiptError } = require('../core/receipt.js'); const { createPoolConfig } = require('./config.js'); @@ -332,7 +333,9 @@ class PoolTransport { const transport = entry.transport; if (seen.has(transport)) continue; seen.add(transport); - disposals.push(dispose(transport)); + // An async disposal: one that throws at once rejects, rather than + // stopping the disposal of the others. + disposals.push(disposeOf(transport)); } // Every transport is disposed of, whichever fails; the failures come @@ -371,7 +374,7 @@ class PoolTransport { onSelectorError: (error, index) => this.log.write( 'warn', - () => ({ index, error: error instanceof Error ? error.message : String(error) }), + () => ({ index, error: messageOf(error) }), 'A pool selector threw; its transport is skipped', ), }); @@ -554,14 +557,4 @@ const hasMessageRejection = (errors) => { return false; }; -// Disposes of a transport, asynchronously or not; an error it throws at once -// rejects rather than stopping the disposal of the others. -const dispose = async (transport) => { - if (typeof transport[Symbol.asyncDispose] === 'function') { - await transport[Symbol.asyncDispose](); - } else if (typeof transport[Symbol.dispose] === 'function') { - transport[Symbol.dispose](); - } -}; - module.exports = { PoolTransport }; diff --git a/src/retry/retry-transport.js b/src/retry/retry-transport.js index 7fb6e43..fa59601 100644 --- a/src/retry/retry-transport.js +++ b/src/retry/retry-transport.js @@ -11,7 +11,13 @@ const { pinMessageId } = require('../core/message-id.js'); const { createLog } = require('../internal/log.js'); const { createRetryConfig } = require('./config.js'); const { abortReason } = require('../internal/sleep.js'); -const { disposeOf, refuseUnlessRaw, verifyWrapped } = require('../internal/decorator.js'); +const { + disposeOf, + messageOf, + refuseUnlessRaw, + verifyWrapped, +} = require('../internal/decorator.js'); +const { iteratorOf } = require('../internal/iterate.js'); /** * Transport decorator that retries transient delivery failures. @@ -190,7 +196,7 @@ class RetryTransport { const controller = new AbortController(); const combinedSignal = combineSignals(controller.signal, options?.signal); const signal = combinedSignal.signal; - const run = new SendManyRun(this, toAsyncIterator(messages), { ...options, signal }); + const run = new SendManyRun(this, iteratorOf(messages), { ...options, signal }); try { run.launch(); while (run.isActive()) { @@ -357,7 +363,7 @@ class RetryTransport { }); } - const message = error instanceof Error ? error.message : String(error); + const message = messageOf(error); const classification = error instanceof Error && error.name === 'AbortError' ? { @@ -557,26 +563,6 @@ class SendManyRun { } } -const toAsyncIterator = (values) => { - const asyncIterator = values[Symbol.asyncIterator]; - if (typeof asyncIterator === 'function') return asyncIterator.call(values); - - const iterator = values[Symbol.iterator](); - return { - async next() { - return await Promise.resolve(iterator.next()); - }, - async return(value) { - return await Promise.resolve( - iterator.return?.(value) ?? { - done: true, - value, - }, - ); - }, - }; -}; - const raceWithAbort = (promise, signal) => { if (signal.aborted) return Promise.reject(abortReason(signal)); From f3ffb6760c7381717654b83b9387fa15163b3c56 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:04:03 +0300 Subject: [PATCH 127/143] refactor: one writer of an address and of metadata header fields for the providers The providers of 1.1 each wrote an address their way ({ email, name }, { address, displayName }, { Email, Name }), seven copies of one function, and six wrote the metadata as X-Metadata- header fields with copies of one loop. mailboxOf, mailboxesOf and visitMetadataHeaders of internal/api.js do both; each provider passes its field names, and its requests are those of before. Co-Authored-By: Claude Opus 5.5 --- src/azure/message-converter.js | 29 ++++++++------------- src/brevo/message-converter.js | 14 +++-------- src/cloudflare/message-converter.js | 31 ++++++++++------------- src/internal/api.js | 39 +++++++++++++++++++++++++++++ src/mailersend/message-converter.js | 26 ++++++++----------- src/mailjet/message-converter.js | 14 +++-------- src/scaleway/message-converter.js | 28 ++++++++------------- src/smtp2go/message-converter.js | 8 ++---- src/zeptomail/message-converter.js | 24 ++++++++---------- 9 files changed, 103 insertions(+), 110 deletions(-) diff --git a/src/azure/message-converter.js b/src/azure/message-converter.js index fea0e34..aab4cc0 100644 --- a/src/azure/message-converter.js +++ b/src/azure/message-converter.js @@ -7,7 +7,12 @@ // and a scheduled message is refused. const { toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { + base64Attachments, + headerFieldsOf, + mailboxesOf, + visitMetadataHeaders, +} = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { trackingSwitchOf } = require('../internal/tracking.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -15,17 +20,7 @@ const { RefusalError } = require('../internal/refusal.js'); /** Azure takes a request of at most 10 MB, its attachments included. */ const MAX_REQUEST_BYTES = 10 * 1024 * 1024; -const addressOf = (address) => { - const result = { address: toAsciiAddress(address.address) }; - if (address.name) result.displayName = address.name; - return result; -}; - -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'address', 'displayName'); // `userEngagementTrackingDisabled` turns opens and clicks off together, and // can only turn off what the resource has on. @@ -42,12 +37,10 @@ const headersOf = (message) => { hasHeaders = true; } // Azure has no metadata of its own: a header field carries it. - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers[`X-Metadata-${key}`] = message.metadata[key]; - hasHeaders = true; - } - } + visitMetadataHeaders(message, (name, value) => { + headers[name] = value; + hasHeaders = true; + }); return hasHeaders ? headers : undefined; }; diff --git a/src/brevo/message-converter.js b/src/brevo/message-converter.js index fd44f2e..24e7a28 100644 --- a/src/brevo/message-converter.js +++ b/src/brevo/message-converter.js @@ -12,21 +12,13 @@ // goes in `X-Mailin-custom`, which Brevo returns in its events. const { generateId } = require('../core/id.js'); -const { toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { base64Attachments, headerFieldsOf, mailboxesOf, mailboxOf } = require('../internal/api.js'); const { toUuid } = require('../internal/uuid.js'); const { RefusalError } = require('../internal/refusal.js'); -const addressOf = (address) => { - const email = toAsciiAddress(address.address); - return address.name ? { email, name: address.name } : { email }; -}; +const addressOf = (address) => mailboxOf(address, 'email', 'name'); -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'email', 'name'); /** * Converts a message to a request of Brevo's `/v3/smtp/email`. diff --git a/src/cloudflare/message-converter.js b/src/cloudflare/message-converter.js index e528cc3..9d30c95 100644 --- a/src/cloudflare/message-converter.js +++ b/src/cloudflare/message-converter.js @@ -8,8 +8,13 @@ // second reply-to address and a scheduled message are refused; tags, which // Cloudflare has no field for, are not sent, as over SMTP. -const { toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { + base64Attachments, + headerFieldsOf, + mailboxesOf, + mailboxOf, + visitMetadataHeaders, +} = require('../internal/api.js'); const { rawMessageText } = require('../internal/raw.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -17,17 +22,9 @@ const { RefusalError } = require('../internal/refusal.js'); // "The combined number of addresses across to, cc, and bcc must not exceed 50." const MAX_RECIPIENTS = 50; -const addressOf = (address) => { - const result = { address: toAsciiAddress(address.address) }; - if (address.name) result.name = address.name; - return result; -}; +const addressOf = (address) => mailboxOf(address, 'address', 'name'); -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'address', 'name'); const checkLimits = (message) => { refuseScheduledAt(message, 'cloudflare'); @@ -48,12 +45,10 @@ const headersOf = (message) => { headers[field.name] = field.value; hasHeaders = true; } - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers[`X-Metadata-${key}`] = message.metadata[key]; - hasHeaders = true; - } - } + visitMetadataHeaders(message, (name, value) => { + headers[name] = value; + hasHeaders = true; + }); return hasHeaders ? headers : undefined; }; diff --git a/src/internal/api.js b/src/internal/api.js index be61948..c829fb9 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -3,6 +3,7 @@ // which a message that cannot be converted fails alone, its attachments and // header fields, and the steps of one send. +const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { toBase64 } = require('./base64.js'); @@ -172,6 +173,41 @@ const sentWith = (provider, messageId, missing) => ? { successful: true, messageId, provider } : createFailedReceipt(missing, { provider, category: 'unknown', deliveryUnknown: true }); +/** + * An address as a provider writes it: `{ [emailKey]: the address in A-labels, + * [nameKey]: the name }`, the name only when there is one. + * + * @internal + */ +const mailboxOf = (address, emailKey, nameKey) => { + const mailbox = { [emailKey]: toAsciiAddress(address.address) }; + if (address.name) mailbox[nameKey] = address.name; + return mailbox; +}; + +/** + * The addresses of a list, each as {@link mailboxOf} writes it. + * + * @internal + */ +const mailboxesOf = (addresses, emailKey, nameKey) => { + const list = []; + for (let i = 0; i < addresses.length; i++) list.push(mailboxOf(addresses[i], emailKey, nameKey)); + return list; +}; + +/** + * Calls `visit(name, value)` with the `X-Metadata-` header field of each + * entry of the metadata of a message, for a provider that keeps none of its + * own but gives the header fields back with its events. + * + * @internal + */ +const visitMetadataHeaders = (message, visit) => { + if (message.metadata === undefined) return; + for (const key in message.metadata) visit(`X-Metadata-${key}`, message.metadata[key]); +}; + module.exports = { ProviderApiError, base64Attachments, @@ -180,10 +216,13 @@ module.exports = { createApiClient, headerFieldsOf, isCallerAbort, + mailboxOf, + mailboxesOf, mergeBatch, parseAnswer, providerFailure, sendConverted, sentWith, truncateErrorText, + visitMetadataHeaders, }; diff --git a/src/mailersend/message-converter.js b/src/mailersend/message-converter.js index df57000..e006152 100644 --- a/src/mailersend/message-converter.js +++ b/src/mailersend/message-converter.js @@ -9,9 +9,14 @@ // key, a second reply-to address is refused, and sently's dropped custom // headers are sent. -const { toAsciiAddress } = require('../core/address.js'); const { parseMessageId } = require('../core/message-id.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { + base64Attachments, + headerFieldsOf, + mailboxesOf, + mailboxOf, + visitMetadataHeaders, +} = require('../internal/api.js'); const { RefusalError } = require('../internal/refusal.js'); const LIMITS = [ @@ -31,16 +36,9 @@ const OWN_FIELDS = new Set([ 'list-unsubscribe-post', ]); -const addressOf = (address) => { - const email = toAsciiAddress(address.address); - return address.name ? { email, name: address.name } : { email }; -}; +const addressOf = (address) => mailboxOf(address, 'email', 'name'); -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'email', 'name'); const checkLimits = (message, config) => { for (const [field, name, limit] of LIMITS) { @@ -88,11 +86,7 @@ const placeHeaders = (payload, message) => { } } // MailerSend has no metadata of its own: a header reaches its events. - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers.push({ name: `X-Metadata-${key}`, value: message.metadata[key] }); - } - } + visitMetadataHeaders(message, (name, value) => headers.push({ name, value })); if (headers.length > 0) payload.headers = headers; }; diff --git a/src/mailjet/message-converter.js b/src/mailjet/message-converter.js index 4a39bbd..8ebae2b 100644 --- a/src/mailjet/message-converter.js +++ b/src/mailjet/message-converter.js @@ -6,8 +6,7 @@ // refused, as `CustomCampaign` is one name; `metadata` goes in `EventPayload`; // and a scheduled message is refused. -const { toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { base64Attachments, headerFieldsOf, mailboxesOf, mailboxOf } = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -59,16 +58,9 @@ const FORBIDDEN_HEADERS = new Set([ 'x-mj-statisticscontactslistid', ]); -const addressOf = (address) => { - const email = toAsciiAddress(address.address); - return address.name ? { Email: email, Name: address.name } : { Email: email }; -}; +const addressOf = (address) => mailboxOf(address, 'Email', 'Name'); -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'Email', 'Name'); /** * The recipients of a message, an address counted once, as Mailjet counts diff --git a/src/scaleway/message-converter.js b/src/scaleway/message-converter.js index 118c40f..4bc2fcd 100644 --- a/src/scaleway/message-converter.js +++ b/src/scaleway/message-converter.js @@ -6,8 +6,14 @@ // matched nothing; tags, which Scaleway has no field for, are not sent, as // over SMTP; and a scheduled message is refused. -const { formatMailbox, toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { formatMailbox } = require('../core/address.js'); +const { + base64Attachments, + headerFieldsOf, + mailboxesOf, + mailboxOf, + visitMetadataHeaders, +} = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -15,17 +21,9 @@ const { RefusalError } = require('../internal/refusal.js'); // "Maximum email size (API): 2 MB, including the email and all attachments". const MAX_EMAIL_BYTES = 2 * 1024 * 1024; -const addressOf = (address) => { - const result = { email: toAsciiAddress(address.address) }; - if (address.name) result.name = address.name; - return result; -}; +const addressOf = (address) => mailboxOf(address, 'email', 'name'); -const addressesOf = (addresses) => { - const list = []; - for (let i = 0; i < addresses.length; i++) list.push(addressOf(addresses[i])); - return list; -}; +const addressesOf = (addresses) => mailboxesOf(addresses, 'email', 'name'); // Scaleway has no reply-to or metadata field: both are header fields, as its // documentation shows for Reply-To. @@ -38,11 +36,7 @@ const headersOf = (message) => { const value = formatAddresses(message.replyRecipients, formatMailbox).join(', '); headers.push({ key: 'Reply-To', value }); } - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers.push({ key: `X-Metadata-${key}`, value: message.metadata[key] }); - } - } + visitMetadataHeaders(message, (key, value) => headers.push({ key, value })); return headers; }; diff --git a/src/smtp2go/message-converter.js b/src/smtp2go/message-converter.js index 82d3ca8..01a1ba4 100644 --- a/src/smtp2go/message-converter.js +++ b/src/smtp2go/message-converter.js @@ -6,7 +6,7 @@ // SMTP2GO has no field for, are not sent, as over SMTP. const { formatMailbox } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { base64Attachments, headerFieldsOf, visitMetadataHeaders } = require('../internal/api.js'); const { formatAddresses } = require('../internal/provider.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -52,11 +52,7 @@ const headersOf = (message) => { const value = formatAddresses(message.replyRecipients, formatMailbox).join(', '); headers.push({ header: 'Reply-To', value }); } - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers.push({ header: `X-Metadata-${key}`, value: message.metadata[key] }); - } - } + visitMetadataHeaders(message, (header, value) => headers.push({ header, value })); return headers; }; diff --git a/src/zeptomail/message-converter.js b/src/zeptomail/message-converter.js index 94757d2..5659378 100644 --- a/src/zeptomail/message-converter.js +++ b/src/zeptomail/message-converter.js @@ -4,8 +4,12 @@ // Changed in the port: a tag is the `client_reference`, a second one // refused, as ZeptoMail has no tags; and a scheduled message is refused. -const { toAsciiAddress } = require('../core/address.js'); -const { base64Attachments, headerFieldsOf } = require('../internal/api.js'); +const { + base64Attachments, + headerFieldsOf, + mailboxOf, + visitMetadataHeaders, +} = require('../internal/api.js'); const { refuseScheduledAt } = require('../internal/schedule.js'); const { RefusalError } = require('../internal/refusal.js'); @@ -14,11 +18,7 @@ const MAX_RECIPIENTS = 500; const MAX_ATTACHMENTS = 60; const MAX_SUBJECT = 500; -const addressOf = (address) => { - const result = { address: toAsciiAddress(address.address) }; - if (address.name) result.name = address.name; - return result; -}; +const addressOf = (address) => mailboxOf(address, 'address', 'name'); const recipientsOf = (addresses) => { const list = []; @@ -56,12 +56,10 @@ const headersOf = (message) => { hasHeaders = true; } // ZeptoMail has no metadata of its own: its events carry the headers. - if (message.metadata !== undefined) { - for (const key in message.metadata) { - headers[`X-Metadata-${key}`] = message.metadata[key]; - hasHeaders = true; - } - } + visitMetadataHeaders(message, (name, value) => { + headers[name] = value; + hasHeaders = true; + }); return hasHeaders ? headers : undefined; }; From bf80e267c64ae78907771a3215137f5d55156eab Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:05:39 +0300 Subject: [PATCH 128/143] refactor: the transports of the providers of 1.1 on one base Each of the ten transports of 1.1 made its config and its client, redacted its secrets in toJSON() and its inspection, and sent a batch through sendManyConcurrently, in the same lines. ProviderTransport of internal/provider-transport.js does it, the id first so that toJSON() keeps its order; a transport is its send(), its verify(), and the sendMany() of Postmark, Mailjet and Azure, which batch their own way. Co-Authored-By: Claude Opus 5.5 --- src/azure/azure-transport.js | 17 ++--------- src/brevo/brevo-transport.js | 25 ++-------------- src/cloudflare/cloudflare-transport.js | 25 ++-------------- src/internal/provider-transport.js | 40 ++++++++++++++++++++++++++ src/mailersend/mailersend-transport.js | 25 ++-------------- src/mailjet/mailjet-transport.js | 17 ++--------- src/postmark/postmark-transport.js | 17 ++--------- src/scaleway/scaleway-transport.js | 25 ++-------------- src/smtp2go/smtp2go-transport.js | 25 ++-------------- src/sparkpost/sparkpost-transport.js | 25 ++-------------- src/zeptomail/zeptomail-transport.js | 25 ++-------------- 11 files changed, 70 insertions(+), 196 deletions(-) create mode 100644 src/internal/provider-transport.js diff --git a/src/azure/azure-transport.js b/src/azure/azure-transport.js index 008f286..2a76739 100644 --- a/src/azure/azure-transport.js +++ b/src/azure/azure-transport.js @@ -11,12 +11,12 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); const { sendConcurrently } = require('../internal/batch.js'); const { isCallerAbort, providerFailure } = require('../internal/api.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { toUuid } = require('../internal/uuid.js'); const { verifyByRequest } = require('../internal/verify.js'); const { createAzureConfig } = require('./config.js'); const { createAzureClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); // The id of no operation, which `verify` reads. const NO_OPERATION = '00000000-0000-0000-0000-000000000000'; @@ -57,20 +57,9 @@ const receiptOf = (answer, requestId) => { /** * A transport over the Email API of Azure Communication Services. */ -class AzureTransport { - id = 'azure'; - +class AzureTransport extends ProviderTransport { constructor(config) { - this.config = createAzureConfig(config); - this.httpClient = createAzureClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('azure', config, createAzureConfig, createAzureClient); } /** diff --git a/src/brevo/brevo-transport.js b/src/brevo/brevo-transport.js index 5559c13..ef15fca 100644 --- a/src/brevo/brevo-transport.js +++ b/src/brevo/brevo-transport.js @@ -12,12 +12,11 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { providerFailure, sendConverted, sentWith } = require('../internal/api.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createBrevoConfig } = require('./config.js'); const { createBrevoClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); const failureOf = (error) => { if (error?.receiptCode === 'brevo.duplicate_parameter') { @@ -39,20 +38,9 @@ const failureOf = (error) => { * A transport over the transactional email API of Brevo (formerly * Sendinblue). */ -class BrevoTransport { - id = 'brevo'; - +class BrevoTransport extends ProviderTransport { constructor(config) { - this.config = createBrevoConfig(config); - this.httpClient = createBrevoClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('brevo', config, createBrevoConfig, createBrevoClient); } /** @@ -78,13 +66,6 @@ class BrevoTransport { }); } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the API key by reading its account (`GET /v3/account`). * diff --git a/src/cloudflare/cloudflare-transport.js b/src/cloudflare/cloudflare-transport.js index c37434f..20fe6f9 100644 --- a/src/cloudflare/cloudflare-transport.js +++ b/src/cloudflare/cloudflare-transport.js @@ -19,11 +19,10 @@ const { sendConverted, } = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createCloudflareConfig } = require('./config.js'); const { createCloudflareClient, errorCodeOf } = require('./http-client.js'); const { convertMessage, convertRawMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); const listOf = (value) => (Array.isArray(value) ? value : []); @@ -77,20 +76,9 @@ const receiptOf = (json, config) => { /** * A transport over the Email Service API of Cloudflare. */ -class CloudflareTransport { - id = 'cloudflare'; - +class CloudflareTransport extends ProviderTransport { constructor(config) { - this.config = createCloudflareConfig(config); - this.httpClient = createCloudflareClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('cloudflare', config, createCloudflareConfig, createCloudflareClient); } /** @@ -131,13 +119,6 @@ class CloudflareTransport { } } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the API token with Cloudflare's own check, as a token of the * account and, failing that, of a user. diff --git a/src/internal/provider-transport.js b/src/internal/provider-transport.js new file mode 100644 index 0000000..6929caa --- /dev/null +++ b/src/internal/provider-transport.js @@ -0,0 +1,40 @@ +// Not ported: what the transports of the HTTP providers of 1.1 are made of +// alike: their configuration and its client, the redaction of the secrets +// they hold, and a sendMany() of single sends at a time. + +const { sendManyConcurrently } = require('./batch.js'); +const { INSPECT, inspectRedacted, redactSecrets } = require('./redact.js'); + +/** + * The base of a transport of an HTTP provider: its `id`; its `config`, made + * by `createConfig`, and its `httpClient`, by `createClient` of it; its + * `toJSON()` and inspection, the secrets redacted; and a `sendMany()` of its + * `send()`, `concurrency` at a time, which a provider with a batch API + * replaces. + * + * @internal + */ +class ProviderTransport { + constructor(id, config, createConfig, createClient) { + this.id = id; + this.config = createConfig(config); + this.httpClient = createClient(this.config); + } + + toJSON() { + return redactSecrets(this); + } + + [INSPECT](_depth, options, inspect) { + return inspectRedacted(this, options, inspect); + } + + /** + * Sends messages `concurrency` at a time, the receipts in their order. + */ + sendMany(messages, options) { + return sendManyConcurrently(this, messages, options); + } +} + +module.exports = { ProviderTransport }; diff --git a/src/mailersend/mailersend-transport.js b/src/mailersend/mailersend-transport.js index 290689f..4fa4fb9 100644 --- a/src/mailersend/mailersend-transport.js +++ b/src/mailersend/mailersend-transport.js @@ -8,30 +8,18 @@ // accepts what may fail later, unknown to the receipts. const { sendConverted, sentWith } = require('../internal/api.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createMailerSendConfig } = require('./config.js'); const { createMailerSendClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); /** * A transport over the email API of MailerSend. */ -class MailerSendTransport { - id = 'mailersend'; - +class MailerSendTransport extends ProviderTransport { constructor(config) { - this.config = createMailerSendConfig(config); - this.httpClient = createMailerSendClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('mailersend', config, createMailerSendConfig, createMailerSendClient); } /** @@ -56,13 +44,6 @@ class MailerSendTransport { }); } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the API token by reading the domains of the account * (`GET /v1/domains`); a token without that scope is verified too. diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index f48efb1..f9e41da 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -17,10 +17,10 @@ const { providerFailure, } = require('../internal/api.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createMailjetConfig } = require('./config.js'); const { createMailjetClient } = require('./http-client.js'); const { MAX_RECIPIENTS, convertMessage, recipientCount } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); // Mailjet takes 15 MB a request at most, attachments included. const MAX_REQUEST_BYTES = 15 * 1000 * 1000; @@ -51,20 +51,9 @@ const answersOf = (error) => /** * A transport over the Send API v3.1 of Mailjet. */ -class MailjetTransport { - id = 'mailjet'; - +class MailjetTransport extends ProviderTransport { constructor(config) { - this.config = createMailjetConfig(config); - this.httpClient = createMailjetClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('mailjet', config, createMailjetConfig, createMailjetClient); } receiptOf(entry) { diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index 9950d31..1cd8709 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -18,10 +18,10 @@ const { sendConverted, } = require('../internal/api.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createPostmarkConfig } = require('./config.js'); const { classifyErrorCode, createPostmarkClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); // The most messages one request of `/email/batch` takes. const MAX_BATCH_SIZE = 500; @@ -54,20 +54,9 @@ const receiptOf = (entry) => { /** * A transport over the API of Postmark (postmarkapp.com). */ -class PostmarkTransport { - id = 'postmark'; - +class PostmarkTransport extends ProviderTransport { constructor(config) { - this.config = createPostmarkConfig(config); - this.httpClient = createPostmarkClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('postmark', config, createPostmarkConfig, createPostmarkClient); } /** diff --git a/src/scaleway/scaleway-transport.js b/src/scaleway/scaleway-transport.js index 56f20e5..1a211db 100644 --- a/src/scaleway/scaleway-transport.js +++ b/src/scaleway/scaleway-transport.js @@ -4,30 +4,18 @@ // Changed in the port: `verify` lists the domains of the project. const { sendConverted, sentWith } = require('../internal/api.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createScalewayConfig } = require('./config.js'); const { createScalewayClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); /** * A transport over the Transactional Email API of Scaleway. */ -class ScalewayTransport { - id = 'scaleway'; - +class ScalewayTransport extends ProviderTransport { constructor(config) { - this.config = createScalewayConfig(config); - this.httpClient = createScalewayClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('scaleway', config, createScalewayConfig, createScalewayClient); } /** @@ -53,13 +41,6 @@ class ScalewayTransport { }); } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the secret key by listing the domains of the project; a key * without that permission is verified too. diff --git a/src/smtp2go/smtp2go-transport.js b/src/smtp2go/smtp2go-transport.js index 294f65f..640e19e 100644 --- a/src/smtp2go/smtp2go-transport.js +++ b/src/smtp2go/smtp2go-transport.js @@ -7,12 +7,11 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { sendConverted } = require('../internal/api.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createSmtp2goConfig } = require('./config.js'); const { createSmtp2goClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); const describeFailure = (failure) => { if (typeof failure === 'string') return failure; @@ -52,20 +51,9 @@ const receiptOf = (json) => { /** * A transport over the email API of SMTP2GO. */ -class Smtp2goTransport { - id = 'smtp2go'; - +class Smtp2goTransport extends ProviderTransport { constructor(config) { - this.config = createSmtp2goConfig(config); - this.httpClient = createSmtp2goClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('smtp2go', config, createSmtp2goConfig, createSmtp2goClient); } /** @@ -85,13 +73,6 @@ class Smtp2goTransport { }); } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the API key by reading the summary of the account * (`POST /v3/stats/email_summary`); a key without that permission is diff --git a/src/sparkpost/sparkpost-transport.js b/src/sparkpost/sparkpost-transport.js index 3dd587f..7fcff2f 100644 --- a/src/sparkpost/sparkpost-transport.js +++ b/src/sparkpost/sparkpost-transport.js @@ -8,12 +8,11 @@ const { createFailedReceipt } = require('../core/receipt.js'); const { isCallerAbort, providerFailure, sendConverted } = require('../internal/api.js'); const { rawMessageFailure, readRawMessage } = require('../internal/raw.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); const { verifyByGet } = require('../internal/verify.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createSparkPostConfig } = require('./config.js'); const { createSparkPostClient } = require('./http-client.js'); const { convertMessage, convertRawMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); // The receipt of `{ results: { id, total_accepted_recipients, // total_rejected_recipients } }`. @@ -46,20 +45,9 @@ const receiptOf = (json) => { /** * A transport over the Transmissions API of SparkPost. */ -class SparkPostTransport { - id = 'sparkpost'; - +class SparkPostTransport extends ProviderTransport { constructor(config) { - this.config = createSparkPostConfig(config); - this.httpClient = createSparkPostClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('sparkpost', config, createSparkPostConfig, createSparkPostClient); } /** @@ -101,13 +89,6 @@ class SparkPostTransport { } } - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } - /** * Verifies the API key by reading the account (`GET /api/v1/account`); a key * without that permission is verified too. diff --git a/src/zeptomail/zeptomail-transport.js b/src/zeptomail/zeptomail-transport.js index 94bad39..d33d805 100644 --- a/src/zeptomail/zeptomail-transport.js +++ b/src/zeptomail/zeptomail-transport.js @@ -6,29 +6,17 @@ // message shared. const { sendConverted, sentWith } = require('../internal/api.js'); -const { sendManyConcurrently } = require('../internal/batch.js'); -const { INSPECT, inspectRedacted, redactSecrets } = require('../internal/redact.js'); const { createZeptoMailConfig } = require('./config.js'); const { createZeptoMailClient } = require('./http-client.js'); const { convertMessage } = require('./message-converter.js'); +const { ProviderTransport } = require('../internal/provider-transport.js'); /** * A transport over the email API of ZeptoMail. */ -class ZeptoMailTransport { - id = 'zeptomail'; - +class ZeptoMailTransport extends ProviderTransport { constructor(config) { - this.config = createZeptoMailConfig(config); - this.httpClient = createZeptoMailClient(this.config); - } - - toJSON() { - return redactSecrets(this); - } - - [INSPECT](_depth, options, inspect) { - return inspectRedacted(this, options, inspect); + super('zeptomail', config, createZeptoMailConfig, createZeptoMailClient); } /** @@ -48,13 +36,6 @@ class ZeptoMailTransport { sentWith('zeptomail', answer.json?.request_id, 'ZeptoMail answered without a request_id.'), }); } - - /** - * Sends messages `concurrency` at a time, the receipts in their order. - */ - sendMany(messages, options) { - return sendManyConcurrently(this, messages, options); - } } module.exports = { ZeptoMailTransport }; From a7b421243183be4e8ee80dbed56c79930a80986a Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:07:50 +0300 Subject: [PATCH 129/143] test: the contracts of delivery over every HTTP provider of the exports map The matrix of delivery.test.js listed the providers by hand; a test now fails when an HTTP provider of the exports map is not in it. The refusal of a plain HTTP URL and the cause of a failure of fetch, which only the providers ported from upyo were held to, cover all eighteen, and a contract of the abort is added: a signal aborted before the send rejects with its reason and makes no request, and one aborted during it rejects with its reason too. Co-Authored-By: Claude Opus 5.5 --- tests/unit/delivery.test.js | 82 +++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/tests/unit/delivery.test.js b/tests/unit/delivery.test.js index 36e745b..b658d38 100644 --- a/tests/unit/delivery.test.js +++ b/tests/unit/delivery.test.js @@ -284,11 +284,15 @@ test('Delivery: the failure of fetch stays the cause of the provider error', asy t.mock.method(globalThis, 'fetch', async () => { throw refused; }); - for (const [name, , create] of portedProviders) { + for (const [name, , create] of providers) { + const transport = create({ retries: 0 }); const body = name === 'Mailgun' ? new FormData() : {}; - const error = await create({ retries: 0 }) - .httpClient.sendMessage(body, undefined) - .catch((caught) => caught); + // The clients ported from upyo send with sendMessage; those of 1.1 with + // the request of the shared client. + const sending = portedProviders.some((provider) => provider[0] === name) + ? transport.httpClient.sendMessage(body, undefined) + : transport.httpClient.request({ url: 'https://api.example.com/send', body: {} }); + const error = await sending.catch((caught) => caught); assert.ok(error instanceof Error, name); let cause = error; while (cause !== refused && cause?.cause != null) cause = cause.cause; @@ -442,3 +446,73 @@ test('Delivery: the logger of an HTTP transport hears its retries, without addre assert.ok(properties.delayMilliseconds > 0); assert.ok(!JSON.stringify(records).includes('bob@example.com'), JSON.stringify(records)); }); + +test('Delivery: the matrix holds every HTTP provider of the exports map', () => { + // Before: the contracts listed the providers by hand, so a provider added + // to the exports map was not held to them. + const fs = require('node:fs'); + const path = require('node:path'); + const pkg = require('../../package.json'); + const root = path.join(__dirname, '../..'); + const http = []; + for (const key of Object.keys(pkg.exports)) { + const subpath = key.slice(2); + if (key === '.' || subpath === 'jmap') continue; + if (fs.existsSync(path.join(root, 'src', subpath, 'http-client.js'))) http.push(subpath); + } + const inMatrix = []; + for (const [, , create] of providers) inMatrix.push(create().id); + assert.deepStrictEqual(inMatrix.sort(), http.sort()); +}); + +test('Delivery: every HTTP provider refuses a plain HTTP URL but to this machine', () => { + // Before: the refusal was checked for the providers ported from upyo alone. + const insecure = 'http://api.example.com'; + for (const [name, , create] of providers) { + if (name === 'SES') continue; // Its endpoint is made of its region alone. + if (name === 'Azure') { + assert.throws( + () => + new AzureTransport({ + connectionString: `endpoint=${insecure}/;accesskey=${AZURE_KEY}`, + }), + TypeError, + name, + ); + continue; + } + const url = name === 'Mailtrap' ? { sendBaseUrl: insecure } : { baseUrl: insecure }; + assert.throws(() => create(url), TypeError, name); + } +}); + +test('Delivery: every HTTP provider rejects with the reason of an abort, before or during', async (t) => { + // Before: no contract held the providers to the reason of the caller's + // signal, which a decorator tells apart from a failure. + for (const [name, , create] of providers) { + const seen = { requests: 0 }; + t.mock.method(globalThis, 'fetch', (_url, init) => { + seen.requests++; + return new Promise((_resolve, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true }); + }); + }); + const early = new AbortController(); + const reason = new Error(`stopped ${name}`); + early.abort(reason); + await assert.rejects(create().send(message(), { signal: early.signal }), (error) => { + assert.strictEqual(error, reason, name); + return true; + }); + assert.strictEqual(seen.requests, 0, name); + const late = new AbortController(); + const sending = create().send(message(), { signal: late.signal }); + while (seen.requests === 0) await new Promise((resolve) => setImmediate(resolve)); + late.abort(reason); + await assert.rejects(sending, (error) => { + assert.strictEqual(error, reason, name); + return true; + }); + t.mock.restoreAll(); + } +}); From 6df3a6b1e7509e9aa70bfaef8ac73d5ab6ac6fd2 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:10:18 +0300 Subject: [PATCH 130/143] fix(postmark, mailjet): a request goes as soon as the next message would not fit The batches were read 500 (Postmark) or 50 (Mailjet) messages at a time and converted whole, their files in base64, before the first request, so a batch of large files was held in memory however many requests it then made. Each message is converted as it is read, and a request goes as soon as the next one would not fit by count or bytes: no more than a request and a message are held at once. The pages say a batch request has no idempotency key, and to send messages with large files one by one. Co-Authored-By: Claude Opus 5.5 --- docs/transports/mailjet.md | 8 +++- docs/transports/postmark.md | 16 +++++-- src/mailjet/mailjet-transport.js | 70 ++++++++++++++++-------------- src/postmark/postmark-transport.js | 67 ++++++++++++++-------------- tests/unit/postmark.test.js | 37 ++++++++++++++++ 5 files changed, 127 insertions(+), 71 deletions(-) diff --git a/docs/transports/mailjet.md b/docs/transports/mailjet.md index 59dee23..5983459 100644 --- a/docs/transports/mailjet.md +++ b/docs/transports/mailjet.md @@ -67,8 +67,12 @@ if (receipt.successful) console.log(receipt.messageId); // the MessageUUID ## Batches and errors `Messages` is Mailjet's batch: `sendMany()` sends requests of at most 50 -recipients, as Mailjet counts them over the whole request, and 15 MB, reading -its input a batch at a time. When one message of a request fails, Mailjet answers 400 with +recipients, as Mailjet counts them over the whole request, and 15 MB, converting +each message as it reads it and sending a request as soon as the next one would +not fit, so that it holds no more than a request at once. A batch request has no +idempotency key: a timeout after it was sent leaves all its messages of unknown +delivery, so messages with large files are better sent with `send()` each. +When one message of a request fails, Mailjet answers 400 with the answers of all of them, so each message gets the receipt of its own answer: those that went succeed, the others fail with the code Mailjet gives (`mailjet.send-0003`) and the fields it names. diff --git a/docs/transports/postmark.md b/docs/transports/postmark.md index 1edc690..78a1914 100644 --- a/docs/transports/postmark.md +++ b/docs/transports/postmark.md @@ -72,10 +72,18 @@ the message is `deliveryUnknown`. ## Batches `sendMany()` sends up to 500 messages and 50 MB a request through `/email/batch`, -reading its input a batch at a time: messages with files go in as many requests -as their size takes, each of which fails, or is of unknown delivery, on its own. Each message has the receipt of its own answer, so an -inactive recipient fails its message alone; a message that cannot be converted -fails alone too. +converting each message as it reads it and sending a request as soon as the +next message would not fit, so that it holds no more than a request at once: +messages with files go in as many requests as their size takes, each of which +fails, or is of unknown delivery, on its own. Each message has the receipt of +its own answer, so an inactive recipient fails its message alone; a message that +cannot be converted fails alone too. + +A batch request has no idempotency key: a timeout or a reset after it was sent +leaves all its messages of unknown delivery at once. Messages with large files, +or any whose delivery must be known one by one, are better sent with `send()` +each (or through `./retry`'s `sendMany`, which sends them one by one at a +bounded concurrency). ## Verifying the configuration diff --git a/src/mailjet/mailjet-transport.js b/src/mailjet/mailjet-transport.js index f9e41da..8db60ef 100644 --- a/src/mailjet/mailjet-transport.js +++ b/src/mailjet/mailjet-transport.js @@ -8,10 +8,9 @@ const { generateId } = require('../core/id.js'); const { createFailedReceipt } = require('../core/receipt.js'); -const { chunks, jsonBytes } = require('../internal/batch.js'); +const { jsonBytes } = require('../internal/batch.js'); const { ProviderApiError, - convertBatch, isCallerAbort, mergeBatch, providerFailure, @@ -131,7 +130,9 @@ class MailjetTransport extends ProviderTransport { /** * Sends messages in requests of up to 50 recipients and 15 MB, each message - * with the receipt of its own answer. + * converted as it is read and a request sent as soon as the next one would + * not fit, so that no more than a request is held at once; each message + * has the receipt of its own answer. * * @param messages The messages. * @param options Optional abort signal and retry callback. @@ -139,39 +140,44 @@ class MailjetTransport extends ProviderTransport { */ async *sendMany(messages, options) { const signal = options?.signal; - for await (const batch of chunks(messages, MAX_RECIPIENTS)) { + // The messages of the request being filled, in order: the index of an + // entry, or the receipt of a message that did not convert. + let slots = []; + let entries = []; + let recipients = 0; + let bytes = 16; + for await (const message of messages) { signal?.throwIfAborted(); - const converted = await convertBatch( - batch, - (message) => convertMessage(message, this.config, signal), - 'mailjet', - signal, - ); - const receipts = []; - let run = []; - let recipients = 0; - let bytes = 16; - for (let i = 0; i < batch.length; i++) { - if (typeof converted.slots[i] !== 'number') continue; - const count = recipientCount(batch[i]); - const entry = converted.payloads[converted.slots[i]]; - const size = jsonBytes(entry); - const full = recipients + count > MAX_RECIPIENTS || bytes + size > MAX_REQUEST_BYTES; - if (run.length > 0 && full) { - for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); - run = []; - recipients = 0; - bytes = 16; - } - run.push(entry); - recipients += count; - bytes += size; + let entry; + try { + entry = await convertMessage(message, this.config, signal); + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + slots.push(providerFailure('mailjet', error)); + continue; } - if (run.length > 0) { - for (const receipt of await this.sendEntries(run, options)) receipts.push(receipt); + const count = recipientCount(message); + const size = jsonBytes(entry); + const full = recipients + count > MAX_RECIPIENTS || bytes + size > MAX_REQUEST_BYTES; + if (entries.length > 0 && full) { + yield* this.#flush(slots, entries, options); + slots = []; + entries = []; + recipients = 0; + bytes = 16; } - yield* mergeBatch(converted.slots, (index) => receipts[index]); + slots.push(entries.length); + entries.push(entry); + recipients += count; + bytes += size; } + yield* this.#flush(slots, entries, options); + } + + // Sends a request, and gives the receipts of its messages in their order. + async *#flush(slots, entries, options) { + const receipts = entries.length === 0 ? [] : await this.sendEntries(entries, options); + yield* mergeBatch(slots, (index) => receipts[index]); } /** diff --git a/src/postmark/postmark-transport.js b/src/postmark/postmark-transport.js index 1cd8709..9a5ead9 100644 --- a/src/postmark/postmark-transport.js +++ b/src/postmark/postmark-transport.js @@ -9,14 +9,8 @@ // sently's JSON parse before the status check is gone with the shared client. const { createFailedReceipt } = require('../core/receipt.js'); -const { chunks, jsonBytes } = require('../internal/batch.js'); -const { - convertBatch, - isCallerAbort, - mergeBatch, - providerFailure, - sendConverted, -} = require('../internal/api.js'); +const { jsonBytes } = require('../internal/batch.js'); +const { isCallerAbort, mergeBatch, providerFailure, sendConverted } = require('../internal/api.js'); const { verifyByGet } = require('../internal/verify.js'); const { createPostmarkConfig } = require('./config.js'); const { classifyErrorCode, createPostmarkClient } = require('./http-client.js'); @@ -78,8 +72,9 @@ class PostmarkTransport extends ProviderTransport { /** * Sends messages with `/email/batch`, 500 and 50 MB a request at most, - * reading the input a batch at a time; each message has the receipt of its - * own answer. + * each message converted as it is read and a request sent as soon as the + * next one would not fit, so that no more than a request is held at once; + * each message has the receipt of its own answer. * * @param messages The messages. * @param options Optional abort signal and retry callback. @@ -87,34 +82,40 @@ class PostmarkTransport extends ProviderTransport { */ async *sendMany(messages, options) { const signal = options?.signal; - for await (const batch of chunks(messages, MAX_BATCH_SIZE)) { + // The messages of the request being filled, in order: the index of a + // payload, or the receipt of a message that did not convert. + let slots = []; + let payloads = []; + let bytes = 2; + for await (const message of messages) { signal?.throwIfAborted(); - const converted = await convertBatch( - batch, - (message) => convertMessage(message, this.config, signal), - 'postmark', - signal, - ); - if (converted.payloads.length === 0) { - yield* mergeBatch(converted.slots, () => undefined); + let payload; + try { + payload = await convertMessage(message, this.config, signal); + } catch (error) { + if (isCallerAbort(error, signal)) throw error; + slots.push(providerFailure('postmark', error)); continue; } - const receipts = []; - let run = []; - let bytes = 2; - for (const payload of converted.payloads) { - const size = jsonBytes(payload); - if (run.length > 0 && bytes + size > MAX_BATCH_BYTES) { - for (const receipt of await this.#sendRun(run, options)) receipts.push(receipt); - run = []; - bytes = 2; - } - run.push(payload); - bytes += size; + const size = jsonBytes(payload); + const full = payloads.length === MAX_BATCH_SIZE || bytes + size > MAX_BATCH_BYTES; + if (payloads.length > 0 && full) { + yield* this.#flush(slots, payloads, options); + slots = []; + payloads = []; + bytes = 2; } - for (const receipt of await this.#sendRun(run, options)) receipts.push(receipt); - yield* mergeBatch(converted.slots, (index) => receipts[index]); + slots.push(payloads.length); + payloads.push(payload); + bytes += size; } + yield* this.#flush(slots, payloads, options); + } + + // Sends a request, and gives the receipts of its messages in their order. + async *#flush(slots, payloads, options) { + const receipts = payloads.length === 0 ? [] : await this.#sendRun(payloads, options); + yield* mergeBatch(slots, (index) => receipts[index]); } // Sends the payloads of one request: a receipt each, in order. diff --git a/tests/unit/postmark.test.js b/tests/unit/postmark.test.js index 4a31aba..44913db 100644 --- a/tests/unit/postmark.test.js +++ b/tests/unit/postmark.test.js @@ -282,3 +282,40 @@ test('Postmark: a batch past 50 MB goes in as many requests as that takes', asyn ); for (const request of requests) assert.ok(Buffer.byteLength(request.body) <= 50 * 1000 * 1000); }); + +test('Postmark: a request goes once the next message would not fit, not after 500', async (t) => { + // Before: 500 messages were read and converted, their files in base64, + // before the first request, so a batch held them all however large. + let read = 0; + const readAtRequests = []; + stubFetch(t, (request) => { + readAtRequests.push(read); + return json( + request.json.map((_payload, index) => ({ ErrorCode: 0, MessageID: `pm-${index}` })), + ); + }); + const big = () => + message({ + attachments: [ + { + filename: 'big.bin', + contentType: 'application/octet-stream', + inline: false, + contentId: '', + content: new Uint8Array(20 * 1000 * 1000), + }, + ], + }); + const input = (function* () { + for (let i = 0; i < 4; i++) { + read++; + yield big(); + } + })(); + const receipts = []; + for await (const receipt of transport().sendMany(input)) receipts.push(receipt); + assert.strictEqual(receipts.length, 4); + // A file of 20 MB is some 27 MB in base64: one fills a request, which goes + // once the next message is read, and the last once the input ends. + assert.deepStrictEqual(readAtRequests, [2, 3, 4, 4]); +}); From 209a09064b49604650ff17ebe46df4bc41770aa0 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:11:19 +0300 Subject: [PATCH 131/143] feat(otel): count failed messages by the category and the code of their error A failed message was counted as a failure alone, so a rate limit, a refused key and a refusal of the provider could not be told apart from the metrics. mailer.email.messages takes error.type and mailer.error.code, the category and code of the first error of the receipt (rate-limit, http.429), as the plan of the review asked of ./otel. Co-Authored-By: Claude Opus 5.5 --- docs/transports/otel.md | 2 +- src/otel/metrics.js | 9 +++++++-- tests/unit/otel.test.js | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/transports/otel.md b/docs/transports/otel.md index 70d4e46..b403d0d 100644 --- a/docs/transports/otel.md +++ b/docs/transports/otel.md @@ -365,7 +365,7 @@ Prometheus turns into its own (`mailer_email_attempts_total`): |---|---|---|---| | `mailer.email.attempts` | counter | `{attempt}` | `mailer.transport.name`, `mailer.operation.name` (`send`, `send_batch`), `mailer.status` (`success`, `failure`, `partial_failure`), `error.type` on a failure | | `mailer.email.duration` | histogram | `s` | as `mailer.email.attempts` | -| `mailer.email.messages` | counter | `{message}` | `mailer.transport.name`, `email.priority`, `mailer.status`, and `mailer.delivery_unknown` (`true`) on a failure the provider may have taken | +| `mailer.email.messages` | counter | `{message}` | `mailer.transport.name`, `email.priority`, `mailer.status`; on a failure, `error.type` and `mailer.error.code`, the category and code of its first error (`rate-limit`, `http.429`), and `mailer.delivery_unknown` (`true`) when the provider may have taken it | | `mailer.email.retries` | counter | `{retry}` | `mailer.transport.name`, `error.type` | | `mailer.email.message.size` | histogram | `By` | `mailer.transport.name`, `email.priority`, `email.content.type` | | `mailer.email.attachments` | histogram | `{attachment}` | as `mailer.email.message.size` | diff --git a/src/otel/metrics.js b/src/otel/metrics.js index 12622a9..1425b3f 100644 --- a/src/otel/metrics.js +++ b/src/otel/metrics.js @@ -97,14 +97,19 @@ class MetricsCollector { /** * Records the receipt of one message, once, with the priority of the - * message, whether it was sent, and, for a failure the provider may have - * taken, `mailer.delivery_unknown`: the messages to reconcile. + * message, whether it was sent, and, for a failure, the category and the + * code of its first error (`error.type`, `mailer.error.code`, such as + * `rate-limit` and `http.429`) and, when the provider may have taken it, + * `mailer.delivery_unknown`: the messages to reconcile. */ recordMessageOutcome(transport, priority, receipt) { + const first = receipt.successful ? undefined : receipt.errors?.[0]; this.messageCounter.add(1, { 'mailer.transport.name': transport, 'email.priority': priority, 'mailer.status': receipt.successful ? 'success' : 'failure', + ...(typeof first?.category === 'string' && { 'error.type': first.category }), + ...(typeof first?.code === 'string' && { 'mailer.error.code': first.code }), ...(receipt.deliveryUnknown === true && { 'mailer.delivery_unknown': true }), }); } diff --git a/tests/unit/otel.test.js b/tests/unit/otel.test.js index 307815c..6c46485 100644 --- a/tests/unit/otel.test.js +++ b/tests/unit/otel.test.js @@ -477,3 +477,25 @@ test('OTel: a failure of unknown delivery is told apart, on the span and the cou } assert.deepStrictEqual(counted, [true, true]); }); + +test('OTel: the message counter takes the category and the code of a failure', async () => { + // Before: a failed message was counted as a failure alone, so a rate + // limit, a refused key and a refusal of the provider could not be told + // apart without the traces. + const limited = { + successful: false, + errorMessages: ['Too many requests.'], + errors: [{ message: 'limited', category: 'rate-limit', code: 'http.429', retryable: true }], + }; + const otel = providers(); + const transport = new OpenTelemetryTransport(plain({}, limited), otel); + await transport.send(createTestMessage()); + const counted = []; + for (const metric of otel.meterProvider.getAllRecords()) { + if (metric.name !== 'mailer.email.messages') continue; + for (const record of metric.records) { + counted.push([record.attributes['error.type'], record.attributes['mailer.error.code']]); + } + } + assert.deepStrictEqual(counted, [['rate-limit', 'http.429']]); +}); From 7cd92ae7b356864108dca30444293782eb9fb91d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:12:48 +0300 Subject: [PATCH 132/143] fix(breaker, tee): a code of their own for a refused send and a failed copy breaker.open named both the change of state, at warn, and every send the circuit refused, at debug, so counting the records by code counted both; a refusal is breaker.refused. The tee wrote MAILER_TEE_MIRROR_FAILED, the code of its process warning, in its records; they are tee.mirror_failed, as the other decorators name theirs. The records of the breaker half-opening and closing and of a key in flight are tested, and tsd holds the logger option of the four decorators and the maxQueue of the rate limit. Co-Authored-By: Claude Opus 5.5 --- docs/transports/breaker.md | 2 +- docs/transports/tee.md | 2 +- src/breaker/breaker-transport.js | 4 ++- src/tee/index.d.ts | 2 +- src/tee/tee-transport.js | 2 +- tests/types/breaker.test-d.ts | 3 ++ tests/types/idempotency.test-d.ts | 3 ++ tests/types/ratelimit.test-d.ts | 4 +++ tests/types/tee.test-d.ts | 3 ++ tests/unit/decorators.test.js | 47 +++++++++++++++++++++++++++++-- 10 files changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/transports/breaker.md b/docs/transports/breaker.md index f3a5201..a403cec 100644 --- a/docs/transports/breaker.md +++ b/docs/transports/breaker.md @@ -66,7 +66,7 @@ const pool = new PoolTransport({ | `isFailure` | the categories above | Whether a receipt counts as a failure. | | `onStateChange` | | Hears each change of state; one that throws becomes a process warning (`MAILER_BREAKER_LISTENER_FAILED`). | | `now` | `Date.now` | The clock. | -| `logger` | | A pino-like logger: a change of state (`breaker.open` at `warn`), a refused send at `debug`. | +| `logger` | | A pino-like logger: a change of state (`breaker.open` at `warn`, `breaker.half-open` and `breaker.closed` at `info`), a refused send (`breaker.refused`) at `debug`. | `sendMany()` goes through the `sendMany()` of the wrapped transport while the circuit is closed, so its batching is kept; a message read while it is open is diff --git a/docs/transports/tee.md b/docs/transports/tee.md index cc525c4..8bc096a 100644 --- a/docs/transports/tee.md +++ b/docs/transports/tee.md @@ -52,7 +52,7 @@ were left out. | `awaitMirrors` | `false` | Waits for the copies before returning. | | `maxPending` | `100` | The most copies in flight while not awaited. | | `onMirror` | | Hears every outcome of a mirror. | -| `logger` | | A pino-like logger: without `onMirror`, a failed copy at `warn`, in place of a process warning. | +| `logger` | | A pino-like logger: without `onMirror`, a failed copy at `warn` (`tee.mirror_failed`), in place of a process warning. | `sendMany()` goes through the `sendMany()` of the primary, each message mirrored once the primary has answered it. `sendRaw()` mirrors to the mirrors that send raw diff --git a/src/breaker/breaker-transport.js b/src/breaker/breaker-transport.js index 9db795f..4e8b87d 100644 --- a/src/breaker/breaker-transport.js +++ b/src/breaker/breaker-transport.js @@ -206,7 +206,9 @@ class CircuitBreakerTransport { #refusal(remaining) { this.#log.write( 'debug', - () => ({ code: 'breaker.open', remaining }), + // The code of the change of state is breaker.open; this record is one + // per send refused, counted apart. + () => ({ code: 'breaker.refused', remaining }), 'The circuit refused a send.', ); const receipt = createFailedReceipt( diff --git a/src/tee/index.d.ts b/src/tee/index.d.ts index 2908b17..023ec53 100644 --- a/src/tee/index.d.ts +++ b/src/tee/index.d.ts @@ -63,7 +63,7 @@ export interface TeeConfig { readonly onMirror?: (outcome: MirrorOutcome) => void; /** * A pino-like logger: without `onMirror`, it hears a failed copy at `warn` - * (`MAILER_TEE_MIRROR_FAILED`) in place of a process warning. + * (`tee.mirror_failed`) in place of a process warning. */ readonly logger?: Logger; } diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index c0d9351..48fef23 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -199,7 +199,7 @@ class TeeTransport { if (outcome.receipt?.successful !== true && this.#log.hasLogger) { this.#log.write( 'warn', - () => ({ code: 'MAILER_TEE_MIRROR_FAILED', mirror: outcome.mirror }), + () => ({ code: 'tee.mirror_failed', mirror: outcome.mirror }), `The mirror ${outcome.mirror} failed: ${describe(outcome)}`, ); } else if (outcome.receipt?.successful !== true) { diff --git a/tests/types/breaker.test-d.ts b/tests/types/breaker.test-d.ts index 19d8044..4cfc7a9 100644 --- a/tests/types/breaker.test-d.ts +++ b/tests/types/breaker.test-d.ts @@ -33,3 +33,6 @@ expectType>>(breaker.send(message)); expectType>>(breaker.sendMany([message])); new CircuitBreakerTransport(inner); expectError(new CircuitBreakerTransport(inner, { failureThreshold: '5' })); +// A pino-like logger, or none. +new CircuitBreakerTransport(inner, { logger: { warn() {}, info() {}, debug() {} } }); +expectError(new CircuitBreakerTransport(inner, { logger: 'pino' })); diff --git a/tests/types/idempotency.test-d.ts b/tests/types/idempotency.test-d.ts index a8db282..c2f5dd6 100644 --- a/tests/types/idempotency.test-d.ts +++ b/tests/types/idempotency.test-d.ts @@ -69,3 +69,6 @@ expectError({ complete: async () => {}, release: () => {}, }); +// A pino-like logger. +new IdempotencyTransport(inner, { logger: { info() {}, debug() {}, warn() {} } }); +expectError(new IdempotencyTransport(inner, { logger: 'pino' })); diff --git a/tests/types/ratelimit.test-d.ts b/tests/types/ratelimit.test-d.ts index 8c9ca92..f1414b3 100644 --- a/tests/types/ratelimit.test-d.ts +++ b/tests/types/ratelimit.test-d.ts @@ -37,3 +37,7 @@ new RateLimitTransport(inner, { limit: 14, store: { take: async () => ({ taken: false, waitMs: 70 }) }, }); +// A pino-like logger, and a bound on the queue. +new RateLimitTransport(inner, { limit: 1, maxQueue: 100, logger: { info() {}, debug() {} } }); +expectError(new RateLimitTransport(inner, { limit: 1, logger: 'pino' })); +expectError(new RateLimitTransport(inner, { limit: 1, maxQueue: '100' })); diff --git a/tests/types/tee.test-d.ts b/tests/types/tee.test-d.ts index 6abaea3..85cee52 100644 --- a/tests/types/tee.test-d.ts +++ b/tests/types/tee.test-d.ts @@ -32,3 +32,6 @@ const message = createMessage({ expectType>>(tee.send(message)); expectType>>(tee.sendMany([message])); expectError(new TeeTransport(primary, {})); +// A pino-like logger. +new TeeTransport(primary, { mirrors: [primary], logger: { warn() {} } }); +expectError(new TeeTransport(primary, { mirrors: [primary], logger: 'pino' })); diff --git a/tests/unit/decorators.test.js b/tests/unit/decorators.test.js index f56494a..566cbb3 100644 --- a/tests/unit/decorators.test.js +++ b/tests/unit/decorators.test.js @@ -179,14 +179,57 @@ test('Decorators: a logger hears the breaker, the rate limit, idempotency and th await tee.send(mail()); assert.deepStrictEqual(records, [ ['warn', 'breaker.open', 'breaker'], - ['debug', 'breaker.open', 'breaker'], + // Changed after the review: a refusal had the code of the change of state. + ['debug', 'breaker.refused', 'breaker'], ['info', 'ratelimit.limited', 'ratelimit'], ['debug', 'idempotency.hit', 'idempotency'], ['warn', 'idempotency.key_reused', 'idempotency'], - ['warn', 'MAILER_TEE_MIRROR_FAILED', 'tee'], + // Changed after the review: the code of a process warning, not of a record. + ['warn', 'tee.mirror_failed', 'tee'], ]); }); +test('Decorators: a logger hears the circuit half-open and close, and a send in flight', async () => { + // The records no test heard: the other changes of state of the breaker, + // and a key whose send is out. + const records = []; + const logger = {}; + for (const level of ['debug', 'info', 'warn', 'error']) { + logger[level] = (fields) => records.push([level, fields.code]); + } + let time = 0; + let fails = true; + const breaker = new CircuitBreakerTransport( + fakeTransport(() => (fails ? failed('network') : sent('ok'))), + { failureThreshold: 1, resetTimeoutMs: 1000, now: () => time, logger }, + ); + await breaker.send(message()); + time = 1000; + fails = false; + await breaker.send(message()); + assert.deepStrictEqual(records, [ + ['warn', 'breaker.open'], + ['info', 'breaker.half-open'], + ['info', 'breaker.closed'], + ]); + records.length = 0; + const held = {}; + const slow = fakeTransport( + () => + new Promise((resolve) => { + held.release = () => resolve(sent('m')); + }), + ); + const idempotent = new IdempotencyTransport(slow, { logger }); + const first = idempotent.send(message({ idempotencyKey: 'out' })); + while (held.release === undefined) await new Promise((resolve) => setImmediate(resolve)); + const second = await idempotent.send(message({ idempotencyKey: 'out' })); + held.release(); + await first; + assert.strictEqual(second.errors[0].code, 'idempotency.in_flight'); + assert.deepStrictEqual(records, [['info', 'idempotency.in_flight']]); +}); + test('Decorators: a jump of the time of day moves no interval of a default clock', async (t) => { // Before: ratelimit, the memory stores of idempotency and of the webhooks // measured on Date.now, so a correction of the system time refilled a From b0b3faafeb6427cfcd923581b4ac9c2eb0fe159c Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:14:32 +0300 Subject: [PATCH 133/143] fix(jmap): verify tells onRetry of the reads it retries verify() retried its Mailbox/get and Identity/get through the client with no onRetry, so those retries were silent, though the type says the JMAP transport calls it for the reads it retries. The onRetry of its options reaches them; the read is a private method of its own. Co-Authored-By: Claude Opus 5.5 --- src/jmap/jmap-transport.js | 58 +++++++++++++++++----------------- tests/unit/jmap-verify.test.js | 28 ++++++++++++++++ 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/src/jmap/jmap-transport.js b/src/jmap/jmap-transport.js index 907fa20..eaff240 100644 --- a/src/jmap/jmap-transport.js +++ b/src/jmap/jmap-transport.js @@ -111,39 +111,17 @@ class JmapTransport { `JMAP account must be writable with mail and submission capability: ${accountId}`, ); } - // oxlint-disable-next-line no-shadow -- upstream - const get = async (method, args, signal) => { - const response = await verificationOperation( - this.config.timeout, - (owned) => - this.httpClient.executeRequest( - session.apiUrl, - { - using: - method === 'Mailbox/get' - ? [JMAP_CAPABILITIES.core, JMAP_CAPABILITIES.mail] - : [JMAP_CAPABILITIES.core, JMAP_CAPABILITIES.submission], - methodCalls: [[method, { accountId, ...args }, 'verify']], - }, - owned, - ), - signal, - ); - return verificationResult(response, method, accountId); - }; - const mailboxes = await get('Mailbox/get', { properties: ['id', 'role'] }, signal); + const get = (method, args) => + this.#verificationGet(session.apiUrl, accountId, method, args, options); + const mailboxes = await get('Mailbox/get', { properties: ['id', 'role'] }); if (findDraftsMailbox(mailboxes.list, false) === undefined) { throw new JmapApiError(`No drafts mailbox found in JMAP account: ${accountId}`); } const identityId = this.config.identityId; - const identities = await get( - 'Identity/get', - { - ids: identityId == null ? null : [identityId], - properties: ['id', 'email'], - }, - signal, - ); + const identities = await get('Identity/get', { + ids: identityId == null ? null : [identityId], + properties: ['id', 'email'], + }); if ( identities.notFound.length > 0 || identities.list.length === 0 || @@ -163,6 +141,28 @@ class JmapTransport { } } + // One read of verify(), retried as a read is and told to onRetry. + async #verificationGet(apiUrl, accountId, method, args, options) { + const response = await verificationOperation( + this.config.timeout, + (owned) => + this.httpClient.executeRequest( + apiUrl, + { + using: + method === 'Mailbox/get' + ? [JMAP_CAPABILITIES.core, JMAP_CAPABILITIES.mail] + : [JMAP_CAPABILITIES.core, JMAP_CAPABILITIES.submission], + methodCalls: [[method, { accountId, ...args }, 'verify']], + }, + owned, + options?.onRetry, + ), + options?.signal, + ); + return verificationResult(response, method, accountId); + } + /** * Sends a single email message. * @param message The message to send. diff --git a/tests/unit/jmap-verify.test.js b/tests/unit/jmap-verify.test.js index 82b3ae0..e599c32 100644 --- a/tests/unit/jmap-verify.test.js +++ b/tests/unit/jmap-verify.test.js @@ -328,3 +328,31 @@ test('verification follows the structured-send account and honors explicit accou await f.close(); } }); + +test('JMAP verification tells onRetry of a read it retries', async (t) => { + // Not ported: before, verify() retried its Mailbox/get and Identity/get + // with no onRetry, though the type says the reads it retries are told. + const fixture = await setup(); + t.after(() => fixture.close()); + t.mock.method(Math, 'random', () => 0); + const served = globalThis.fetch; + let failed = false; + t.mock.method(globalThis, 'fetch', async (input, init) => { + if (!failed && String(input).endsWith('/api') && String(init.body).includes('Mailbox/get')) { + failed = true; + return new Response('busy', { status: 503 }); + } + return served(input, init); + }); + const transport = new JmapTransport({ + sessionUrl: fixture.base + '/session', + bearerToken: 'token', + identityId: 'identity', + baseUrl: fixture.base, + retries: 1, + }); + const events = []; + await transport.verify({ onRetry: (event) => events.push(event) }); + assert.equal(events.length, 1); + assert.equal(events[0].error.statusCode, 503); +}); From ce0c1de258e7c22c3410822cb168b591dbdccd34 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:15:54 +0300 Subject: [PATCH 134/143] fix(attachments): inlineDataUrls bounds its HTML before it reads it The plan of the review asked for the length of the input of inlineDataUrls to be checked before any scan: maxLength (64 Mi characters) refuses longer HTML with a RangeError. The page says what fromPath checks at each reading (maxBytes, the file by its device and inode, a file renamed over it refused), and the advice on a fetch that checks the address it connects to names one that can be written without undici. Co-Authored-By: Claude Opus 5.5 --- docs/messages/attachments.md | 20 ++++++++++++++------ src/attachments/data-uri.js | 21 ++++++++++++++++++--- src/attachments/index.d.ts | 12 +++++++++++- tests/types/attachments.test-d.ts | 2 ++ tests/unit/attachments.test.js | 11 +++++++++++ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/docs/messages/attachments.md b/docs/messages/attachments.md index c41f15d..33d3343 100644 --- a/docs/messages/attachments.md +++ b/docs/messages/attachments.md @@ -60,11 +60,14 @@ const message = createMessage({ }); ``` -- **`fromPath(path, options)`** checks that the path is a regular file and reads it - again each time the message is read (a DKIM signature, a retry, another transport - of a pool), never holding it. When the path comes from a user, pass `root`: the - file must be inside it once both are resolved with `realpath`, so neither `..` nor - a symbolic link leads out. +- **`fromPath(path, options)`** checks that the path is a regular file of at most + `maxBytes` (25 MiB) and reads it again each time the message is read (a DKIM + signature, a retry, another transport of a pool), never holding it. When the + path comes from a user, pass `root`: the file must be inside it once both are + resolved with `realpath`, so neither `..` nor a symbolic link leads out. Each + reading opens the file that was checked, by its device and inode: one that a + link, a renamed directory or a file renamed over it has put in its place since, + or that has grown past `maxBytes`, fails the reading. - **`fromUrl(url, options)`** fetches http and https URLs once, through the `fetch` you pass or the global one, and holds the bytes. It follows at most `maxRedirects` (5), checks every URL with `allowUrl`, and sends `headers` to the origin of the @@ -79,7 +82,9 @@ const message = createMessage({ `64:ff9b::/96` and `64:ff9b:1::/48`, 6to4 `2002::/16`), are refused. A name that answers the fetch with another address than it gave the check (DNS rebinding) passes, so an application that fetches URLs its users give passes a - `fetch` that checks the address it connects to. + `fetch` that checks the address it connects to: one of its own on `node:https` + whose `lookup` refuses the same ranges, or one that goes through a proxy that + does. - **`fromDataUri(uri, options)`** decodes a base64 or percent-encoded `data:` URI (RFC 2397) up to `maxBytes`, and refuses one that does not parse. - **`fromStream(source, options)`** reads a Node.js `Readable`, a web @@ -102,6 +107,9 @@ const { html, attachments } = inlineDataUrls(renderedHtml, { domain: 'example.co const message = createMessage({ ...fields, content: { html }, attachments }); ``` +Each image holds at most `maxBytes` (25 MiB), and HTML longer than `maxLength` +(64 Mi characters) is refused with a `RangeError` before it is read. + ## Multiple attachments You can attach multiple files to a single message by providing an array of diff --git a/src/attachments/data-uri.js b/src/attachments/data-uri.js index 8a3fff3..c4da9d6 100644 --- a/src/attachments/data-uri.js +++ b/src/attachments/data-uri.js @@ -99,6 +99,10 @@ const fromDataUri = (uri, options = {}) => { // was `[^;]+`, so each `]{0,1024} src\s{0,20}=[\s"']{0,20})(data:([^;"'\s>]{1,127});[^"'>\s]+)/gi; @@ -108,14 +112,25 @@ const DATA_IMAGE = * data URIs. * * @param html The HTML. - * @param options `maxBytes` of one image (25 MiB), `generateId` for the left - * half of each Content-ID and `domain` for its right half (`localhost`). + * @param options `maxBytes` of one image (25 MiB), `maxLength` of the HTML in + * characters (64 Mi), `generateId` for the left half of each Content-ID and + * `domain` for its right half (`localhost`). * @returns The HTML with `cid:` URLs, and the attachments to send with it. * @throws {TypeError} If an image's data URI does not parse. - * @throws {RangeError} If an image holds more than `maxBytes`. + * @throws {RangeError} If an image holds more than `maxBytes`, or the HTML is + * longer than `maxLength`, refused before it is read. */ const inlineDataUrls = (html, options = {}) => { if (typeof html !== 'string') throw new TypeError('inlineDataUrls() needs HTML.'); + const maxLength = options.maxLength ?? DEFAULT_MAX_LENGTH; + if (typeof maxLength !== 'number' || !(maxLength >= 0)) { + throw new TypeError(`maxLength must be a non-negative number: ${String(maxLength)}`); + } + if (html.length > maxLength) { + throw new RangeError( + `inlineDataUrls() takes HTML of at most ${maxLength} characters: ${html.length}`, + ); + } const domain = options.domain ?? 'localhost'; if (typeof domain !== 'string' || !isDotAtomText(domain)) { throw new TypeError(`Invalid Content-ID domain: ${JSON.stringify(domain)}`); diff --git a/src/attachments/index.d.ts b/src/attachments/index.d.ts index 0c69705..e346bbf 100644 --- a/src/attachments/index.d.ts +++ b/src/attachments/index.d.ts @@ -43,7 +43,11 @@ export interface FromPathOptions extends AttachmentOptions { /** Options of {@link fromUrl}. */ export interface FromUrlOptions extends AttachmentOptions { - /** The `fetch` to use, such as one with an undici dispatcher. */ + /** + * The `fetch` to use, such as one that checks the address it connects to + * (a `node:https` request whose `lookup` refuses the ranges `isPublicUrl` + * refuses), which closes the DNS rebinding the default check leaves. + */ readonly fetch?: typeof globalThis.fetch; /** Request header fields, sent to the origin of the URL only. */ readonly headers?: Readonly>; @@ -114,6 +118,12 @@ export interface InlineDataUrlsOptions { * @default 26214400 */ readonly maxBytes?: number; + /** + * The longest HTML taken, in characters: a longer one is refused with a + * `RangeError` before it is read. + * @default 67108864 + */ + readonly maxLength?: number; /** Makes the left half of each Content-ID instead of a UUID. */ readonly generateId?: IdGenerator; /** diff --git a/tests/types/attachments.test-d.ts b/tests/types/attachments.test-d.ts index 2afd90f..4ec60fe 100644 --- a/tests/types/attachments.test-d.ts +++ b/tests/types/attachments.test-d.ts @@ -45,3 +45,5 @@ await fromUrl('https://example.com/a', { }); await fromPath('./a.pdf', { root: '.', maxBytes: 1024 }); expectError(fromUrl('https://example.com/a', { lookup: 'dns' })); +inlineDataUrls('

', { maxLength: 1024 }); +expectError(inlineDataUrls('

', { maxLength: '1024' })); diff --git a/tests/unit/attachments.test.js b/tests/unit/attachments.test.js index 84c2ccf..7afe987 100644 --- a/tests/unit/attachments.test.js +++ b/tests/unit/attachments.test.js @@ -658,3 +658,14 @@ test('Attachments: fromUrl refuses a name that resolves to a private address', a assert.strictEqual(fetched.length, 3); await assert.rejects(fromUrl('https://a.example/', { lookup: 'x' }), /lookup must be a function/); }); + +test('Attachments: inlineDataUrls refuses HTML past maxLength before it reads it', () => { + // Before: nothing bounded the HTML read; the plan of the review asked for + // its length to be checked before any scan. + assert.throws( + () => inlineDataUrls('

x

'.repeat(10), { maxLength: 79 }), + /at most 79 characters: 80/, + ); + assert.strictEqual(inlineDataUrls('

x

', { maxLength: 8 }).html, '

x

'); + assert.throws(() => inlineDataUrls('

', { maxLength: -1 }), /maxLength/); +}); From 4791022a32ce53a9bd7fce69e89a78c76948e77f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:17:22 +0300 Subject: [PATCH 135/143] chore(bench): the cases the plan of the review asked for, and a baseline of them - Gate, every tenth answered in place: one counter was raised by the admissions and the receipts, so nothing was answered in place; they are counted apart, and ten of the hundred are. - Svix and SendGrid timed their signing with their verification; the requests are signed once. - New: an SNS RSA message verified with its certificate kept, fingerprintOf of 1 MiB of HTML, inlineDataUrls of 1 MB with twenty images, and 10,000 sends waiting for one bucket of the rate limit. bench/baseline.json is taken again with the twenty cases, on Node 24.14.1. Co-Authored-By: Claude Opus 5.5 --- bench/baseline.json | 50 +++++++++++++++++++++++++-------------- bench/bench.js | 57 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 23 deletions(-) diff --git a/bench/baseline.json b/bench/baseline.json index 25edd70..f1c87c6 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -1,70 +1,86 @@ { "node": "v24.14.1", - "date": "2026-10-18T00:04:42.115Z", + "date": "2026-10-18T10:17:09.248Z", "results": [ { "name": "MIME — compose a text and HTML message", - "opsPerSec": 9401 + "opsPerSec": 9370 }, { "name": "MIME — compose a message with a 1 MiB attachment", - "opsPerSec": 406 + "opsPerSec": 407 }, { "name": "DKIM — compose and sign a text message (RSA-2048)", - "opsPerSec": 1312 + "opsPerSec": 1359 }, { "name": "SMTP — dot-stuff and write 1 MiB of message data", - "opsPerSec": 745 + "opsPerSec": 714 }, { "name": "Address — isEmailAddress", - "opsPerSec": 3559030 + "opsPerSec": 3580508 }, { "name": "Receipt — classifyReceiptError of a reset", - "opsPerSec": 32283909 + "opsPerSec": 31523312 }, { "name": "Pool — select by priority among 10 transports", - "opsPerSec": 10120402 + "opsPerSec": 10251219 }, { "name": "MIME — compose 100 KB of Cyrillic (base64)", - "opsPerSec": 594 + "opsPerSec": 596 }, { "name": "MIME — compose 100 KB of Latin (quoted-printable)", - "opsPerSec": 750 + "opsPerSec": 724 }, { "name": "Render — htmlToText of 1 MB", - "opsPerSec": 38 + "opsPerSec": 40 }, { "name": "Idempotency — fingerprintOf a message", - "opsPerSec": 74060 + "opsPerSec": 75229 + }, + { + "name": "Idempotency — fingerprintOf 1 MiB of HTML", + "opsPerSec": 685 }, { "name": "Webhooks — verify a Svix (Resend) request", - "opsPerSec": 181926 + "opsPerSec": 291797 }, { "name": "Webhooks — verify a SendGrid ECDSA request", - "opsPerSec": 12227 + "opsPerSec": 17352 }, { "name": "Webhooks — verify a Mailtrap HMAC request", - "opsPerSec": 329750 + "opsPerSec": 322455 + }, + { + "name": "Webhooks — verify an SNS RSA message, its certificate kept", + "opsPerSec": 43484 + }, + { + "name": "Attachments — inlineDataUrls of 1 MB with 20 images", + "opsPerSec": 3212 + }, + { + "name": "RateLimit — 10,000 sends waiting for one bucket", + "opsPerSec": 264 }, { "name": "Gate — 100 messages through five decorators", - "opsPerSec": 1847 + "opsPerSec": 1859 }, { "name": "Gate — 100 messages, every tenth answered in place", - "opsPerSec": 15761 + "opsPerSec": 15928 } ] } diff --git a/bench/bench.js b/bench/bench.js index 9b2baa8..4a1a33d 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -28,6 +28,7 @@ const { readAttachmentContent, } = require('../index.js'); const { composeMessage } = require('../src/mime/index.js'); +const { inlineDataUrls } = require('../src/attachments/index.js'); const { htmlToText } = require('../src/render/index.js'); const { fingerprintOf } = require('../src/idempotency/fingerprint.js'); const { gatedSendMany } = require('../src/internal/gate.js'); @@ -35,7 +36,9 @@ const { createMailtrapWebhookVerifier, createResendWebhookVerifier, createSendGridWebhookVerifier, + createSesWebhookVerifier, } = require('../src/webhooks/index.js'); +const sns = require('../tests/helpers/webhooks/sns.js'); const { CircuitBreakerTransport } = require('../src/breaker/index.js'); const { HooksTransport } = require('../src/hooks/index.js'); const { IdempotencyTransport } = require('../src/idempotency/index.js'); @@ -145,6 +148,17 @@ const latinMessage = createMessage({ const megabyteHtml = '

Hello world, & more

'.repeat( 16000, ); +// The message of a fingerprint of 1 MiB of HTML. +const htmlMessage = createMessage({ + from: ADDRESS, + to: ADDRESS, + subject: 'Newsletter', + content: { html: megabyteHtml, text: 'Newsletter' }, +}); +// 1 MB of HTML with twenty small images as data URIs. +const IMAGE = + 'logo'; +const imageHtml = `${IMAGE}${megabyteHtml.slice(0, 50_000)}`.repeat(20); // Requests signed as each provider signs them, verified without a replay store. const SVIX_SECRET = 'whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'; @@ -210,6 +224,14 @@ const decorated = new HooksTransport( ); const hundred = Array.from({ length: 100 }, () => textMessage); +// 10,000 sends waiting for one bucket, all served by the next refill. +const queued = async () => { + const limiter = new RateLimitTransport(instant, { limit: 1e7, intervalMs: 1000, burst: 1 }); + const sends = []; + for (let i = 0; i < 10_000; i++) sends.push(limiter.send(textMessage)); + return (await Promise.all(sends)).length; +}; + const main = async () => { if (!json) console.log(`Node ${process.version} | ${new Date().toISOString()}\n`); const { privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 }); @@ -242,6 +264,20 @@ const main = async () => { secret: 'mt', replayStore: null, }); + // Signed once: the cases time the verification, not the signing. + const svixSigned = svixRequest(); + const sendgridSigned = sendgridRequest(); + // An SNS message of SES, its certificate read once and kept. + const ses = createSesWebhookVerifier({ + topicArns: [sns.TOPIC], + fetch: sns.certificateFetch().fetch, + now: () => Date.parse('2027-01-01T12:00:30Z'), + replayStore: null, + }); + const snsRequest = { + body: JSON.stringify(sns.notification({ eventType: 'Send', mail: { messageId: 'm' } })), + }; + await ses.verify(snsRequest); const scenarios = [ ['MIME — compose a text and HTML message', () => composeAndRead(textMessage), slow], @@ -278,9 +314,13 @@ const main = async () => { ['MIME — compose 100 KB of Latin (quoted-printable)', () => composeAndRead(latinMessage), slow], ['Render — htmlToText of 1 MB', () => htmlToText(megabyteHtml), slow], ['Idempotency — fingerprintOf a message', () => fingerprintOf(textMessage)], - ['Webhooks — verify a Svix (Resend) request', () => resend.verify(svixRequest())], - ['Webhooks — verify a SendGrid ECDSA request', () => sendgrid.verify(sendgridRequest())], + ['Idempotency — fingerprintOf 1 MiB of HTML', () => fingerprintOf(htmlMessage), slow], + ['Webhooks — verify a Svix (Resend) request', () => resend.verify(svixSigned)], + ['Webhooks — verify a SendGrid ECDSA request', () => sendgrid.verify(sendgridSigned)], ['Webhooks — verify a Mailtrap HMAC request', () => mailtrap.verify(mailtrapRequest)], + ['Webhooks — verify an SNS RSA message, its certificate kept', () => ses.verify(snsRequest)], + ['Attachments — inlineDataUrls of 1 MB with 20 images', () => inlineDataUrls(imageHtml), slow], + ['RateLimit — 10,000 sends waiting for one bucket', queued, slow], [ 'Gate — 100 messages through five decorators', async () => { @@ -293,14 +333,19 @@ const main = async () => { [ 'Gate — 100 messages, every tenth answered in place', async () => { - let n = 0; + // The admissions are counted apart from the receipts, so that every + // tenth message is answered in place. + let admitted = 0; + let sent = 0; for await (const receipt of gatedSendMany(instant, hundred, undefined, { admit: (message) => - ++n % 10 === 0 ? { receipt: { successful: true, messageId: 'own' } } : { message }, + ++admitted % 10 === 0 + ? { receipt: { successful: true, messageId: 'own' } } + : { message }, })) { - if (receipt.successful) n++; + if (receipt.successful) sent++; } - return n; + return sent; }, slow, ], From b6dad9804e09be5bb9d6e00f5352a72f04b7452e Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:18:50 +0300 Subject: [PATCH 136/143] docs: what several processes share, the bugs the audit of the plan found Getting started has a table of the parts that keep state, what each keeps in one process and what several share through a store (SC1 of the plan). CLAUDE.md lists R21 to R41, found by the audit of the review's plan against the code, records the decision to keep one ./webhooks entry, names the modules internal/ gained (abort.js, iterate.js, warn.js, provider-transport.js), and says SES is on the shared client too. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 73 +++++++++++++++++++++++++++++++---- docs/guide/getting-started.md | 19 +++++++++ 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8d77539..969f19b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -128,14 +128,15 @@ Copyright 2025 Hong Minhee; the notice is in `LICENSE`): one subpath per upyo pa test that fails without it, not during the port. - **The HTTP clients are this package's, the API of each provider upstream's.** The fixes 24 to 39 had to land in seven copies of the same client, and one missed a copy (Mailtrap, bug 29), so - since 1.1 the clients of Resend, SendGrid, Mailgun, Maileroo, Lettermint, Plunk and Mailtrap - are their requests over `createApiClient` (`src/internal/api-client.js`), as those of 1.1 are: + since 1.1 the clients of SES, Resend, SendGrid, Mailgun, Maileroo, Lettermint, Plunk and + Mailtrap are their requests over `createApiClient` (`src/internal/api-client.js`), as those of + 1.1 are, SES's SigV4 signature the `authorize` of each attempt: what stays upstream's is the provider's side (its authorization, the messages it is read for, what its 2xx must hold, its error classes), passed in as hooks. Their public errors (`SendGridApiError`…) extend `ProviderApiError` with the names, arguments and fields of 1.0, and - the transports build their receipts with `providerFailure`. SES, which signs each attempt with - SigV4, and JMAP, with its session, keep their own clients; `tests/unit/api.test.js` fails when - another provider calls `requestWithRetry` on its own. + the transports build their receipts with `providerFailure`. JMAP, with its session, keeps its + own client; `tests/unit/api.test.js` fails when another provider calls `requestWithRetry` on + its own. The transports of 1.1 extend `ProviderTransport` (`src/internal/provider-transport.js`). - A lint rule that upstream code breaks is suppressed on the line, `// oxlint-disable-next-line -- upstream`, rather than rewritten; lifting those is a later refactor, like TS `private` fields, which stay plain properties (the tests reach into them). @@ -744,6 +745,58 @@ with a test that fails without it; R for review): `bounced` hard, soft or undetermined as it says; `rejected` with nothing sent; `unknown` rather than a guess) holds for all, and `tests/unit/webhooks-mapping.test.js` pins every provider's table. +- R21. gate: the first message of a run was admitted before the wrapped `sendMany()` took it, + so one that threw or returned at once left it neither failed nor abandoned (a key pending, a + probe held). Its slot is there from the start, and a message never taken goes to + `gate.unsent`, which `./idempotency` answers by freeing its key. +- R22. breaker: a `now()` that threw fell back to `Date.now`, another clock, and put a circuit + some fifty years off; it gives the last time it gave. +- R23. ratelimit: a bucket not yet full was dropped as readily as a full one, silently; any + number of messages could wait; a message whose tokens were there missed an abort between two + of a batch; a shared store that gave no `waitMs` was asked every millisecond. Full buckets go + first and an early drop is told (`MAILER_RATELIMIT_EVICTED`), `maxQueue` bounds the waiters, + the signal is read before the tokens, a store is asked again after the rate, and + `ratelimit.waited` tells the wait. +- R24. sendmail: a message that could not be read sent SIGINT alone, which a process could + ignore for good; it gets SIGKILL after the grace. +- R25. tee: the copies started before the primary's send; they start once it has answered. +- R26. hooks: a copy a `beforeSend` returned skipped the checks of `createMessage`, so a line + break in a name or an address reached the converters; `checkMessage` of `src/core/message.js` + runs them again. +- R27. webhooks: SNS certificates were cached by the URL as written, and kept though the + signature failed, so spellings and forged messages read certificates and pushed the one in use + out. A certificate URL is taken only as the URL parser writes it, and kept once it verified. +- R28. webhooks: a Scaleway anchor that was no CA, or no longer valid, issued; the chain must + hold CAs, valid when they are trusted. +- R29. webhooks: SparkPost's classes 25 and 26 (its own refusals), 60, 80 and 100 (no failure) + and 90 (an unsubscribe) were bounces; a Maileroo failure was hard and a Maileroo or Scaleway + deferral a soft bounce, though neither provider said so. They follow the rule of the guide. +- R30. webhooks: the verifiers of providers that sign no id took `replayStore` and `leaseMs` + and never read them; an Azure `validationUrl` with a password passed; the lease was said to + be shorter than SNS's first retry, which it is not. +- R31. webhooks: `commit` and `release` of the replay store took the id alone, so a request + whose lease ran out could free the reservation of the next; a delivery reserves under a token + of its own, and the memory store tells when it drops committed ids early. +- R32. webhooks: the guide found a message of unknown delivery by `rfcMessageId`, which the HTTP + APIs write themselves; an event carries `correlationId` of the message's metadata. +- R33. a timeout over 2^31 - 1 ms fired at once, and `AbortSignal.timeout` threw past 2^32 - 1; + `timerMs` of `internal/sleep.js` cuts each to the longest a timer keeps. +- R34. a logger that threw was a warning per record with its text as it was; it is told once + per log by its name, and `./idempotency` redacts the addresses of a key in its records. +- R35. receipts of the library's own conditions had no category; each names one, or the status + it is told by, and `tests/unit/internal.test.js` fails on a call that names none. +- R36. api: `classify()` of a provider reached the receipt alone, so a retryable 420 of + SparkPost was not retried in the send; the shared client passes `isRetryable`. +- R37. verify: a 403 verified the key of every provider; only those whose sending keys are + refused the read with one (`acceptForbidden`: SES, Mailgun, SparkPost, MailerSend, + Scaleway), and with no error marked verified to carry it. +- R38. postmark, mailjet: a batch was converted whole, its files in base64, before its first + request; a request goes as soon as the next message would not fit. +- R39. breaker, tee: `breaker.open` named the change of state and every refused send, and the + tee wrote the code of its process warning in its records: `breaker.refused`, + `tee.mirror_failed`. +- R40. jmap: `verify()` retried its reads with no `onRetry`. +- R41. attachments: `inlineDataUrls` read HTML of any length; `maxLength` bounds it. ## Repository layout @@ -754,13 +807,14 @@ src/ version.js the package name and version (OpenTelemetry scope), kept equal to package.json / one directory per subpath: index.js + index.d.ts + its modules internal/ private modules shared by subpaths, not in the exports map: the request - layer (http.js, api-client.js for every HTTP provider but SES and JMAP, + layer (http.js, api-client.js for every HTTP provider but JMAP, api.js for the conversions of the providers of 1.1, body.js, unsent.js, delivery-unknown.js, config.js, loopback.js, validate-ssl.js, verify.js), receipts (provider.js, refusal.js), message fields (headers.js, raw.js, schedule.js, tracking.js, capabilities.js, batch.js, base64.js, uuid.js), the decorators of 1.1 (gate.js, callback.js, options.js, decorator.js), - and redact.js, redact-addresses.js, sleep.js, log.js + the base of the transports of 1.1 (provider-transport.js), and abort.js, + iterate.js, warn.js, redact.js, redact-addresses.js, sleep.js, log.js tests/ unit/*.test.js node:test suites, one per area (-.test.js for a subpath) runtime/*.test.js the package by its name on Node, Bun and Deno, with no mock @@ -872,6 +926,11 @@ engine of its own; done in 1.1, `./render`); more providers (done in 1.1). ported from upyo"), and the correctness suite of Phase 1 still applies to it. - **BullMQ is one driver, not the base:** it ties the queue to Redis/Valkey; the MongoDB outbox gives a queue with no new infrastructure. +- **One `./webhooks` entry, not one per provider.** Its fifteen verifiers come to some 15 KB + min+gzip, which a function that verifies one provider pays at its cold start; fifteen entries + more would be fifteen more parts of the API to keep under semver, for less than a kilobyte + each. If that cost is ever measured to matter, the entries can come later without a change + to the one there is. - **Eighteen providers, then few new ones.** The eight HTTP providers of upyo came with the port; 1.1 adds the ten most used of unemail and sently (Postmark, Brevo, Mailjet, MailerSend, SparkPost, ZeptoMail, SMTP2GO, Azure Communication Services, Scaleway and Cloudflare). Each diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 59890d2..355ebfe 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -204,3 +204,22 @@ recipients, threading and List-Unsubscribe, [attachments](/messages/attachments) See [Why @alexify/mailer?](/guide/why) for what the library is built to do, and the [API](/api/exports) for every entry point. + +## Several processes + +Some parts keep state, and by default they keep it in the process. Several +processes, or a function that makes its transports for each request, each have +their own unless the part is given a store they share: + +| Part | What it keeps | In one process | Shared by several | +| --- | --- | --- | --- | +| [`./ratelimit`](/transports/ratelimit) | the token buckets | at most `maxKeys` buckets | a `store` of `take()`, such as a script on Redis, so that N processes send no more than the one limit | +| [`./idempotency`](/transports/idempotency) | the keys reserved and the receipts | `createMemoryIdempotencyStore()` | a store of `reserve`, `complete` and `release`, the outbox to come among them | +| [`./webhooks`](/guide/webhooks#time-and-replays) | the ids of the requests taken | `createMemoryReplayStore()` | a store of `reserve`, `commit` and `release`; required, `null` to keep none | +| [`./breaker`](/transports/breaker) | the state of the circuit | one per transport | none: each process opens its own circuit on the failures it sees | +| [`./pool`](/transports/pool) | the turn of round-robin | one per transport | none | +| [`./smtp`](/transports/smtp) | pooled connections, OAuth 2.0 tokens | one pool per transport | none | + +A store of a rate limit or of idempotency that each process makes for itself +gives no more than the memory one: the limit is spent N times over, and a key is +sent once per process. From a983baa710d91948ed4a7605811e6f97744b2669 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:19:34 +0300 Subject: [PATCH 137/143] docs(changelog): what the audit of the plan changed in 1.1 Upgrading from 1.0: verify() takes a 403 only where a sending key gets one, and a logger that throws is told once. Added: the bounded queue of the rate limit, the new log codes, the tee mirroring after the primary, the token of the replay store, the category and code of a failure in the metrics, and the maxLength of inlineDataUrls. Fixed: a timeout past what a timer keeps, and the retries of JMAP's verify. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6b1909..e2b7b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,11 +43,17 @@ additive but the defaults listed below. transport that can send it; and a TLS certificate refused in the handshake is a non-retryable `configuration` failure at the first attempt, where SMTP reported it as `unknown` and the HTTP transports retried it and reported a retryable `network` failure (`deliveryUnknown` on Deno). -- **The HTTP providers of 1.0 share one client:** Resend, SendGrid, Mailgun, Maileroo, +- **The HTTP providers of 1.0 share one client:** SES, Resend, SendGrid, Mailgun, Maileroo, Lettermint, Plunk and Mailtrap request through the client of the providers of 1.1. Their errors (`SendGridApiError`, `MailtrapTimeoutError`…) keep their names, arguments and fields, and carry `provider`, `category` and `retryable` besides; an error answer whose body cannot be read keeps its status; and a message Mailgun cannot express is an `unsupported` failure, as elsewhere. +- **`verify()` takes a 403 only where a key that may only send gets one:** SES, Mailgun and, of + 1.1, SparkPost, MailerSend and Scaleway. A 403 of SendGrid or Resend fails `verify()`, where it + passed it. +- **A logger that throws is told once:** the first failure of the logger of a transport or + decorator is a process warning (`MAILER_LOGGER_FAILED`) by the name and code of its error, not + every record with its text, which could name a recipient. - **On Deno, give the HTTP transports a `fetch` that keeps no idle connection** (`Deno.createHttpClient({ poolMaxIdlePerHost: 0 })`): Deno's own sends a request again when a reused connection closes before its answer. See @@ -74,7 +80,8 @@ additive but the defaults listed below. host as written: no `localhost`, and no IP literal of a loopback, private, link-local, multicast, reserved or documentation range, NAT64 and 6to4 included, as written or as a name resolves), refuses a redirect from https to http, and bounds time and bytes; `fromPath` can be - held to a `root` by real path and to `maxBytes`, and reads the file it checked. + held to a `root` by real path and to `maxBytes`, and reads the file it checked; + `inlineDataUrls` refuses HTML longer than `maxLength` before it reads it. - **`./render`:** `renderContent(renderer, input)` for the content of `createMessage`, `createReactEmailRenderer({ render })` with the `render` of `@react-email/render` passed in, and `htmlToText`, linear in the length of its input. @@ -105,8 +112,9 @@ additive but the defaults listed below. `probeTimeoutMs`. - **`./ratelimit`:** a token bucket per `key`, waiting first in, first out, or refusing with a retryable `rate-limit` receipt; an aborted send leaves the queue, and a cost over `burst` is - refused. At most `maxKeys` buckets are kept in the process, or a `store` of buckets that - processes share holds them (`take(key, cost, limits)`). + refused. At most `maxKeys` buckets are kept in the process, the full ones dropped first, or a + `store` of buckets that processes share holds them (`take(key, cost, limits)`); at most + `maxQueue` messages wait in a bucket. - **`./idempotency`:** a key reserved in a store before the send, so a repeat is answered with the first receipt, a send in flight as of unknown delivery, and the same key for another message refused. A reservation lasts `leaseMs` (10 minutes), so a key a crash left reserved is @@ -116,10 +124,11 @@ additive but the defaults listed below. - **`./hooks`:** `beforeSend` may replace or refuse a message, `afterSend`, `onRetry` and `onError` hear each send, with a context made for each that carries its signal. - **A `logger` for `./breaker`, `./ratelimit`, `./idempotency` and `./tee`,** pino-like, with a - code for each event (`breaker.open`, `ratelimit.waiting`, `idempotency.in_flight`…). -- **`./tee`:** mirrors each message to other transports, with one Message-ID, each with a - timeout of its own and not awaited by default, the primary's receipt the one returned; - disposal waits for the copies in flight. + code for each event (`breaker.open`, `breaker.refused`, `ratelimit.waiting`, + `ratelimit.waited`, `idempotency.in_flight`, `tee.mirror_failed`…). +- **`./tee`:** mirrors each message to other transports once the primary has answered it, with + one Message-ID, each copy with a timeout of its own and not awaited by default, the primary's + receipt the one returned; disposal waits for the copies in flight. - **Delivery events and `./webhooks`:** the `DeliveryEvent` model at the root, one event per message and recipient, and a verifier per provider for SES, Resend, SendGrid, Mailgun, Postmark, MailerSend, Mailtrap, Brevo, Mailjet, SparkPost, Lettermint, Maileroo, SMTP2GO, Azure @@ -130,12 +139,15 @@ additive but the defaults listed below. handled its events, so a delivery whose handler failed is taken again, and another delivery of it meanwhile is refused as `pending`. `replayStore` is required where a verifier keeps ids (`createMemoryReplayStore()` in one process, a shared store of `reserve`, `commit` and - `release`, or `null`), and a refusal carries the `status` to answer the provider with. + `release` under the token of each delivery, or `null`), and a refusal carries the `status` to + answer the provider with. `onVerify` hears each request taken or refused, and an event carries the `correlationId` of the message's metadata, given back by every provider but Azure, to find the events of a message of unknown delivery, and the `rfcMessageId` the provider reports. - **`./pool` `verifyAll()`:** how each transport did, `verified`, `failed` or `unverifiable`, where `verify()` passes when one does. +- **Failures stand out in the metrics:** `mailer.email.messages` of `./otel` takes the category + and the code of a failure (`error.type`, `mailer.error.code`). - **A failure of unknown delivery stands out:** `./otel` sets `mailer.delivery_unknown` on its span and its message counter, and `./logger` writes `deliveryUnknown` in `email.failed`, for the messages to reconcile. @@ -178,6 +190,10 @@ additive but the defaults listed below. could send (`validation`, `rejected`) stops it. - **A pooled SMTP connection was reused for any number of messages,** while servers limit the messages of one connection (`maxMessages`, above). +- **A timeout over 2^31 - 1 ms fired at once:** `timeout: 3e9` timed out every request of an + HTTP provider, and an SMTP socket at once; such a timeout is now the longest a timer keeps, in + effect none. +- **`verify()` of `./jmap` told `onRetry` of none of the reads it retried;** it tells it of each. ## [1.0.0] - 2026-10-17 From ad710c1a36e8bab26ab360806d780f758eb914aa Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:20:14 +0300 Subject: [PATCH 138/143] test(runtime): the default check of fromUrl resolves a name on every runtime The plan of the review asked for the lookup of the default allowUrl to be checked in tests/runtime, which runs on Node, Bun and Deno: a name that resolves to a private or the metadata address is refused before any request, and one that resolves to a public address is read. Co-Authored-By: Claude Opus 5.5 --- tests/runtime/http.test.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/runtime/http.test.js b/tests/runtime/http.test.js index cbaf751..1c209b4 100644 --- a/tests/runtime/http.test.js +++ b/tests/runtime/http.test.js @@ -192,3 +192,33 @@ test('Runtime attachments: fromUrl reads through the fetch of the runtime, check ); assert.strictEqual(requests.length, before + 1); }); + +test('Runtime attachments: the default check resolves a name, a private answer refused', async () => { + // The default allowUrl on each runtime: a name whose lookup gives a private + // or a cloud metadata address is refused before any request, and one that + // gives a public address passes the check. + const asked = []; + const lookup = async (hostname) => { + asked.push(hostname); + if (hostname === 'internal.example.com') return [{ address: '10.0.0.1', family: 4 }]; + if (hostname === 'metadata.example.com') return [{ address: '169.254.169.254', family: 4 }]; + return [{ address: '93.184.215.14', family: 4 }]; + }; + let fetched = 0; + const fetch = async () => { + fetched++; + return new Response('ok', { headers: { 'content-type': 'text/plain' } }); + }; + for (const host of ['internal.example.com', 'metadata.example.com']) { + await assert.rejects(fromUrl(`https://${host}/a.txt`, { lookup, fetch }), RangeError); + } + assert.strictEqual(fetched, 0); + const attachment = await fromUrl('https://public.example.com/a.txt', { lookup, fetch }); + assert.strictEqual(attachment.filename, 'a.txt'); + assert.strictEqual(fetched, 1); + assert.deepStrictEqual(asked, [ + 'internal.example.com', + 'metadata.example.com', + 'public.example.com', + ]); +}); From a2d2229d209073297718f45ef9f3e664a81e03dc Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 13:22:02 +0300 Subject: [PATCH 139/143] refactor: drop convertBatch and the reject of verify, which nothing reaches Postmark and Mailjet convert each message as they read it, so nothing calls convertBatch; and verify's requests take every answer through accepts(), so its reject is never called. Co-Authored-By: Claude Opus 5.5 --- src/internal/api.js | 33 +++------------------------------ src/internal/verify.js | 1 - 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/internal/api.js b/src/internal/api.js index c829fb9..8b8dd35 100644 --- a/src/internal/api.js +++ b/src/internal/api.js @@ -1,7 +1,7 @@ // Not ported: what the HTTP providers added in 1.1 share, once rather than a -// copy in each: the client of api-client.js, and the conversion of a batch in -// which a message that cannot be converted fails alone, its attachments and -// header fields, and the steps of one send. +// copy in each: the client of api-client.js, the receipts of a batch in the +// order of its messages, their attachments, addresses and header fields, and +// the steps of one send. const { toAsciiAddress } = require('../core/address.js'); const { readAttachmentContent } = require('../core/attachment.js'); @@ -18,32 +18,6 @@ const { truncateErrorText, } = require('./api-client.js'); -/** - * Converts each message of a batch, one that fails to convert failing alone. - * - * @param messages The messages. - * @param convert Converts one message. - * @param provider The provider's id, for the receipt of a failure. - * @param signal Rethrows the caller's abort. - * @returns `{ payloads, slots }`: a slot is the index of a payload, or the - * receipt of a message that did not convert. - * @internal - */ -const convertBatch = async (messages, convert, provider, signal) => { - const payloads = []; - const slots = []; - for (let i = 0; i < messages.length; i++) { - try { - payloads.push(await convert(messages[i])); - slots.push(payloads.length - 1); - } catch (error) { - if (isCallerAbort(error, signal)) throw error; - slots.push(providerFailure(provider, error)); - } - } - return { payloads, slots }; -}; - /** * The receipts of a batch in the order of its messages: those of * `receiptOf(index)` for the payloads that were sent, and the failures of the @@ -212,7 +186,6 @@ module.exports = { ProviderApiError, base64Attachments, checkCredential, - convertBatch, createApiClient, headerFieldsOf, isCallerAbort, diff --git a/src/internal/verify.js b/src/internal/verify.js index b07a93f..60b0de3 100644 --- a/src/internal/verify.js +++ b/src/internal/verify.js @@ -55,7 +55,6 @@ const verifyByRequest = (request) => if (request.check?.(status, body) === true) return; throw failureOf(request.provider, status, body); }, - reject: async (response) => failureOf(request.provider, response.status, ''), }); /** From ce804cd7da13945cb75185eb15123f63ba0ef19f Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 16:53:07 +0300 Subject: [PATCH 140/143] fix(tee): the deadline of an awaited mirror keeps the process alive The deadline of every mirror was unref'd, as a detached copy should not hold the process. An awaited one is what its caller waits for, so a script whose mirror hung on a promise ended with the send unsettled, and on Node 22 the suite of ./tee was cancelled from its awaitMirrors test on. Only a detached copy's deadline is unref'd now. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++-- CLAUDE.md | 3 +++ src/tee/tee-transport.js | 7 +++++-- tests/unit/tee.test.js | 27 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b7b69..e3ab669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,8 +127,9 @@ additive but the defaults listed below. code for each event (`breaker.open`, `breaker.refused`, `ratelimit.waiting`, `ratelimit.waited`, `idempotency.in_flight`, `tee.mirror_failed`…). - **`./tee`:** mirrors each message to other transports once the primary has answered it, with - one Message-ID, each copy with a timeout of its own and not awaited by default, the primary's - receipt the one returned; disposal waits for the copies in flight. + one Message-ID, each copy with a timeout of its own and not awaited by default (with + `awaitMirrors`, awaited until that timeout at most, which keeps the process alive for it), the + primary's receipt the one returned; disposal waits for the copies in flight. - **Delivery events and `./webhooks`:** the `DeliveryEvent` model at the root, one event per message and recipient, and a verifier per provider for SES, Resend, SendGrid, Mailgun, Postmark, MailerSend, Mailtrap, Brevo, Mailjet, SparkPost, Lettermint, Maileroo, SMTP2GO, Azure diff --git a/CLAUDE.md b/CLAUDE.md index 969f19b..6c07249 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -797,6 +797,9 @@ with a test that fails without it; R for review): `tee.mirror_failed`. - R40. jmap: `verify()` retried its reads with no `onRetry`. - R41. attachments: `inlineDataUrls` read HTML of any length; `maxLength` bounds it. +- R42. tee: the deadline of an awaited mirror was unref'd, as that of a detached one is, so a + script whose mirror hung on nothing that kept the process alive ended with its send unsettled + (on Node 22, so did the suite of `./tee`). The caller waits for it, so it keeps the process. ## Repository layout diff --git a/src/tee/tee-transport.js b/src/tee/tee-transport.js index 48fef23..a6c82bd 100644 --- a/src/tee/tee-transport.js +++ b/src/tee/tee-transport.js @@ -28,7 +28,7 @@ const describe = (outcome) => { // A send of a mirror that does not settle by `timeout`, as one that ignores // its signal would not, fails then all the same. -const withDeadline = (sending, timeout, mirror) => { +const withDeadline = (sending, timeout, mirror, detached) => { let timer; const deadline = new Promise((resolve, reject) => { timer = setTimeout( @@ -36,7 +36,9 @@ const withDeadline = (sending, timeout, mirror) => { timerMs(timeout), ); // A detached send does not keep the process alive; disposal waits for it. - timer.unref?.(); + // An awaited one does, as its caller waits for the deadline: a mirror hung + // on nothing else would end the process with the send unsettled. + if (detached) timer.unref?.(); }); return Promise.race([sending, deadline]).finally(() => clearTimeout(timer)); }; @@ -173,6 +175,7 @@ class TeeTransport { : mirror.send(message, { signal: mirrorSignal }), this.config.timeout, mirror.id, + detached, ); const settled = sending.then( (receipt) => this.#report({ mirror: mirror.id, message, receipt }), diff --git a/tests/unit/tee.test.js b/tests/unit/tee.test.js index eb82ba0..28cb7d3 100644 --- a/tests/unit/tee.test.js +++ b/tests/unit/tee.test.js @@ -1,5 +1,6 @@ // Not ported: tests of ./tee, ported from unemail's tee driver, against fake // transports. +const { execFileSync } = require('node:child_process'); const { test } = require('node:test'); const assert = require('node:assert'); const { createMessage } = require('../../index.js'); @@ -77,6 +78,32 @@ test('Tee: awaitMirrors waits for them, bounded, and they follow the caller', as assert.strictEqual(outcomes.at(-1), 'caller left'); }); +test( + 'Tee: an awaited mirror hung on nothing fails at its timeout, the process kept for it', + { skip: globalThis.Deno !== undefined && 'spawns the CLI with -e, which Deno does not take' }, + () => { + // Before: its deadline did not keep the process alive, so a script whose + // mirror hung on a promise ended with the send unsettled, nothing said. + const script = ` + const { createMessage } = require(${JSON.stringify(require.resolve('../../index.js'))}); + const { TeeTransport } = require(${JSON.stringify(require.resolve('../../src/tee/index.js'))}); + const primary = { id: 'p', send: async () => ({ successful: true, messageId: 'p' }) }; + const tee = new TeeTransport(primary, { + mirrors: [{ id: 'm', send: () => new Promise(() => {}) }], + awaitMirrors: true, + timeout: 20, + onMirror: (outcome) => console.log(outcome.error.message), + }); + const message = createMessage({ + from: 'a@example.com', to: 'b@example.com', subject: 's', content: { text: 'x' }, + }); + tee.send(message).then((receipt) => console.log('sent', receipt.messageId)); + `; + const output = execFileSync(process.execPath, ['-e', script], { encoding: 'utf8' }); + assert.strictEqual(output, 'The mirror m did not answer within 20 ms.\nsent p\n'); + }, +); + test('Tee: a mirror never changes the primary, which alone throws', async (t) => { const warnings = []; t.mock.method(process, 'emitWarning', (text, options) => warnings.push([text, options.code])); From 28ddf0bccbcc98eaa834ebf1a5ad87a4dab57e90 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 16:53:09 +0300 Subject: [PATCH 141/143] test(webhooks): the hanging certificate fetch holds a handle, as a request does The stub held nothing that kept the process alive, and the certificate timeout of SNS is an AbortSignal.timeout, unref'd, so Node 22 ended the test before the timeout fired. A real request holds its socket. Co-Authored-By: Claude Opus 5.5 --- tests/unit/webhooks-ses.test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/webhooks-ses.test.js b/tests/unit/webhooks-ses.test.js index 75ca67d..d96172c 100644 --- a/tests/unit/webhooks-ses.test.js +++ b/tests/unit/webhooks-ses.test.js @@ -500,7 +500,13 @@ test('Webhooks SES: the signature versions taken, and the time a certificate is certificateTimeoutMs: 5, fetch: (url, init) => new Promise((resolve, reject) => { - init.signal.addEventListener('abort', () => reject(init.signal.reason)); + // A request in flight holds its socket, which keeps the process + // alive until the timeout; without one, Node 22 ends the test first. + const socket = setTimeout(() => {}, 60_000); + init.signal.addEventListener('abort', () => { + clearTimeout(socket); + reject(init.signal.reason); + }); }), }); await assert.rejects( From 404a702d4ede744218f617c50a8b73ad47566a15 Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 16:53:09 +0300 Subject: [PATCH 142/143] test(runtime): count the attempts at an untrusted certificate through a relay The HTTPS server of Bun emits no `connection`, so the count stayed 0 there though the receipt was right. A TCP relay in front of it counts each connection on Node, Bun and Deno alike. Co-Authored-By: Claude Opus 5.5 --- tests/runtime/http.test.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/runtime/http.test.js b/tests/runtime/http.test.js index 1c209b4..4a9b71e 100644 --- a/tests/runtime/http.test.js +++ b/tests/runtime/http.test.js @@ -4,11 +4,18 @@ const { test } = require('node:test'); const assert = require('node:assert'); const http = require('node:http'); const https = require('node:https'); +const net = require('node:net'); const { createMessage, readAttachmentContent } = require('@alexify/mailer'); const { PostmarkTransport } = require('@alexify/mailer/postmark'); const { fromUrl } = require('@alexify/mailer/attachments'); const { credentials } = require('../helpers/smtp/servers.js'); +// Listens on a free port of the loopback address, closed after the test. +const listen = async (t, listener) => { + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => listener.close(() => resolve()))); +}; + // A server that records each request with its body, then answers as // `answer(request, response, count)` says. const server = async (t, answer) => { @@ -147,17 +154,23 @@ test( test('Runtime HTTP: a certificate of an unknown CA fails at once, as configuration', async (t) => { // An HTTPS server with the certificate of the test CA, which the runtime - // does not trust; each TCP connection is an attempt. - let connections = 0; + // does not trust, behind a TCP relay that counts each connection, an + // attempt: the HTTPS server of Bun emits no `connection`. const listener = https.createServer(credentials, (_request, response) => accepted(response)); - listener.on('connection', (socket) => { + listener.on('tlsClientError', () => {}); + await listen(t, listener); + let connections = 0; + const relay = net.createServer((socket) => { connections++; + const upstream = net.connect(listener.address().port, '127.0.0.1'); socket.on('error', () => {}); + upstream.on('error', () => {}); + socket.on('close', () => upstream.destroy()); + upstream.on('close', () => socket.destroy()); + socket.pipe(upstream).pipe(socket); }); - listener.on('tlsClientError', () => {}); - await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); - t.after(() => new Promise((resolve) => listener.close(() => resolve()))); - const url = `https://localhost:${listener.address().port}`; + await listen(t, relay); + const url = `https://localhost:${relay.address().port}`; const receipt = await postmark(url, { retries: 2 }).send(message()); assert.strictEqual(receipt.successful, false); assert.strictEqual(receipt.errors[0].category, 'configuration', receipt.errorMessages.join()); From 5dd05b7ed468545019f0aba43ca86eb0739ca85d Mon Sep 17 00:00:00 2001 From: Alex Dolid Date: Sun, 18 Oct 2026 17:19:51 +0300 Subject: [PATCH 143/143] test(render): time htmlToText at 256 KB and 1 MB, four times apart Between 1 and 2 MB, linear is 2 and quadratic 4, and a runner of CI (Windows on Node 22 and 24, Deno) took 3.1 to 3.3 for `<`, a garbage collection more in the larger run, though the code is linear (2.0 on a machine at rest). Four times apart, linear is 4 and quadratic 16, and 8 is as far from each; the smaller inputs also hold less for the collector and halve the time of the test. The quadratic code still fails it. Co-Authored-By: Claude Opus 5.5 --- tests/unit/render.test.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/unit/render.test.js b/tests/unit/render.test.js index 65b72dc..c7710a9 100644 --- a/tests/unit/render.test.js +++ b/tests/unit/render.test.js @@ -149,12 +149,17 @@ test('Render: htmlToText takes time in proportion to its input, not its square', }; const megabyte = 1024 * 1024; for (const piece of ['a

', '<', 'x']) { - const one = bestOf(piece.repeat(Math.ceil(megabyte / piece.length))); - if (piece === 'a

') assert.ok(one < 1000, `1 MB of ${piece} took ${one} ms`); - const two = bestOf(piece.repeat(Math.ceil((2 * megabyte) / piece.length))); - // Linear is about 2; the floor keeps a few milliseconds of noise from - // deciding it. - assert.ok(two <= 3 * Math.max(one, 25), `${piece}: 1 MB ${one} ms, 2 MB ${two} ms`); + const quarter = bestOf(piece.repeat(Math.ceil(megabyte / 4 / piece.length))); + const whole = bestOf(piece.repeat(Math.ceil(megabyte / piece.length))); + if (piece === 'a

') assert.ok(whole < 1000, `1 MB of ${piece} took ${whole} ms`); + // Linear is about 4 and quadratic 16: 8 is as far from each, so that the + // noise of a busy runner (a garbage collection more in the larger run) + // does not decide it, as it did between 1 and 2 MB; the floor keeps a few + // milliseconds of it from deciding either. + assert.ok( + whole <= 8 * Math.max(quarter, 25), + `${piece}: 256 KB ${quarter} ms, 1 MB ${whole} ms`, + ); } });