From 77ec53a40ec8842c5040dc2865bb567e72531a66 Mon Sep 17 00:00:00 2001 From: robobun Date: Fri, 2 Oct 2026 23:49:55 -0700 Subject: [PATCH 01/15] node:tls: send a client's FIN after the first step of its handshake (#43962) ### What A TLS client that calls `end()` before its first handshake step (in the tick of `tls.connect()`, in the next tick, or inside `'connect'`) sent a bare FIN with no ClientHello. It never reported a certificate that it refuses, and against a server that keeps its side open it never exited. Node sends the ClientHello first. This is a regression that has not shipped: #42181 made `_final` send the FIN at once. The script of #17190 passes on 1.4.2 and fails on `main`. ### How - `us_internal_ssl_on_open` runs the `'connect'` listeners and then sends the ClientHello. `us_internal_ssl_shutdown` now holds the FIN back in between, and `us_internal_ssl_on_open` sends it right after the ClientHello. - Only `tls.connect()` opts in, from a `'connect'` listener that it registers first. That is where Node registers `_start` ([wrap.js#L1795](https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L1795)). `Bun.connect`, `upgradeTLS`, a second `connect()` of the same `TLSSocket` and `new TLSSocket().connect()` never set the bit. ### Evidence - `node-tls-duplex-end-verify.test.ts`: 68 pass on a debug and on a release build. Under `node --test` (v26.7.0), 67 pass and 1 is skipped. - Regression test for #17190. ### Not in this PR With the ClientHello sent, an early `end()` behaves like an `end()` one turn after `'connect'` does on `main`. Three differences from Node that `main` has there now show for an early `end()` too. Measured on earlier heads of this branch: - TLS 1.2, `rejectUnauthorized`, untrusted chain: `error UNABLE_TO_VERIFY_LEAF_SIGNATURE`, where Node reports `end`, `error ECONNRESET`. - More than 10 such clients at once in one process: the ones that wait in the low-priority handshake queue report `ECONNRESET` and never read the server's flight. - A TLS 1.2 session that is offered for another server name: `error ERR_TLS_CERT_ALTNAME_INVALID`, where Node offers no session and reports `ECONNRESET`. Also measured on earlier heads, inside the tick in which the FIN is held: - `end()` and then `resetAndDestroy()`: the FIN or the reset is missing, where `main` sends both. Node throws on `resetAndDestroy()` of a TLS socket. - `_handle.write()` after `_handle.shutdown()` returns 0, where `main` returns -1. `end()` over a Duplex, or over a wrapped socket that is still connecting, is still lost, as on `main`. --------- Co-authored-by: Ciro Spaciari MacBook Co-authored-by: Dylan Conway --- packages/bun-usockets/src/crypto/openssl.c | 16 ++ packages/bun-usockets/src/internal/internal.h | 4 + packages/bun-usockets/src/libusockets.h | 2 + src/js/node/tls.ts | 8 + src/runtime/socket/mod.rs | 5 +- src/runtime/socket/socket_body.rs | 14 ++ src/uws_sys/socket.rs | 7 + src/uws_sys/us_socket_t.rs | 6 + .../tls/node-tls-duplex-end-verify.test.ts | 151 ++++++++++++++++++ test/regression/issue/17190.test.ts | 26 +++ 10 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 test/regression/issue/17190.test.ts diff --git a/packages/bun-usockets/src/crypto/openssl.c b/packages/bun-usockets/src/crypto/openssl.c index 0af5f87c3093..22d8caa9ca23 100644 --- a/packages/bun-usockets/src/crypto/openssl.c +++ b/packages/bun-usockets/src/crypto/openssl.c @@ -1143,6 +1143,10 @@ void us_socket_set_inline_reject(struct us_socket_t *s) { SSL_set_verify(s_ssl(s), SSL_VERIFY_PEER, us_inline_reject_verify_callback); } +void us_socket_set_first_flight_before_fin(struct us_socket_t *s) { + s->ssl_first_flight_before_fin = 1; +} + /* Drop the strdup'd passphrase. Called as soon as private-key load completes * (the only consumer of the passwd_cb), so the secret never outlives ctx * construction and SSL_CTX_free() is sufficient on every later path. Also @@ -1714,6 +1718,8 @@ void us_internal_ssl_attach(struct us_socket_t *s, SSL_CTX *ctx, s->ssl_in_use = 0; s->ssl_pending_detach = 0; s->ssl_pending_close_code = 0; + s->ssl_first_flight_before_fin = 0; + s->ssl_shutdown_after_first_flight = 0; s->ssl_is_server = is_client ? 0 : 1; s->ssl_inline_reject = 0; s->ssl_verify_failed = 0; @@ -2227,6 +2233,12 @@ struct us_socket_t *us_internal_ssl_on_open(struct us_socket_t *s, int is_client /* Kick the handshake immediately — some peers stall waiting for ClientHello. */ ssl_set_loop_data(result); ssl_update_handshake(result, 1); + if (ssl_gone(result)) return result; + result->ssl_first_flight_before_fin = 0; + if (result->ssl_shutdown_after_first_flight) { + result->ssl_shutdown_after_first_flight = 0; + us_internal_ssl_shutdown(result); + } return result; } @@ -2818,6 +2830,10 @@ int us_internal_ssl_writev(struct us_socket_t *s, const struct us_iovec_t *iov, void us_internal_ssl_shutdown(struct us_socket_t *s) { if (us_socket_is_closed(s) || us_internal_ssl_is_shut_down(s)) return; + if (s->ssl_first_flight_before_fin) { + s->ssl_shutdown_after_first_flight = 1; + return; + } /* Spilled ciphertext is data the layers above already count as written; * a FIN/close_notify now would cut it off. Finish the shutdown from the diff --git a/packages/bun-usockets/src/internal/internal.h b/packages/bun-usockets/src/internal/internal.h index a42d4e449ccb..d0364eb665a5 100644 --- a/packages/bun-usockets/src/internal/internal.h +++ b/packages/bun-usockets/src/internal/internal.h @@ -361,6 +361,10 @@ struct us_socket_t { * inside a handshake callback must still RST, not FIN, when it is finally * performed). */ unsigned char ssl_pending_close_code : 2; + /* us_socket_set_first_flight_before_fin: the first handshake step is still due, and a shutdown waits for it. */ + unsigned char ssl_first_flight_before_fin : 1; + /* us_internal_ssl_shutdown held its FIN back for that step. */ + unsigned char ssl_shutdown_after_first_flight : 1; /* Consecutive send() failures with an errno that is neither * would-block/transient nor a known peer-gone error (see * us_socket_write_check_error). Reset by any send that makes progress. diff --git a/packages/bun-usockets/src/libusockets.h b/packages/bun-usockets/src/libusockets.h index a623ac7ad2ac..a11ffb59fa34 100644 --- a/packages/bun-usockets/src/libusockets.h +++ b/packages/bun-usockets/src/libusockets.h @@ -389,6 +389,8 @@ void us_socket_start_tls_handshake(us_socket_r s) nonnull_fn_decl; * server that fails verification. Must run before the handshake is driven * (on_open, or between adopt_tls and start_tls_handshake). No-op otherwise. */ void us_socket_set_inline_reject(us_socket_r s) nonnull_fn_decl; +/* Call it from on_open. A shutdown before the first handshake step sends its FIN after that step. */ +void us_socket_set_first_flight_before_fin(us_socket_r s) nonnull_fn_decl; /* ── Listen ─────────────────────────────────────────────────────────────── * The listener owns: an embedded group for accepted sockets, the SSL_CTX diff --git a/src/js/node/tls.ts b/src/js/node/tls.ts index 2a1c29228c56..64c2b6c0a8fc 100644 --- a/src/js/node/tls.ts +++ b/src/js/node/tls.ts @@ -5,6 +5,7 @@ const Duplex = require("internal/streams/duplex"); const EventEmitter = require("node:events"); const addServerName = $newRustFunction("Listener.rs", "jsAddServerName", 3); const setListenerSecureContext = $newRustFunction("Listener.rs", "jsSetSecureContext", 2); +const firstFlightBeforeFin = $newRustFunction("runtime/socket/socket.rs", "jsFirstFlightBeforeFin", 1); const { throwNotImplemented } = require("internal/shared"); const { idnaToASCII } = require("internal/url"); const { @@ -1670,9 +1671,16 @@ function connect(...args) { if (timeout) { tlssock.setTimeout(timeout); } + // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L1795 + if (!options.socket) tlssock.once("connect", onConnectStart); return tlssock.connect(normal); } +// The engine sends the ClientHello when the listeners of 'connect' have run. An end() of theirs must not get ahead of it. +function onConnectStart() { + firstFlightBeforeFin(this._handle); +} + function getCiphers() { return getDefaultCiphers().split(":"); } diff --git a/src/runtime/socket/mod.rs b/src/runtime/socket/mod.rs index bfe0c55935a1..51aec89f319e 100644 --- a/src/runtime/socket/mod.rs +++ b/src/runtime/socket/mod.rs @@ -120,8 +120,9 @@ pub(crate) use udp_socket::UDPSocket; /// the name the generator expects rather than special-casing the generator. pub(crate) mod socket { pub(crate) use super::socket_body::{ - js_create_socket_pair, js_get_buffered_amount, js_is_named_pipe_socket, - js_set_socket_options, js_upgrade_duplex_to_tls, js_upgrade_tls_deferred, testing_ap_is, + js_create_socket_pair, js_first_flight_before_fin, js_get_buffered_amount, + js_is_named_pipe_socket, js_set_socket_options, js_upgrade_duplex_to_tls, + js_upgrade_tls_deferred, testing_ap_is, }; } diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs index 1bf3d1a77988..84dfc2309afc 100644 --- a/src/runtime/socket/socket_body.rs +++ b/src/runtime/socket/socket_body.rs @@ -4656,6 +4656,20 @@ pub(crate) fn js_upgrade_tls_deferred( Err(global.throw(format_args!("Expected a socket instance"))) } +/// `tls.connect()`'s first 'connect' listener, where node sends the ClientHello. +#[bun_jsc::host_fn] +pub(crate) fn js_first_flight_before_fin( + _global: &JSGlobalObject, + callframe: &CallFrame, +) -> JsResult { + jsc::mark_binding!(); + let [socket] = callframe.arguments_as_array::<1>(); + if let Some(this) = socket.as_class_ref::() { + this.socket.get().set_first_flight_before_fin(); + } + Ok(JSValue::UNDEFINED) +} + #[bun_jsc::host_fn] pub(crate) fn js_upgrade_duplex_to_tls( global: &JSGlobalObject, diff --git a/src/uws_sys/socket.rs b/src/uws_sys/socket.rs index f9faf3445070..32d37ee3bae6 100644 --- a/src/uws_sys/socket.rs +++ b/src/uws_sys/socket.rs @@ -572,6 +572,13 @@ impl NewSocketHandler { } } + /// A shutdown before the first handshake step sends its FIN after that step. + pub fn set_first_flight_before_fin(&self) { + if let InternalSocket::Connected(s) = self.socket { + sock(s).set_first_flight_before_fin(); + } + } + /// The session an SSLWrapper-backed socket got last from the new-session callback, borrowed. pub fn wrapper_latest_session(&self) -> *mut bun_boringssl_sys::SSL_SESSION { match self.socket { diff --git a/src/uws_sys/us_socket_t.rs b/src/uws_sys/us_socket_t.rs index db3284f2564f..09f8e8d86611 100644 --- a/src/uws_sys/us_socket_t.rs +++ b/src/uws_sys/us_socket_t.rs @@ -309,6 +309,11 @@ impl us_socket_t { c::us_socket_set_inline_reject(self); } + /// A shutdown before the first handshake step sends its FIN after that step. + pub fn set_first_flight_before_fin(&mut self) { + c::us_socket_set_first_flight_before_fin(self); + } + /// Feed bytes that were already read off the wire (e.g. a ClientHello the /// plain-TCP layer consumed before the upgrade) through the same decrypt /// path as bytes arriving from the kernel. @@ -591,6 +596,7 @@ mod c { ) -> *mut us_socket_t; pub(super) safe fn us_socket_start_tls_handshake(s: &mut us_socket_t); pub(super) safe fn us_socket_set_inline_reject(s: &mut us_socket_t); + pub(super) safe fn us_socket_set_first_flight_before_fin(s: &mut us_socket_t); } } diff --git a/test/js/node/tls/node-tls-duplex-end-verify.test.ts b/test/js/node/tls/node-tls-duplex-end-verify.test.ts index 81dd7b370813..f4a8c29d2cb3 100644 --- a/test/js/node/tls/node-tls-duplex-end-verify.test.ts +++ b/test/js/node/tls/node-tls-duplex-end-verify.test.ts @@ -981,3 +981,154 @@ test("a bad record behind the client's Finished does not make a server accept an // Node reports the bad record. Its code depends on the cipher, so only the class of the error is fixed. assert.match(events[0], isBun ? /^tlsClientError DEPTH_ZERO_SELF_SIGNED_CERT$/ : /^tlsClientError ERR_SSL_/); }); + +// A client calls end() before the first step of its handshake. Returns the ordered events of the client. +async function endBeforeClientHello( + when, + maxVersion, + rejectUnauthorized, + { trusted = false, servername = "agent1" } = {}, +) { + const events = []; + const { promise, resolve } = Promise.withResolvers(); + const server = tls.createServer({ key, cert, maxVersion }, socket => socket.on("error", () => {})); + server.on("tlsClientError", () => {}); + await new Promise(listening => server.listen(0, "127.0.0.1", listening)); + const client = tls.connect({ + port: server.address().port, + host: "127.0.0.1", + servername, + rejectUnauthorized, + maxVersion, + ...(trusted && { ca: serverCA }), + }); + if (when === "in the same tick") client.end(); + else if (when === "in the next tick") process.nextTick(() => client.end()); + else client.on("connect", () => client.end()); + for (const event of ["finish", "secureConnect", "end"]) client.on(event, () => events.push(event)); + client.on("error", err => events.push(`error ${err.code}`)); + client.on("close", () => { + events.push("close"); + resolve(); + }); + await promise; + server.close(); + return events; +} + +for (const when of ["in the same tick", "in the next tick", "inside 'connect'"]) { + test(`TLSv1.3: end() ${when} still refuses an untrusted certificate`, async () => { + assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", true), [ + "finish", + "error UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "close", + ]); + }); + + test(`TLSv1.3: end() ${when} still refuses a certificate for another name`, async () => { + const wrongName = { trusted: true, servername: "another.name" }; + assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", true, wrongName), [ + "finish", + "error ERR_TLS_CERT_ALTNAME_INVALID", + "close", + ]); + }); + + test(`TLSv1.3: end() ${when} still completes the handshake`, async () => { + assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", false), [ + "finish", + "secureConnect", + "end", + "close", + ]); + }); + + test(`TLSv1.2: end() ${when} reports the handshake that the server cannot complete`, async () => { + // The client cannot send its second flight after the FIN, so the handshake ends when the server closes. + assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.2", false), [ + "finish", + "end", + "error ECONNRESET", + "close", + ]); + }); +} + +test("end() inside 'connect' still reports a ClientHello that the client cannot build", async () => { + const events = []; + const { promise, resolve } = Promise.withResolvers(); + const server = tls.createServer({ key, cert }, socket => socket.on("error", () => {})); + server.on("tlsClientError", () => {}); + await new Promise(listening => server.listen(0, "127.0.0.1", listening)); + // No protocol version is inside this window, so the first step of the handshake fails. + const client = tls.connect({ + port: server.address().port, + host: "127.0.0.1", + rejectUnauthorized: false, + minVersion: "TLSv1.3", + maxVersion: "TLSv1.2", + }); + client.on("connect", () => client.end()); + client.on("secureConnect", () => events.push("secureConnect")); + client.on("error", err => events.push(`error ${err.code}`)); + client.on("close", () => { + events.push("close"); + resolve(); + }); + await promise; + server.close(); + // OpenSSL and BoringSSL name the reason differently. + assert.match(events.join(", "), /^error ERR_SSL_NO_(PROTOCOLS_AVAILABLE|SUPPORTED_VERSIONS_ENABLED), close$/); +}); + +test("end() inside 'connect' sends the ClientHello before the FIN", async () => { + const { promise, resolve, reject } = Promise.withResolvers(); + let accepted; + const server = net.createServer({ allowHalfOpen: true }, socket => { + accepted = socket; + const received = []; + socket.on("error", reject); + socket.on("data", chunk => received.push(chunk)); + socket.on("end", () => resolve(Buffer.concat(received))); + }); + await new Promise(listening => server.listen(0, "127.0.0.1", listening)); + const client = tls.connect({ port: server.address().port, host: "127.0.0.1", rejectUnauthorized: false }); + client.on("error", () => {}); + client.on("connect", () => client.end()); + const beforeFin = await promise; + client.destroy(); + accepted.destroy(); + server.close(); + // One complete handshake record: the ClientHello. + assert.deepStrictEqual( + { type: beforeFin[0], complete: beforeFin.length >= 5 && beforeFin.length === 5 + beforeFin.readUInt16BE(3) }, + { type: 22, complete: true }, + ); +}); + +test("end() after a second connect() of the same socket sends no ClientHello", async () => { + // Only tls.connect() starts a handshake: https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L1795 + const received = []; + const second = Promise.withResolvers(); + const server = net.createServer(socket => { + let bytes = 0; + socket.on("error", second.reject); + socket.on("data", chunk => (bytes += chunk.length)); + socket.on("end", () => received.push(bytes) === 2 && second.resolve()); + }); + await new Promise(listening => server.listen(0, "127.0.0.1", listening)); + const where = { port: server.address().port, host: "127.0.0.1" }; + const client = tls.connect({ ...where, rejectUnauthorized: false }); + client.on("error", () => {}); + client.end(); + await new Promise(closed => client.once("close", closed)); + client.connect(where); + client.end(); + try { + await second.promise; + assert.strictEqual(received[1], 0); + } finally { + client.destroy(); + server.close(); + } +}); diff --git a/test/regression/issue/17190.test.ts b/test/regression/issue/17190.test.ts new file mode 100644 index 000000000000..ff045e325a7b --- /dev/null +++ b/test/regression/issue/17190.test.ts @@ -0,0 +1,26 @@ +// https://github.com/oven-sh/bun/issues/17190 +import { expect, test } from "bun:test"; +import { tls as selfSigned } from "harness"; +import { once } from "node:events"; +import type { AddressInfo } from "node:net"; +import tls from "node:tls"; + +test("tls.connect() followed at once by end() reports a self-signed certificate", async () => { + const server = tls.createServer(selfSigned, socket => socket.on("error", () => {})); + server.on("tlsClientError", () => {}); + await once(server.listen(0, "127.0.0.1"), "listening"); + try { + const socket = tls.connect({ + host: "127.0.0.1", + port: (server.address() as AddressInfo).port, + rejectUnauthorized: true, + }); + const { promise, resolve } = Promise.withResolvers(); + socket.on("error", err => resolve(`error ${(err as NodeJS.ErrnoException).code}`)); + socket.on("close", () => resolve("close with no error")); + socket.end(); + expect(await promise).toBe("error DEPTH_ZERO_SELF_SIGNED_CERT"); + } finally { + server.close(); + } +}); From bb35d1b811726bf34ef7a7505d278c32119634a8 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 2 Oct 2026 23:57:35 -0700 Subject: [PATCH 02/15] clone() of a Bun.file() body no longer freezes what the Bun.file() answers or throws for a body that cannot be read (#44504) ### What does this PR do? Fixes regressions on main from #42053, which is in no release. Replaces #43916. ```js const f = Bun.file(path); // 5 bytes new Response(f).clone(); // nothing reads a body await Bun.write(path, "123456789"); f.size; // 1.4.2: 9, main: 5 ``` After `clone()` of a body that is a `Bun.file(path)`: | | 1.4.2 | main | this PR | |---|---|---|---| | file grows 5 to 9 bytes, then `f.size` | 9 | 5 | 9 | | same, then `new Response(f).body`, or `.body` of either clone | `123456789` | `12345` | `123456789` | | `f.lastModified` after `utimes`, `f.exists()` after `unlink` | fresh | stale | fresh | | the path is a directory | both bodies reject with `EISDIR` | `clone()` throws | both bodies reject with `EISDIR` | | `Bun.file(fd)` of a directory, or of an fd that is not open | both bodies reject (`EISDIR`, `EBADF`) | `clone()` throws | both bodies reject (`EISDIR`, `EBADF`) | | a procfs file: both bodies and `f.text()` | whole | `""` | whole | The same holds for `Request.clone()`, `new Request(request)`, `new Request(request, init)` and `new Request(url, response)`. **Cause.** `clone()` has to know whether a path names something that yields its bytes once (a pipe, a terminal), because two Blobs over that would compete and the body is teed instead. #42053 asks with `resolve_file_stat`, which writes `mode`, `seekable`, `max_size` and `last_modified` on the `Store`. The body shares that `Store` with the `Bun.file()` it was made from, and nothing invalidates those fields, so `clone()` became one more call that freezes them. Separately, its rule was "tee whatever is not a regular file", and a tee over a directory cannot start. **Fix.** `store_yields_bytes_once` (new, `src/runtime/webcore/Blob.rs`) answers for a Blob body, and `Value::clone_with_readable_stream` calls it in place of `store_reads_repeatably`. - It keeps the mode it stats in a field of its own, `mode_seen_by_clone`, which nothing else reads. That is the fix for the stale answers and for procfs. - It does not count a directory as yielding bytes once, by path or by fd, nor an fd that `fstat` fails for. There are no bytes to compete for, so the body is duped like a regular file and each body fails when it is read, as in 1.4.2. A failed `clone()` on main also leaves the original body used. - For an fd it asks with `resolve_file_stat`, which is what the tee of an fd does next (`to_readable_stream` resolves the size). So the `fstat` only moves earlier, and an fd body is cached as on main. - It still stats once per `Store` and still uses a stat that `size` or `.body` cached earlier, so `clone()` makes the same syscalls as on main. **Kept from #42053:** a body over an fd, `Bun.stdin`, a FIFO by path and a terminal by path is teed, and a cloned `Bun.file().stream()` body keeps its type. **This is a partial fix.** The arm for an unread `.stream()` body is not touched, so `new Response(f.stream()).clone()` still caches the stat: `f.size` and the rest stay stale after it, and a procfs `.stream()` body reads `""`. Taking the cache away there makes the bodies, which #42053 turns back into Blobs, behave in ways that neither 1.4.2 nor main has. Most of those are the right value, but `console.log` of a cloned `slice().stream()` body would print `0.0 ZB`. That needs its own decision, so it is left out. **Nothing that main answers correctly after a `clone()` gets worse.** Main has two answers right only because `clone()` writes the cache. Taking the write away alone would bring two older bugs back for code that clones, so they are fixed where they are: - `f.exists()` stayed `false` for good once a stat had failed, with or without a `clone()` (`await f.exists()`, create the file, `await f.exists()`). `get_exists_sync` now stats again while no stat has succeeded. - `console.log` of a `Response` or `Request` over `Bun.file(p).slice(1, 4)` prints `0.0 ZB` until something stats the file, and on main `clone()` was that something. `get_size_for_bindings` now also accepts the mode `clone()` saw, so a cloned slice prints `3 bytes` as on main. - `0.0 ZB` is `u64::MAX`, "unknown", run through the byte formatter. It is no longer printed: `Response {` and `Request {` carry no size when it is unknown. This is what makes the label right for a file that was deleted after the clone, where main prints the size it had. - Printing such a body left the 0 of the failed stat in it. The next `console.log` would then have printed `0 KB`, and `text()` read `""` even after the file was created, with or without a `clone()`, on 1.4.2 too. `get_size_for_bindings` no longer keeps that 0. A slice that was never cloned still has no size that `console.log` can print. It printed `0.0 ZB` and now prints none. An empty file and a procfs file give the same `stat`, so cutting the slice to `st_size` gets one of them wrong, and I left that alone. **Why not #43916.** It drops the stat for a Blob body, so a FIFO or a terminal given by path as the Blob is duped again and one body gets nothing. It stats uncached for a `.stream()` body, which costs 1000 `newfstatat` per 1000 clones of streams from one shared `Bun.file()`, where main pays 1. And it has the new behaviours described under "partial fix". ### How did you verify your code works? Tests, each run on a debug build of main, on this branch and on 1.4.2: | | main | here | 1.4.2 | |---|---|---|---| | `body-clone.test.ts`, 12 regression tests: `size` and a later `.body` through each of the five ways to clone, `lastModified` and `exists()`, both bodies after the file grew, the size `console.log` prints, a procfs file, a directory by path and by fd, an fd that is not open | fail | pass | 11 pass, the printed size fails | | `body-clone.test.ts`, 3 tests that pin what main has right: a FIFO given by path as the Blob itself is teed, so is a regular file given by fd, and `exists()` sees a file that was created before the clone | pass | pass | fail | | `bun-file-exists.test.js`, 2 tests: one `Bun.file()` sees a file that is created after `exists()` or `size` found none | fail | pass | fail | | `inspect.test.js`, 2 tests: no size is printed for a missing file or a directory, and printing a body whose file does not exist yet leaves it as it is | fail | pass | fail | With the `is_fd` clause of `store_yields_bytes_once` disabled, the fd test is the only one in `body-clone.test.ts` that fails. A differential run on Linux x64 of this branch against a debug build of main at the same base. It crosses 12 kinds of body (regular, empty, large, symlink, missing, directory, `/dev/null`, procfs, fd of a file, fd of a directory, fd that is not open, in-memory Blob), 4 earlier reads of the handle, 6 body forms, the 5 ways to clone plus 2 controls that do not clone, up to 12 changes to the file before or after the clone, and 29 observations, each in a fresh directory with a fresh handle. Two of them do a thing twice: print twice, and print and then read. The rule it checks: **a row that differs from main has to be correct**, not merely what an older version did. Each scenario records what is on disk at that moment (existence, size, mtime, content). A changed row has to match that, or, where no such truth can be derived, what the same scenario answers without a `clone()`. A row where main matches the truth and this branch does not is reported as a regression on its own. There is none. | of 187,404 rows | | |---|---| | same as main | 183,155 | | differs, matches what is on disk | 3,108 | | differs, same as without a `clone()` | 775 | | differs, prints no size where main prints `0.0 ZB` and the size could be known | 342 | | differs, wrong | 0 | | a `/dev/null` body served with `Bun.serve`, which flips between a reset and a 500 page on main between its own runs | 2 | | not an observation of `clone()`: `fetch()` refuses a GET `Request` with a body, for every kind of body on both builds | 22 | Extra syscalls for 1000 clones (`strace`, N=1000 minus N=0), `newfstatat` / `fstat` / `openat`: | | before #42053 | main | this PR | |---|---|---|---| | `new Response(Bun.file(p))` | 0/0/0 | 1000/0/0 | 1000/0/0 | | one shared `Bun.file()` | 0/0/0 | 1/0/0 | 1/0/0 | | `Bun.file(p).stream()` body | 0/1000/1000 | 1000/0/0 | 1000/0/0 | | `shared.stream()` body | 0/1000/1000 | 1/0/0 | 1/0/0 | | `Bun.file(fd)` body | not measured | 0/2000/1 | 0/2000/1 | | one shared `Bun.file(fd)` | not measured | 0/1001/1 | 0/1001/1 | Unchanged from main: 48 cases over a FIFO, `/dev/stdin`, `Bun.stdin` and a file replaced by a FIFO (3 runs each), `/dev/tty` and `/dev/stdin` under a pty, and `/dev/null` and `/dev/zero` slices. Suites on the debug build, no failures: `body-clone`, `body`, `response`, `blob`, `blob-write`, `body-stream`, `request`, `FormData`, `bun-file`, `bun-file-exists`, `inspect`, `console-log`, `bun-serve-file`, `bun-serve-static`. In `fetch.test.ts` and `bun-write.test.js` one test each fails with `EFBIG`, here and on main, because I ran them under a 200 MB file size limit. `bun run rust:check-all` passes for `x86_64-pc-windows-msvc` and `aarch64-apple-darwin`. Not verified: Windows and macOS at runtime. The two fd tests are skipped on Windows. --- src/jsc/webcore_types.rs | 3 + src/runtime/webcore/Blob.rs | 52 ++++++- src/runtime/webcore/Body.rs | 16 ++- src/runtime/webcore/Request.rs | 15 +- src/runtime/webcore/Response.rs | 17 ++- test/js/bun/util/bun-file-exists.test.js | 15 +- test/js/bun/util/inspect.test.js | 28 ++++ test/js/web/fetch/body-clone.test.ts | 168 ++++++++++++++++++++++- 8 files changed, 288 insertions(+), 26 deletions(-) diff --git a/src/jsc/webcore_types.rs b/src/jsc/webcore_types.rs index 591d3e8e2238..cbb70e72303f 100644 --- a/src/jsc/webcore_types.rs +++ b/src/jsc/webcore_types.rs @@ -791,6 +791,8 @@ pub mod store { pub max_size: SizeType, /// Milliseconds since ECMAScript epoch. pub last_modified: crate::JSTimeType, + /// `st_mode` as a body's `clone()` saw it, kept apart from what the `Bun.file()` sharing this store answers from. + pub mode_seen_by_clone: Option, } impl Default for File { @@ -803,6 +805,7 @@ pub mod store { seekable: None, max_size: MAX_SIZE, last_modified: crate::INIT_TIMESTAMP, + mode_seen_by_clone: None, } } } diff --git a/src/runtime/webcore/Blob.rs b/src/runtime/webcore/Blob.rs index f85d1129ff0f..fc47dea081f4 100644 --- a/src/runtime/webcore/Blob.rs +++ b/src/runtime/webcore/Blob.rs @@ -1254,8 +1254,13 @@ impl BlobExt for Blob { } fn get_exists_sync(&self) -> JSValue { - if self.size.get() == MAX_SIZE { - self.resolve_size(); + match self.store.get() { + _ if self.size.get() == MAX_SIZE => self.resolve_size(), + // A failed stat is not an answer to keep: the file may exist by now. + Some(store) if matches!(&store.data, store::Data::File(file) if file.seekable.is_none()) => { + resolve_file_stat(store) + } + _ => {} } // If there's no store that means it's empty and we just return true @@ -1996,7 +2001,8 @@ impl BlobExt for Blob { } fn get_size_for_bindings(&self) -> u64 { - if self.size.get() == MAX_SIZE { + let has_size = self.size.get() != MAX_SIZE; + if !has_size { self.resolve_size(); } @@ -2004,7 +2010,17 @@ impl BlobExt for Blob { // signal that the size is unknown. if let Some(store) = self.store.get() { if let store::Data::File(file) = &store.data { - if !file.seekable.unwrap_or(false) { + // A slice has its size without a stat, and `clone()` learns the mode without filling `seekable`. + let mode_seen_by_clone = file.mode_seen_by_clone.filter(|_| has_size); + if !file + .seekable + .or_else(|| mode_seen_by_clone.map(bun_sys::S::ISREG)) + .unwrap_or(false) + { + // Printing is not to leave the 0 of a failed stat behind: the body would read as empty. + if !has_size && file.seekable.is_none() { + self.size.set(MAX_SIZE); + } return u64::MAX; } } @@ -5836,6 +5852,34 @@ pub(crate) fn store_reads_repeatably(store: &RefPtr) -> bool { } } +/// Whether two Blobs over `store` would compete for its bytes (an fd, a pipe, a terminal). A directory or a closed fd has none: each Blob fails when read. +pub(crate) fn store_yields_bytes_once(store: &RefPtr) -> bool { + if !matches!(store.data, store::Data::File(_)) { + return false; + } + let is_fd = Store::data_mut(store).as_file().pathlike.is_fd(); + // What the tee of an fd does next, so this costs no extra `fstat`. + if is_fd && Store::data_mut(store).as_file().seekable.is_none() { + resolve_file_stat(store); + } + let file = Store::data_mut(store).as_file_mut(); + let mode = if file.seekable.is_some() { + Some(file.mode) + } else if let PathOrFileDescriptor::Path(path) = &file.pathlike { + // Not `resolve_file_stat`: the `Bun.file()` sharing `store` answers from what that caches. + if file.mode_seen_by_clone.is_none() { + let mut buffer = bun_paths::path_buffer_pool::get(); + if let bun_sys::Result::Ok(stat) = bun_sys::stat(path.slice_z(&mut buffer)) { + file.mode_seen_by_clone = Some(stat.st_mode as bun_sys::Mode); + } + } + file.mode_seen_by_clone + } else { + None + }; + mode.is_some_and(|mode| !bun_sys::S::ISDIR(mode) && (is_fd || !bun_sys::S::ISREG(mode))) +} + // ────────────────────────────────────────────────────────────────────────── // toStringWithBytes / toString / toJSON / toFormData / toArrayBuffer{View} // ────────────────────────────────────────────────────────────────────────── diff --git a/src/runtime/webcore/Body.rs b/src/runtime/webcore/Body.rs index 66e2e1315c08..8ede5c11927c 100644 --- a/src/runtime/webcore/Body.rs +++ b/src/runtime/webcore/Body.rs @@ -127,8 +127,8 @@ impl Body { unsafe { self.value.get_mut() } } - pub(crate) fn len(&self) -> blob::SizeType { - self.value_mut().size() + pub(crate) fn known_len(&self) -> Option { + self.value_mut().known_size() } } @@ -767,6 +767,14 @@ impl Value { } } + /// `None` for a file that does not exist or is not seekable. + pub(crate) fn known_size(&mut self) -> Option { + match self.size() { + u64::MAX => None, + size => Some(size as usize), + } + } + pub(crate) fn memory_cost(&self) -> usize { match self { Value::InternalBlob(b) => b.memory_cost(), @@ -1571,9 +1579,7 @@ impl Value { } if let Value::Blob(b) = self { - if b.store() - .is_some_and(|store| !blob::store_reads_repeatably(store)) - { + if b.store().is_some_and(blob::store_yields_bytes_once) { // A pipe or other fd yields its bytes once: read it as one // stream and tee that. self.to_readable_stream(cx)?; diff --git a/src/runtime/webcore/Request.rs b/src/runtime/webcore/Request.rs index 1fe691fb59e2..ca3ae8efdc8f 100644 --- a/src/runtime/webcore/Request.rs +++ b/src/runtime/webcore/Request.rs @@ -540,12 +540,15 @@ impl Request { } else { "BunRequest" }; - writeln!( - writer, - "{} ({}) {{", - class_label, - bun_fmt::size(self.body_value_mut().size() as usize, Default::default()) - )?; + match self.body_value_mut().known_size() { + Some(size) => writeln!( + writer, + "{} ({}) {{", + class_label, + bun_fmt::size(size, Default::default()) + )?, + None => writeln!(writer, "{} {{", class_label)?, + } { // RAII guard restores indent on every exit incl. `?` error paths. // Shadows `formatter` for the block; auto-derefs to `&mut F`. diff --git a/src/runtime/webcore/Response.rs b/src/runtime/webcore/Response.rs index 7608783c95a6..cfb5970bc8fd 100644 --- a/src/runtime/webcore/Response.rs +++ b/src/runtime/webcore/Response.rs @@ -427,10 +427,6 @@ impl Response { impl Response { #[inline] - pub(crate) fn get_body_len(&self) -> usize { - self.body.get().len() as usize - } - pub(crate) fn get_form_data_encoding( &self, ) -> JsResult>> { @@ -646,11 +642,14 @@ impl Response { // `fmt::Error`. let js_err = |_: JsError| core::fmt::Error; - writeln!( - writer, - "Response ({}) {{", - bun_core::fmt::size(self.get_body_len(), Default::default()) - )?; + match self.body.get().known_len() { + Some(len) => writeln!( + writer, + "Response ({}) {{", + bun_core::fmt::size(len, Default::default()) + )?, + None => writeln!(writer, "Response {{")?, + } { let mut formatter = formatter.indented(); diff --git a/test/js/bun/util/bun-file-exists.test.js b/test/js/bun/util/bun-file-exists.test.js index dc1a810c497c..f8526c84f68f 100644 --- a/test/js/bun/util/bun-file-exists.test.js +++ b/test/js/bun/util/bun-file-exists.test.js @@ -1,6 +1,7 @@ import { write } from "bun"; import { expect, test } from "bun:test"; -import { unlinkSync } from "fs"; +import { unlinkSync, writeFileSync } from "fs"; +import { tempDir } from "harness"; import { tmpdir } from "os"; import { join } from "path"; test("bun-file-exists", async () => { @@ -18,3 +19,15 @@ test("bun-file-exists", async () => { unlinkSync(temp); expect(await Bun.file(temp).exists()).toBeFalse(); }); + +test.each([ + ["exists()", file => file.exists()], + ["size", file => file.size], +])("one Bun.file() sees a file that is created after %s found none", async (_, look) => { + using dir = tempDir("bun-file-exists-later", {}); + const file = Bun.file(join(String(dir), "later.txt")); + await look(file); + expect(await file.exists()).toBeFalse(); + writeFileSync(file.name, "boop"); + expect(await file.exists()).toBeTrue(); +}); diff --git a/test/js/bun/util/inspect.test.js b/test/js/bun/util/inspect.test.js index 931e45c783d0..3549700bb920 100644 --- a/test/js/bun/util/inspect.test.js +++ b/test/js/bun/util/inspect.test.js @@ -141,6 +141,34 @@ it("Blob inspect", () => { }`); }); +it("Response and Request print no size for a Bun.file() body whose size is unknown", () => { + using dir = tempDir("inspect-unknown-size", {}); + const firstLines = file => [ + Bun.inspect(new Response(file)).split("\n")[0], + Bun.inspect(new Request("http://example.com/", { method: "POST", body: file })).split("\n")[0], + ]; + expect({ + missing: firstLines(Bun.file(join(String(dir), "missing.txt"))), + directory: firstLines(Bun.file(String(dir))), + }).toEqual({ + missing: ["Response {", "Request {"], + directory: ["Response {", "Request {"], + }); +}); + +it("printing a Response over a Bun.file() that does not exist yet leaves its body as it is", async () => { + using dir = tempDir("inspect-missing-then-created", {}); + const path = join(String(dir), "later.txt"); + const response = new Response(Bun.file(path)); + const before = Bun.inspect(response).split("\n")[0]; + await Bun.write(path, "created"); + expect({ before, after: Bun.inspect(response).split("\n")[0], text: await response.text() }).toEqual({ + before: "Response {", + after: "Response (7 bytes) {", + text: "created", + }); +}); + it("utf16 property name", () => { var { Database } = require("bun:sqlite"); const db = Database.open(":memory:"); diff --git a/test/js/web/fetch/body-clone.test.ts b/test/js/web/fetch/body-clone.test.ts index 2b1a40941e22..19e732ce56a4 100644 --- a/test/js/web/fetch/body-clone.test.ts +++ b/test/js/web/fetch/body-clone.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isASAN, isDebug, isWindows, tempDirWithFiles } from "harness"; +import { bunEnv, bunExe, isASAN, isDebug, isLinux, isWindows, tempDirWithFiles } from "harness"; +import { closeSync, openSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; import net from "node:net"; import { join } from "node:path"; @@ -1171,6 +1172,18 @@ describe("clone() of a body over an unread native stream keeps the Blob behind i expect(await cloneInChild("Bun.stdin")).toEqual(bothBodiesReadStdin); }); + // Reads of one fd share its offset, so the second Blob would start where the first one ended. + test("a body over a regular file given by file descriptor is teed, not duped", async () => { + const fd = openSync(join(tempDirWithFiles("body-clone-fd", { "a.txt": "hello world" }), "a.txt"), "r"); + try { + const original = new Response(Bun.file(fd)); + const clone = original.clone(); + expect(await Promise.all([original.text(), clone.text()])).toEqual(["hello world", "hello world"]); + } finally { + closeSync(fd); + } + }); + // The same store kind reached by path: stat says it is not a regular file. test.skipIf(isWindows)("a body over a FIFO opened by path is still teed", async () => { const fifo = join(tempDirWithFiles("body-clone-fifo", {}), "body.fifo"); @@ -1183,6 +1196,159 @@ describe("clone() of a body over an unread native stream keeps the Blob behind i expect(await cloneInChild("Bun.file(process.argv.at(-1)).stream()", [fifo])).toEqual(bothBodiesReadStdin); expect(await writer.exited).toBe(0); }); + + test.skipIf(isWindows)("a body over a FIFO given by path as the Blob itself is teed, not duped", async () => { + const fifo = join(tempDirWithFiles("body-clone-fifo-blob", {}), "body.fifo"); + expect(Bun.spawnSync({ cmd: ["mkfifo", fifo] }).exitCode).toBe(0); + await using writer = Bun.spawn({ + cmd: ["sh", "-c", `printf 'hello world' > "$1"`, "sh", fifo], + stdout: "ignore", + stderr: "inherit", + }); + expect(await cloneInChild("Bun.file(process.argv.at(-1))", [fifo])).toEqual(bothBodiesReadStdin); + expect(await writer.exited).toBe(0); + }); + + // The body shares its store with the Bun.file() it was made from. A stat cached + // there is what `size`, `lastModified`, `exists()` and a later `.body` answer from. + describe("clone() of a Bun.file() body does not change what the Bun.file() answers", () => { + const post = (body: Bun.BunFile) => new Request("http://example.com/", { method: "POST", body }); + const cloners: Array<[string, (file: Bun.BunFile) => unknown]> = [ + ["Response.clone()", file => new Response(file).clone()], + ["Request.clone()", file => post(file).clone()], + ["new Request(request)", file => new Request(post(file))], + ["new Request(request, init)", file => new Request(post(file), { headers: { "x-test": "1" } })], + // @ts-expect-error Bun takes a Response as init and clones its body. + ["new Request(url, response)", file => new Request("http://example.com/", new Response(file))], + ]; + + async function afterClone( + clone: (file: Bun.BunFile) => unknown, + change: (path: string) => void, + read: (file: Bun.BunFile) => T | Promise, + ) { + const path = join(tempDirWithFiles("body-clone-answers", { "log.txt": "12345" }), "log.txt"); + const file = Bun.file(path); + clone(file); + change(path); + return await read(file); + } + const grow = (path: string) => writeFileSync(path, "123456789"); + + test.each(cloners)("%s: size and a later body see a file that grew", async (_, clone) => { + expect({ + size: await afterClone(clone, grow, file => file.size), + body: await afterClone(clone, grow, file => Bun.readableStreamToText(new Response(file).body!)), + }).toEqual({ size: 9, body: "123456789" }); + }); + + test("lastModified and exists() see a file that changed", async () => { + const clone = cloners[0][1]; + const mtime = new Date("2033-05-18T03:33:20.000Z"); + expect({ + lastModified: await afterClone( + clone, + path => utimesSync(path, mtime, mtime), + file => file.lastModified, + ), + exists: await afterClone(clone, unlinkSync, file => file.exists()), + }).toEqual({ lastModified: mtime.getTime(), exists: false }); + }); + + test("both bodies see a file that grew", async () => { + const path = join(tempDirWithFiles("body-clone-grew", { "log.txt": "12345" }), "log.txt"); + const original = new Response(Bun.file(path)); + const clone = original.clone(); + grow(path); + expect(await Promise.all([original.body, clone.body].map(body => Bun.readableStreamToText(body!)))).toEqual([ + "123456789", + "123456789", + ]); + }); + + test("exists() sees a file that was created before the clone", async () => { + const path = join(tempDirWithFiles("body-clone-created", {}), "later.txt"); + const file = Bun.file(path); + expect(await file.exists()).toBe(false); + writeFileSync(path, "12345"); + new Response(file).clone(); + expect(await file.exists()).toBe(true); + }); + + test("the size console.log prints for the clone", () => { + const sizeOf = (change: (path: string) => void, body: (file: Bun.BunFile) => Blob) => { + const path = join(tempDirWithFiles("body-clone-label", { "log.txt": "12345" }), "log.txt"); + const clone = new Response(body(Bun.file(path))).clone(); + change(path); + return [Bun.inspect(clone), Bun.inspect(clone)].map(printed => printed.split("\n")[0]).join(" "); + }; + expect({ + whole: sizeOf( + () => {}, + file => file, + ), + slice: sizeOf( + () => {}, + file => file.slice(1, 4), + ), + grew: sizeOf(grow, file => file), + deleted: sizeOf(unlinkSync, file => file), + }).toEqual({ + whole: "Response (5 bytes) { Response (5 bytes) {", + slice: "Response (3 bytes) { Response (3 bytes) {", + grew: "Response (9 bytes) { Response (9 bytes) {", + deleted: "Response { Response {", + }); + }); + + // procfs reports `st_size == 0` for a file that has content, so with that stat cached all three read "". + test.skipIf(!isLinux)("a procfs file is read whole", async () => { + const file = Bun.file("/proc/sys/kernel/ostype"); + const original = new Response(file); + const clone = original.clone(); + expect({ + original: await original.text(), + clone: await clone.text(), + file: await file.text(), + }).toEqual({ original: "Linux\n", clone: "Linux\n", file: "Linux\n" }); + }); + }); + + // No bytes to compete for: it is duped like a regular file, and each body fails when it is read. + describe("clone() of a body that cannot be read does not throw", () => { + const sizeRead = (file: Bun.BunFile) => (file.size, file); + const directory = () => tempDirWithFiles("body-clone-dir", {}); + + async function codes(file: Bun.BunFile) { + const original = new Response(file); + const clone = original.clone(); + const results = await Promise.allSettled([original.text(), clone.text()]); + return results.map(result => (result.status === "rejected" ? result.reason?.code : result.status)); + } + + test("a directory by path", async () => { + expect({ + fresh: await codes(Bun.file(directory())), + sizeRead: await codes(sizeRead(Bun.file(directory()))), + }).toEqual({ fresh: ["EISDIR", "EISDIR"], sizeRead: ["EISDIR", "EISDIR"] }); + }); + + test.skipIf(isWindows)("a directory by file descriptor", async () => { + const fd = openSync(directory(), "r"); + try { + expect({ + fresh: await codes(Bun.file(fd)), + sizeRead: await codes(sizeRead(Bun.file(fd))), + }).toEqual({ fresh: ["EISDIR", "EISDIR"], sizeRead: ["EISDIR", "EISDIR"] }); + } finally { + closeSync(fd); + } + }); + + test.skipIf(isWindows)("a file descriptor that is not open", async () => { + expect(await codes(Bun.file(1_000_000))).toEqual(["EBADF", "EBADF"]); + }); + }); }); test("Blob type from a consumed Response keeps the original content-type after clones with different content-types are consumed", async () => { From 0ba17d5f7cc607262aa14e9d45ab425233fcee02 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sat, 3 Oct 2026 01:07:44 -0700 Subject: [PATCH 03/15] dns(macOS): use DNSServiceQueryRecord so split DNS can fail over (#44519) Fixes #44075. Closes #44449. On macOS, dns.lookup(), fetch() and sockets now resolve split-DNS names, for example a VPN resolver behind iCloud Private Relay. getaddrinfo(3) resolves these names, but Bun reported ENOTFOUND. mDNSResponder asks the next resolver after a negative answer only when the client sends the failover attribute. DNSServiceGetAddrInfoEx (#40675) accepts that attribute but never sends it, and the daemon reads it only for QueryRecord requests. This PR does what libinfo does: one DNSServiceQueryRecordWithAttribute per family. macOS 12 falls back to plain DNSServiceQueryRecord on the same code path. Thanks to @mimi1vx for the diagnosis. Compared with #44449: the hostname is sent as written (a trailing dot turns off search domains, so http://intranet/ would stop resolving), a link-local AAAA keeps its interface scope (fe80::1%7), and removed records are still ignored. Verified: new tests put a scripted mDNSResponder behind the unix socket of the daemon and check the requests and the answers. All 10 pass with bun bd test test/js/bun/dns/. The 9 that existed at that time all fail on 1.4.3 with USE_SYSTEM_BUN=1, and the tenth was not run there. The requests are now identical to those of getaddrinfo(3), and 21 lookups through the real daemon match 1.4.3. Not verified: real failover, which needs a split-DNS machine. @mimi1vx @sassdavid, can you try this build with bunx bun-pr? --- src/runtime/dns_jsc/dns.rs | 63 +--- src/runtime/dns_jsc/dns_sd.rs | 361 ++++++++++++++--------- test/js/bun/dns/mdnsresponder-fixture.ts | 222 ++++++++++++++ test/js/bun/dns/resolve-dns.test.ts | 242 ++++++++++++++- 4 files changed, 686 insertions(+), 202 deletions(-) create mode 100644 test/js/bun/dns/mdnsresponder-fixture.ts diff --git a/src/runtime/dns_jsc/dns.rs b/src/runtime/dns_jsc/dns.rs index 5634a548d18d..b06683e2cf98 100644 --- a/src/runtime/dns_jsc/dns.rs +++ b/src/runtime/dns_jsc/dns.rs @@ -118,7 +118,7 @@ bun_output::declare_scope!(DNSResolver, visible); const IANA_DNS_PORT: i32 = 53; // ────────────────────────────────────────────────────────────────────────── -// dns_sd (macOS): DNSServiceGetAddrInfo over one shared mDNSResponder connection, no per-lookup threads. +// dns_sd (macOS): DNSServiceQueryRecord over one shared mDNSResponder connection, no per-lookup threads. // ────────────────────────────────────────────────────────────────────────── #[cfg(target_os = "macos")] @@ -1209,33 +1209,6 @@ impl GetAddrInfoRequest { request } - /// Reply callback (inside `DNSServiceProcessResult`): records state; completion happens in `on_readable`. - /// # Safety - /// `context` is the registered `*mut GetAddrInfoRequest`; `address`, if non-null, is a valid sockaddr. - #[cfg(target_os = "macos")] - pub(crate) unsafe extern "C" fn dns_sd_reply( - _sd_ref: dns_sd::DNSServiceRef, - flags: u32, - _interface_index: u32, - error_code: i32, - _hostname: *const c_char, - address: *const Sockaddr, - ttl: u32, - context: *mut c_void, - ) { - dns_sd::SharedConnection::note_reply(context); - // SAFETY: context is the *mut GetAddrInfoRequest passed to start(). - let this: *mut Self = context.cast(); - // SAFETY: `this` is the live heap request (JS thread); `address` is valid per dns_sd.h. - unsafe { - (*this) - .backend - .as_dns_sd_mut() - .query - .record_reply(flags, error_code, address, ttl); - } - } - /// Complete a dns_sd-backed request; `this` is the live heap request, consumed on every path. #[cfg(target_os = "macos")] pub(crate) fn complete_dns_sd(this: *mut Self) { @@ -2779,39 +2752,7 @@ pub(crate) mod internal { query: dns_sd::QueryState::new(protocol), }; } - let Some(_) = shared.start( - dns_sd::Inflight::Internal(req), - protocol, - host, - dns_sd_reply, - req.cast::(), - ) else { - return false; - }; - - true - } - - #[cfg(target_os = "macos")] - unsafe extern "C" fn dns_sd_reply( - _sd_ref: dns_sd::DNSServiceRef, - flags: u32, - _interface_index: u32, - error_code: i32, - _hostname: *const c_char, - address: *const Sockaddr, - ttl: u32, - context: *mut c_void, - ) { - dns_sd::SharedConnection::note_reply(context); - let req: *mut Request = context.cast(); - // SAFETY: `context` is the registered `req` (event-loop thread); `address` is valid per dns_sd.h. - unsafe { - (*req) - .dns_sd - .query - .record_reply(flags, error_code, address, ttl) - }; + shared.start(dns_sd::Inflight::Internal(req), protocol, host) } /// Complete an internal request: build an addrinfo chain and reuse `process_results` (happy-eyeballs order). diff --git a/src/runtime/dns_jsc/dns_sd.rs b/src/runtime/dns_jsc/dns_sd.rs index a2a7d844f6a4..95d0b2a01bc8 100644 --- a/src/runtime/dns_jsc/dns_sd.rs +++ b/src/runtime/dns_jsc/dns_sd.rs @@ -1,9 +1,9 @@ -//! macOS DNSServiceGetAddrInfo backend: all lookups share one mDNSResponder connection (see dns.rs banner). +//! macOS DNSServiceQueryRecord backend: all lookups share one mDNSResponder connection (see dns.rs banner). use super::*; use bun_collections::index_sort; -pub(crate) type DNSServiceRef = *mut c_void; +type DNSServiceRef = *mut c_void; type DNSServiceFlags = u32; type DNSServiceErrorType = i32; pub(crate) type DNSServiceProtocol = u32; @@ -18,18 +18,23 @@ const FLAGS_TIMEOUT: DNSServiceFlags = 0x10000; pub(crate) const PROTOCOL_IPV4: DNSServiceProtocol = 0x01; pub(crate) const PROTOCOL_IPV6: DNSServiceProtocol = 0x02; +const TYPE_A: u16 = 1; +const TYPE_AAAA: u16 = 28; +const CLASS_IN: u16 = 1; + pub(crate) const ERR_NO_ERROR: DNSServiceErrorType = 0; -pub(crate) const ERR_NO_SUCH_RECORD: DNSServiceErrorType = -65554; pub(crate) const ERR_TIMEOUT: DNSServiceErrorType = -65568; -const ERR_DEFUNCT_CONNECTION: DNSServiceErrorType = -65569; -type GetAddrInfoReply = unsafe extern "C" fn( +type QueryRecordReply = unsafe extern "C" fn( sd_ref: DNSServiceRef, flags: DNSServiceFlags, interface_index: u32, error_code: DNSServiceErrorType, - hostname: *const c_char, - address: *const Sockaddr, + fullname: *const c_char, + rrtype: u16, + rrclass: u16, + rdlen: u16, + rdata: *const c_void, ttl: u32, context: *mut c_void, ); @@ -40,35 +45,37 @@ unsafe extern "C" { fn DNSServiceRefSockFD(sd_ref: DNSServiceRef) -> c_int; fn DNSServiceProcessResult(sd_ref: DNSServiceRef) -> DNSServiceErrorType; fn DNSServiceRefDeallocate(sd_ref: DNSServiceRef); - fn DNSServiceGetAddrInfo( + fn DNSServiceQueryRecord( sd_ref: *mut DNSServiceRef, flags: DNSServiceFlags, interface_index: u32, - protocol: DNSServiceProtocol, - hostname: *const c_char, - callback: GetAddrInfoReply, + fullname: *const c_char, + rrtype: u16, + rrclass: u16, + callback: QueryRecordReply, context: *mut c_void, ) -> DNSServiceErrorType; } -/// SPI: `DNSServiceGetAddrInfo` plus the attribute libinfo's getaddrinfo passes. Absent on macOS 12, so resolved at runtime. -type GetAddrInfoExFn = unsafe extern "C" fn( +/// `DNSServiceQueryRecord` plus an attribute. Absent on macOS 12, so resolved at runtime. +type QueryRecordWithAttributeFn = unsafe extern "C" fn( sd_ref: *mut DNSServiceRef, flags: DNSServiceFlags, interface_index: u32, - protocol: DNSServiceProtocol, - hostname: *const c_char, + fullname: *const c_char, + rrtype: u16, + rrclass: u16, attr: *const DNSServiceAttribute, - callback: GetAddrInfoReply, + callback: QueryRecordReply, context: *mut c_void, ) -> DNSServiceErrorType; -/// `DNSServiceGetAddrInfoEx` with `kDNSServiceAttrAllowFailover` (lets mDNSResponder fail a query over to -/// scoped/supplemental resolvers, as getaddrinfo does), when this OS has both. -fn getaddrinfo_ex() -> Option<(GetAddrInfoExFn, *const DNSServiceAttribute)> { +/// libinfo's getaddrinfo call: only QueryRecord sends `kDNSServiceAttrAllowFailover` to the daemon (`DNSServiceGetAddrInfoEx` drops it). +fn query_record_allowing_failover() +-> Option<(QueryRecordWithAttributeFn, *const DNSServiceAttribute)> { let f = bun_sys::dlsym_with_handle!( - GetAddrInfoExFn, - "DNSServiceGetAddrInfoEx", + QueryRecordWithAttributeFn, + "DNSServiceQueryRecordWithAttribute", Some(libc::RTLD_DEFAULT) )?; let attr = bun_sys::dlsym_with_handle!( @@ -150,41 +157,75 @@ enum Attempt { Reissued, } +/// The A and the AAAA subordinate of one lookup; null for a family that was not asked for. +type FamilyRefs = [DNSServiceRef; 2]; + +fn deallocate_refs(sd_refs: FamilyRefs) { + for sd_ref in sd_refs { + if !sd_ref.is_null() { + // SAFETY: FFI; a non-null entry is a live subordinate that nothing else releases. + unsafe { DNSServiceRefDeallocate(sd_ref) }; + } + } +} + +/// What dnssd_clientstub's `handle_addrinfo_response` builds: a link-local address is scoped to the interface it was seen on. +fn address_from_record( + rrtype: u16, + rdata: &[u8], + interface_index: u32, +) -> Option { + match rrtype { + TYPE_A => { + let ip = Ipv4Addr::from(<[u8; 4]>::try_from(rdata).ok()?); + Some(bun_dns::Address::from_ip(ip.into(), 0)) + } + TYPE_AAAA => { + let ip = Ipv6Addr::from(<[u8; 16]>::try_from(rdata).ok()?); + let mut address = bun_dns::Address::from_ip(ip.into(), 0); + if ip.is_unicast_link_local() { + address.set_scope_id(interface_index); + } + Some(address) + } + _ => None, + } +} + /// Per-query state shared by the JS `dns.lookup` path and the internal connect path. pub(crate) struct QueryState { - pub(crate) sd_ref: DNSServiceRef, + sd_refs: FamilyRefs, pub(crate) results: bun_dns::ResultList, - /// First hard error (NoSuchRecord/Timeout are per-family negatives, not errors). - pub(crate) sd_error: DNSServiceErrorType, /// A family timed out: an unsuppressed reissue would only wait out the timeout again. saw_timeout: bool, /// Last reply had `MoreComing` and no other request's reply followed: more is queued daemon-side. awaiting_more: bool, - /// Protocol bits with no reply yet; any family-tagged callback clears its bit. + /// Protocol bits with no reply yet; an address record or any error for a family's question clears its bit. pub(crate) pending_proto: DNSServiceProtocol, stragglers: Stragglers, attempt: Attempt, /// Kept so `finish()` can reissue the query for the retry. hostname: bun::ZBox, - callback: Option, } impl QueryState { pub(crate) fn new(protocol: DNSServiceProtocol) -> Self { Self { - sd_ref: ptr::null_mut(), + sd_refs: [ptr::null_mut(); 2], results: Default::default(), - sd_error: 0, saw_timeout: false, awaiting_more: false, pending_proto: protocol, stragglers: Stragglers::None, attempt: Attempt::Plain, hostname: bun::ZBox::from_bytes(b""), - callback: None, } } + fn deallocate_refs(&mut self) { + deallocate_refs(core::mem::replace(&mut self.sd_refs, [ptr::null_mut(); 2])); + } + /// Back to a fresh in-flight state for the unsuppressed reissue. pub(crate) fn reset_for_retry(&mut self, protocol: DNSServiceProtocol) { self.attempt = Attempt::Reissued; @@ -200,49 +241,40 @@ impl QueryState { /// A suppressed query that returned nothing at all gets one unsuppressed retry. fn should_retry_unsuppressed(&self) -> bool { - self.attempt == Attempt::Suppressed - && self.results.is_empty() - && self.sd_error == 0 - && !self.saw_timeout + self.attempt == Attempt::Suppressed && self.results.is_empty() && !self.saw_timeout } - /// Absorb one callback. SAFETY: `address`, if non-null, is a valid sockaddr (dnssd_clientstub guarantees it). - pub(crate) unsafe fn record_reply( + /// Absorb one callback for the question `sd_ref`. + fn record_reply( &mut self, + sd_ref: DNSServiceRef, flags: DNSServiceFlags, + interface_index: u32, error_code: DNSServiceErrorType, - address: *const Sockaddr, + rrtype: u16, + rdata: &[u8], ttl: u32, ) { self.awaiting_more = flags & FLAGS_MORE_COMING != 0; - // Only PolicyDenied passes a null sockaddr; A/AAAA replies (incl. negatives) are family-tagged. - if address.is_null() { - if self.sd_error == 0 { - self.sd_error = error_code; - } - return; + // As in libinfo, any error ends only its own family. The daemon's refusals carry no rrtype, so `sd_ref` names the family. + if error_code != ERR_NO_ERROR || matches!(rrtype, TYPE_A | TYPE_AAAA) { + self.pending_proto &= !if sd_ref == self.sd_refs[0] { + PROTOCOL_IPV4 + } else { + PROTOCOL_IPV6 + }; } - // SAFETY: caller contract. - let fam = unsafe { (*address).sa_family } as i32; - // Any reply retires the family's bit; completeness is tracked by `awaiting_more`. - self.pending_proto &= !if fam == netc::AF_INET6 { - PROTOCOL_IPV6 - } else { - PROTOCOL_IPV4 - }; - if error_code == ERR_NO_ERROR && flags & FLAGS_ADD != 0 { - self.results.push(GetAddrInfoResult { - // SAFETY: caller contract. - address: unsafe { bun_dns::Address::init_posix(address.cast()) }, - ttl: ttl as i32, - }); - } else if error_code == ERR_TIMEOUT { - self.saw_timeout = true; - } else if error_code != ERR_NO_ERROR - && error_code != ERR_NO_SUCH_RECORD - && self.sd_error == 0 - { - self.sd_error = error_code; + match error_code { + ERR_NO_ERROR if flags & FLAGS_ADD != 0 => { + if let Some(address) = address_from_record(rrtype, rdata, interface_index) { + self.results.push(GetAddrInfoResult { + address, + ttl: ttl as i32, + }); + } + } + ERR_TIMEOUT => self.saw_timeout = true, + _ => {} } self.stragglers = match (self.only_stragglers_left(), self.stragglers) { (false, _) => Stragglers::None, @@ -258,9 +290,7 @@ impl QueryState { } pub(crate) fn is_ready(&self) -> bool { - self.sd_error != 0 - || self.stragglers == Stragglers::GaveUp - || (self.pending_proto == 0 && !self.awaiting_more) + self.stragglers == Stragglers::GaveUp || (self.pending_proto == 0 && !self.awaiting_more) } /// Deadline for giving up on stragglers (a silent second family, or a dangling `MoreComing`). @@ -295,6 +325,21 @@ impl Inflight { } } + /// The `on_reply` that turns `context()` back into this variant. + fn reply_callback(&self) -> QueryRecordReply { + match *self { + Inflight::Jsc(_) => on_reply::, + Inflight::Internal(_) => on_reply::, + } + } + + fn complete(self) { + match self { + Inflight::Jsc(r) => GetAddrInfoRequest::complete_dns_sd(r), + Inflight::Internal(r) => internal::dns_sd_complete(r), + } + } + /// SAFETY: the request behind `self` is live (pinned in `inflight`); /// the `&mut` derives from the stored raw pointer, not from a borrow. unsafe fn query<'a>(self) -> &'a mut QueryState { @@ -308,6 +353,41 @@ impl Inflight { } } +/// Records a reply; `on_readable` completes. SAFETY: `context` is a live request of the `INTERNAL` kind; `rdata` spans `rdlen` bytes. +unsafe extern "C" fn on_reply( + sd_ref: DNSServiceRef, + flags: DNSServiceFlags, + interface_index: u32, + error_code: DNSServiceErrorType, + _fullname: *const c_char, + rrtype: u16, + _rrclass: u16, + rdlen: u16, + rdata: *const c_void, + ttl: u32, + context: *mut c_void, +) { + SharedConnection::note_reply(context); + let owner = if INTERNAL { + Inflight::Internal(context.cast()) + } else { + Inflight::Jsc(context.cast()) + }; + // SAFETY: caller contract; the slice is only read before this callback returns. + let rdata = unsafe { bun::ffi::slice(rdata.cast::(), rdlen as usize) }; + // SAFETY: caller contract; event-loop thread, and no other borrow of the query is live here. + let query = unsafe { owner.query() }; + query.record_reply( + sd_ref, + flags, + interface_index, + error_code, + rrtype, + rdata, + ttl, + ); +} + /// One per event loop: owns the primary `DNSServiceRef` + `FilePoll`; lookups are ShareConnection subordinates. pub(crate) struct SharedConnection { main_ref: DNSServiceRef, @@ -400,76 +480,88 @@ impl SharedConnection { Self::current() } - /// Start a subordinate query and track it (keeps the process alive); `None` if the daemon refused. + /// Start `owner`'s queries and track it (keeps the process alive); `false` if the daemon refused. pub(crate) fn start( &mut self, owner: Inflight, protocol: DNSServiceProtocol, hostname: &ZStr, - callback: GetAddrInfoReply, - context: *mut c_void, - ) -> Option { + ) -> bool { let suppress = addrconfig_flags(protocol); - let sub = self.issue(protocol, suppress, hostname, callback, context)?; + let Some(sd_refs) = self.issue(owner, protocol, suppress, hostname) else { + return false; + }; if self.inflight.is_empty() { let ctx = self.ctx; self.file_poll().enable_keeping_process_alive(ctx); } // SAFETY: `owner` is the caller's live request, tracked here until `finish()`. let q = unsafe { owner.query() }; - q.sd_ref = sub; + q.sd_refs = sd_refs; q.attempt = if suppress != 0 { Attempt::Suppressed } else { Attempt::Plain }; q.hostname = bun::ZBox::from_bytes(hostname.as_bytes()); - q.callback = Some(callback); self.inflight.push(owner); - Some(sub) + true } + /// One query per family in `protocol`, as libinfo does; `None`, with none left running, if the daemon refused one. fn issue( &mut self, + owner: Inflight, protocol: DNSServiceProtocol, suppress: DNSServiceFlags, hostname: &ZStr, - callback: GetAddrInfoReply, - context: *mut c_void, - ) -> Option { - // ShareConnection requires `sub` to start as a copy of the primary ref. - let mut sub: DNSServiceRef = self.main_ref; + ) -> Option { let flags = FLAGS_SHARE_CONNECTION | FLAGS_TIMEOUT | FLAGS_RETURN_INTERMEDIATES | suppress; - let hostname = hostname.as_ptr().cast::(); - // SAFETY: FFI; `hostname` is NUL-terminated (copied by dns_sd); `context` is only stored. - let err = unsafe { - match getaddrinfo_ex() { - Some((ex, attr)) => ex( - &raw mut sub, - flags, - 0, - protocol, - hostname, - attr, - callback, - context, - ), - None => DNSServiceGetAddrInfo( - &raw mut sub, - flags, - 0, - protocol, - hostname, - callback, - context, - ), + // Sent as given: the daemon tries search domains only for a name with no trailing dot. + let name = hostname.as_ptr().cast::(); + let (callback, context) = (owner.reply_callback(), owner.context()); + let mut sd_refs: FamilyRefs = [ptr::null_mut(); 2]; + let families = [(PROTOCOL_IPV4, TYPE_A), (PROTOCOL_IPV6, TYPE_AAAA)]; + for (i, (family, rrtype)) in families.into_iter().enumerate() { + if protocol & family == 0 { + continue; } - }; - if err != ERR_NO_ERROR { - bun_output::scoped_log!(dns, "DNSServiceGetAddrInfo failed: {}", err); - return None; + // ShareConnection requires `sub` to start as a copy of the primary ref. + let mut sub: DNSServiceRef = self.main_ref; + // SAFETY: FFI; `name` is NUL-terminated (copied by dns_sd); `context` is only stored. + let err = unsafe { + match query_record_allowing_failover() { + Some((query_record, allow_failover)) => query_record( + &raw mut sub, + flags, + 0, + name, + rrtype, + CLASS_IN, + allow_failover, + callback, + context, + ), + None => DNSServiceQueryRecord( + &raw mut sub, + flags, + 0, + name, + rrtype, + CLASS_IN, + callback, + context, + ), + } + }; + if err != ERR_NO_ERROR { + bun_output::scoped_log!(dns, "DNSServiceQueryRecord failed: {}", err); + deallocate_refs(sd_refs); + return None; + } + sd_refs[i] = sub; } - Some(sub) + Some(sd_refs) } /// Called first from every reply callback: a different `context` ends the previous request's `MoreComing` run. @@ -507,7 +599,7 @@ impl SharedConnection { let ready = core::mem::take(&mut this.inflight); let detached = SHARED.replace(ptr::null_mut()); for inf in ready { - Self::finish(inf, Some(rc)); + Self::fail(inf); } // SAFETY: `detached` was just removed from SHARED and drained. unsafe { Self::destroy(detached) }; @@ -516,7 +608,7 @@ impl SharedConnection { let ready = this.take_ready(|q| q.is_ready()); this.arm_early_out(); for inf in ready { - Self::finish(inf, None); + Self::finish(inf); } } @@ -600,7 +692,7 @@ impl SharedConnection { ready }; for inf in ready { - Self::finish(inf, None); + Self::finish(inf); } } @@ -626,22 +718,23 @@ impl SharedConnection { drop(conn); } - /// `force_err` drops partial results so teardown rejects instead of resolving. - fn finish(inf: Inflight, force_err: Option) { + fn finish(inf: Inflight) { // SAFETY: `inf` is a live heap request just removed from `inflight`. let q = unsafe { inf.query() }; - // SAFETY: FFI; `sd_ref` is this request's live subordinate. - unsafe { DNSServiceRefDeallocate(q.sd_ref) }; - if let Some(e) = force_err { - q.results.clear(); - q.sd_error = e; - } else if q.should_retry_unsuppressed() && Self::retry_unsuppressed(inf) { + q.deallocate_refs(); + if q.should_retry_unsuppressed() && Self::retry_unsuppressed(inf) { return; } - match inf { - Inflight::Jsc(r) => GetAddrInfoRequest::complete_dns_sd(r), - Inflight::Internal(r) => internal::dns_sd_complete(r), - } + inf.complete(); + } + + /// The connection is going away: drops partial results so the request rejects instead of resolving. + fn fail(inf: Inflight) { + // SAFETY: `inf` is a live heap request just removed from `inflight`. + let q = unsafe { inf.query() }; + q.deallocate_refs(); + q.results.clear(); + inf.complete(); } /// Reissue `inf`'s query without SuppressUnusable; `false` if it couldn't be reissued. @@ -652,11 +745,8 @@ impl SharedConnection { // SAFETY: `inf` is a live heap request removed from `inflight` by the caller. let q = unsafe { inf.query() }; let (protocol, hostname) = (protocol_for_pending(q), q.hostname.clone()); - let Some(callback) = q.callback else { - return false; - }; q.reset_for_retry(protocol); - let Some(sub) = this.issue(protocol, 0, &hostname, callback, inf.context()) else { + let Some(sd_refs) = this.issue(inf, protocol, 0, &hostname) else { return false; }; bun_output::scoped_log!( @@ -664,7 +754,7 @@ impl SharedConnection { "retrying {} without SuppressUnusable", bstr::BStr::new(hostname.as_bytes()) ); - q.sd_ref = sub; + q.sd_refs = sd_refs; if this.inflight.is_empty() { let ctx = this.ctx; this.file_poll().enable_keeping_process_alive(ctx); @@ -687,13 +777,12 @@ impl SharedConnection { // have waiters on other threads (and its outcome is cached): this // thread going away is not an answer. Finish it on the work pool. Inflight::Internal(req) => { - // SAFETY: `inf` is a live heap request just removed from `inflight`; - // FFI releases this thread's subordinate for it. - unsafe { DNSServiceRefDeallocate(inf.query().sd_ref) }; + // SAFETY: `inf` is a live heap request just removed from `inflight`. + unsafe { inf.query() }.deallocate_refs(); internal::run_on_work_pool(req); } // A dns.lookup() from this thread's script: only this VM waits on it. - Inflight::Jsc(_) => Self::finish(inf, Some(ERR_DEFUNCT_CONNECTION)), + Inflight::Jsc(_) => Self::fail(inf), } } // SAFETY: `this` is detached and drained. @@ -755,13 +844,7 @@ pub(crate) fn lookup( let promise_value = unsafe { (*request).head.promise.value() }; let name_z = bun::ZBox::from_bytes(query.name.as_ref()); - let Some(_) = shared.start( - Inflight::Jsc(request), - protocol, - &name_z, - GetAddrInfoRequest::dns_sd_reply, - request.cast::(), - ) else { + if !shared.start(Inflight::Jsc(request), protocol, &name_z) { // SAFETY: request is exclusively owned; dns_sd never accepted it. unsafe { if let Some(pos) = (*request).pending_slot { @@ -775,7 +858,7 @@ pub(crate) fn lookup( drop(bun_core::heap::take(request)); } return lib_c::lookup(this, query, global_this, context); - }; + } this.request_sent(this.vm()); diff --git a/test/js/bun/dns/mdnsresponder-fixture.ts b/test/js/bun/dns/mdnsresponder-fixture.ts new file mode 100644 index 000000000000..fca1c89771f6 --- /dev/null +++ b/test/js/bun/dns/mdnsresponder-fixture.ts @@ -0,0 +1,222 @@ +// A scripted stand-in for mDNSResponder. libsystem_dnssd connects to $DNSSD_UDS_PATH when it is set. +// +// bun mdnsresponder-fixture.ts +// +// Runs the client script in a child Bun, answers its queries from `answers`, and after the client has +// gone prints every request it made as one line of JSON. The client's own stdout comes first. +import { dlopen, ptr } from "bun:ffi"; +import { closeSync, writeSync } from "node:fs"; + +export type Answer = { + rrtype: number; + /** An IPv4 or IPv6 address as bytes, or any other rdata. */ + rdata?: number[]; + /** Defaults to kDNSServiceFlagsAdd. */ + flags?: number; + error?: number; + ifindex?: number; + ttl?: number; + /** Held back until a query for this name arrives. */ + after?: string; +}; +/** Keyed by `${name} ${rrtype}`. A query with no entry gets kDNSServiceErr_NoSuchRecord. */ +export type Answers = Record; + +const [socketPath, answersJSON, clientScript] = process.argv.slice(2); +const answers: Answers = JSON.parse(answersJSON); + +const { symbols: libc } = dlopen("libSystem.B.dylib", { + socket: { args: ["i32", "i32", "i32"], returns: "i32" }, + bind: { args: ["i32", "ptr", "u32"], returns: "i32" }, + listen: { args: ["i32", "i32"], returns: "i32" }, + accept: { args: ["i32", "ptr", "ptr"], returns: "i32" }, + poll: { args: ["ptr", "u32", "i32"], returns: "i32" }, + recvmsg: { args: ["i32", "ptr", "i32"], returns: "i64" }, +}); + +const AF_UNIX = 1; +const SOCK_STREAM = 1; +const SOL_SOCKET = 0xffff; +const SCM_RIGHTS = 1; +const POLLIN = 1; + +const HEADER_SIZE = 28; +const IPC_FLAGS_TRAILING_TLVS = 2; +const ops = { 1: "connection", 8: "query", 15: "addrinfo", 63: "cancel" } as const; +const QUERY_REPLY = 68; +const ADDRINFO_REPLY = 72; + +const FLAGS_ADD = 0x2; +const ERR_NO_SUCH_RECORD = -65554; +const ERR_UNSUPPORTED = -65544; +const TYPE_A = 1; +const TYPE_AAAA = 28; + +function check(rc: number, what: string) { + if (rc < 0) throw new Error(`${what} failed`); + return rc; +} + +function listenOn(path: string) { + const pathBytes = Buffer.from(path); + if (pathBytes.length >= 104) throw new Error(`socket path is too long for sun_path: ${path}`); + const addr = new Uint8Array(106); + addr[0] = addr.length; + addr[1] = AF_UNIX; + addr.set(pathBytes, 2); + const fd = check(libc.socket(AF_UNIX, SOCK_STREAM, 0), "socket"); + check(libc.bind(fd, ptr(addr), addr.length), "bind"); + check(libc.listen(fd, 8), "listen"); + return fd; +} + +/** One recvmsg(): the bytes read (empty at EOF) and any descriptors that came with them. */ +function receive(fd: number) { + const data = new Uint8Array(4096); + const control = new Uint8Array(64); + const iov = new BigUint64Array([BigInt(ptr(data)), BigInt(data.length)]); + const msg = new DataView(new ArrayBuffer(48)); + msg.setBigUint64(16, BigInt(ptr(iov)), true); + msg.setInt32(24, 1, true); + msg.setBigUint64(32, BigInt(ptr(control)), true); + msg.setUint32(40, control.length, true); + const n = Number(libc.recvmsg(fd, ptr(new Uint8Array(msg.buffer)), 0)); + const fds: number[] = []; + const cmsg = new DataView(control.buffer); + if (msg.getUint32(40, true) >= 16 && cmsg.getInt32(4, true) === SOL_SOCKET && cmsg.getInt32(8, true) === SCM_RIGHTS) { + for (let at = 12; at + 4 <= cmsg.getUint32(0, true); at += 4) fds.push(cmsg.getInt32(at, true)); + } + return { bytes: Buffer.from(data.subarray(0, Math.max(n, 0))), fds }; +} + +function reply(op: number, context: Buffer, name: string, qtype: number) { + const fullname = Buffer.from((name.endsWith(".") ? name : name + ".") + "\0"); + const list = answers[`${name} ${qtype}`] ?? [{ rrtype: qtype, error: ERR_NO_SUCH_RECORD }]; + return list.map(({ rrtype, rdata = [], flags = FLAGS_ADD, error = 0, ifindex = 0, ttl = 60, after }) => { + const body = Buffer.alloc(12 + fullname.length + 6 + rdata.length + 4); + body.writeUInt32BE(flags, 0); + body.writeUInt32BE(ifindex, 4); + body.writeInt32BE(error, 8); + let at = 12 + fullname.copy(body, 12); + at = body.writeUInt16BE(rrtype, at); + at = body.writeUInt16BE(1, at); + at = body.writeUInt16BE(rdata.length, at); + at += Buffer.from(rdata).copy(body, at); + body.writeUInt32BE(ttl, at); + const header = Buffer.alloc(HEADER_SIZE); + header.writeUInt32BE(1, 0); + header.writeUInt32BE(body.length, 4); + header.writeUInt32BE(op, 12); + context.copy(header, 16); + return { after, bytes: Buffer.concat([header, body]) }; + }); +} + +function cstring(data: Buffer, at: number) { + const end = data.indexOf(0, at); + return { value: data.toString("latin1", at, end), next: end + 1 }; +} + +function tlvs(data: Buffer, at: number) { + const out: Record = {}; + while (at + 4 <= data.length) { + const length = data.readUInt16BE(at + 2); + out[data.readUInt16BE(at)] = [...data.subarray(at + 4, at + 4 + length)]; + at += 4 + length; + } + return out; +} + +const requests: object[] = []; +let held: { conn: number; after: string; bytes: Buffer }[] = []; + +function handle(conn: number, header: Buffer, data: Buffer, errorFds: number[]) { + const op = header.readUInt32BE(12); + const context = header.subarray(16, 24); + const hasTLVs = (header.readUInt32BE(8) & IPC_FLAGS_TRAILING_TLVS) !== 0; + const status = Buffer.alloc(4); + + if (ops[op] === "connection") { + writeSync(conn, status); + return; + } + if (ops[op] === "cancel") return; + + // Byte 0 is the empty control path that marks a request whose status goes to a descriptor of its own. + const flags = data.readUInt32BE(1); + const ifindex = data.readUInt32BE(5); + const replies: ReturnType = []; + if (ops[op] === "query") { + const { value: name, next } = cstring(data, 9); + const rrtype = data.readUInt16BE(next); + for (const { conn, bytes } of held.filter(({ after }) => after === name)) writeSync(conn, bytes); + held = held.filter(({ after }) => after !== name); + requests.push({ op: "query", name, rrtype, flags, ifindex, tlvs: hasTLVs ? tlvs(data, next + 4) : {} }); + replies.push(...reply(QUERY_REPLY, context, name, rrtype)); + } else if (ops[op] === "addrinfo") { + const protocol = data.readUInt32BE(9); + const { value: name, next } = cstring(data, 13); + requests.push({ op: "addrinfo", name, protocol, flags, ifindex, tlvs: hasTLVs ? tlvs(data, next) : {} }); + if (protocol & 2) replies.push(...reply(ADDRINFO_REPLY, context, name, TYPE_AAAA)); + if (protocol & 1) replies.push(...reply(ADDRINFO_REPLY, context, name, TYPE_A)); + } else { + requests.push({ op }); + status.writeInt32BE(ERR_UNSUPPORTED); + } + + const errorFd = errorFds.shift(); + if (errorFd === undefined) throw new Error(`request with op ${op} came without a descriptor for its status`); + writeSync(errorFd, status); + closeSync(errorFd); + for (const { after, bytes } of replies) { + if (after === undefined) writeSync(conn, bytes); + else held.push({ conn, after, bytes }); + } +} + +const listener = listenOn(socketPath); +const client = Bun.spawn({ + cmd: [process.execPath, "-e", clientScript], + env: { ...process.env, DNSSD_UDS_PATH: socketPath }, + stdio: ["ignore", "inherit", "inherit", "pipe"], +}); +// Only the client holds the other end, so this becomes readable when the client is gone. +const clientGone = client.stdio[3] as number; + +const conns = new Map(); +serving: while (true) { + const fds = [clientGone, listener, ...conns.keys()]; + const pollfds = new DataView(new ArrayBuffer(fds.length * 8)); + fds.forEach((fd, i) => { + pollfds.setInt32(i * 8, fd, true); + pollfds.setInt16(i * 8 + 4, POLLIN, true); + }); + check(libc.poll(ptr(new Uint8Array(pollfds.buffer)), fds.length, -1), "poll"); + for (let i = 0; i < fds.length; i++) { + if (pollfds.getInt16(i * 8 + 6, true) === 0) continue; + const fd = fds[i]; + if (fd === clientGone) break serving; + if (fd === listener) { + conns.set(check(libc.accept(listener, null, null), "accept"), { pending: Buffer.alloc(0), errorFds: [] }); + continue; + } + const conn = conns.get(fd)!; + const { bytes, fds: passed } = receive(fd); + if (bytes.length === 0) { + closeSync(fd); + conns.delete(fd); + continue; + } + conn.errorFds.push(...passed); + conn.pending = Buffer.concat([conn.pending, bytes]); + while (conn.pending.length >= HEADER_SIZE) { + const size = HEADER_SIZE + conn.pending.readUInt32BE(4); + if (conn.pending.length < size) break; + handle(fd, conn.pending.subarray(0, HEADER_SIZE), conn.pending.subarray(HEADER_SIZE, size), conn.errorFds); + conn.pending = conn.pending.subarray(size); + } + } +} + +console.log(JSON.stringify(requests)); +process.exitCode = await client.exited; diff --git a/test/js/bun/dns/resolve-dns.test.ts b/test/js/bun/dns/resolve-dns.test.ts index a9d7efde6fa9..6f2fec086ab6 100644 --- a/test/js/bun/dns/resolve-dns.test.ts +++ b/test/js/bun/dns/resolve-dns.test.ts @@ -1,9 +1,11 @@ import { SystemError, dns } from "bun"; -import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isASAN, isWindows, withoutAggressiveGC } from "harness"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, isASAN, isMacOS, isWindows, tempDir, withoutAggressiveGC } from "harness"; import { isIP, isIPv4, isIPv6 } from "node:net"; import { join } from "node:path"; +import type { Answer, Answers } from "./mdnsresponder-fixture"; +const cc = Bun.which("cc") || Bun.which("clang"); const backends = ["system", "libc", "c-ares"]; const validHostnames = ["localhost", "example.com"]; const invalidHostnames = ["adsfa.asdfasdf.asdf.com"]; // known invalid @@ -298,4 +300,240 @@ describe("dns", () => { ); }); }); + + // On macOS the system backend talks to mDNSResponder over a unix socket. These put a scripted responder + // behind that socket, so each test sees the requests Bun makes and chooses the replies Bun gets. + describe.skipIf(!isMacOS || !cc)("system backend over the mDNSResponder socket", () => { + // libsystem_dnssd has a DNSSD_UDS_PATH override of its own, but on macOS 26 only a setuid process reads it. + const redirectConnect = /* c */ ` + #include + #include + #include + #include + + int connect_nocancel(int, const struct sockaddr *, socklen_t) __asm("_connect$NOCANCEL"); + + static const struct sockaddr *redirect(const struct sockaddr *addr, struct sockaddr_un *to) { + const char *path = getenv("DNSSD_UDS_PATH"); + if (!path || addr->sa_family != AF_UNIX || + strcmp(((const struct sockaddr_un *)addr)->sun_path, "/var/run/mDNSResponder")) + return addr; + to->sun_len = sizeof(*to); + to->sun_family = AF_UNIX; + strlcpy(to->sun_path, path, sizeof(to->sun_path)); + return (const struct sockaddr *)to; + } + + static int redirected_connect(int fd, const struct sockaddr *addr, socklen_t len) { + struct sockaddr_un to; + return connect(fd, redirect(addr, &to), len); + } + + static int redirected_connect_nocancel(int fd, const struct sockaddr *addr, socklen_t len) { + struct sockaddr_un to; + return connect_nocancel(fd, redirect(addr, &to), len); + } + + __attribute__((used, section("__DATA,__interpose"))) static const struct { + const void *replacement, *original; + } interpose[] = { + {(const void *)redirected_connect, (const void *)connect}, + {(const void *)redirected_connect_nocancel, (const void *)connect_nocancel}, + }; + `; + + let dir: ReturnType; + let sockets = 0; + + beforeAll(async () => { + dir = tempDir("dns-sd", { "redirect.c": redirectConnect }); + await using proc = Bun.spawn({ + cmd: [cc!, "-dynamiclib", "-o", "redirect.dylib", "redirect.c"], + cwd: String(dir), + env: bunEnv, + stdout: "inherit", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); + if (exitCode !== 0) throw new Error(`could not compile redirect.c: ${stderr}`); + }); + + afterAll(() => dir?.[Symbol.dispose]()); + + const A = 1; + const CNAME = 5; + const AAAA = 28; + // kDNSServiceFlagsShareConnection | kDNSServiceFlagsTimeout | kDNSServiceFlagsReturnIntermediates + const baseFlags = 0x4000 | 0x10000 | 0x1000; + const suppressUnusable = 0x8000; + const alias: Answer = { rrtype: CNAME, rdata: [5, ...Buffer.from("alias"), 4, ...Buffer.from("test"), 0] }; + const a = (...rdata: number[]): Answer => ({ rrtype: A, rdata }); + const aaaa = (first: number, last: number, more: Partial = {}): Answer => ({ + rrtype: AAAA, + rdata: [first >> 8, first & 0xff, ...Buffer.alloc(13), last], + ...more, + }); + + /** Runs `script` with `answers` behind the socket: the one line of JSON it prints, and the requests it made. */ + async function exchange(answers: Answers, script: string) { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + join(import.meta.dir, "mdnsresponder-fixture.ts"), + join(String(dir), `${sockets++}.sock`), + JSON.stringify(answers), + script, + ], + env: { ...bunEnv, DYLD_INSERT_LIBRARIES: join(String(dir), "redirect.dylib"), NO_PROXY: "*", no_proxy: "*" }, + stdout: "pipe", + stderr: "inherit", + }); + const [stdout, exitCode] = await Promise.all([proc.stdout.text(), proc.exited]); + const [printed, requests] = stdout + .trim() + .split("\n") + .map(line => JSON.parse(line)); + return { + printed, + // TLV 4 is IPC_TLV_TYPE_SERVICE_ATTR_FAILOVER_POLICY; 1 is kDNSServiceFailoverPolicyAllow. + requests: requests.map(({ tlvs, ...request }) => ({ ...request, failoverPolicy: tlvs?.[4]?.at(-1) })), + exitCode, + }; + } + + const lookup = (name: string, options: object = {}) => + `console.log(JSON.stringify(await Bun.dns.lookup(${JSON.stringify(name)}, ${JSON.stringify(options)}).catch(e => e.code)))`; + + const queries = (name: string, flags: number, rrtypes = [A, AAAA]) => + rrtypes.map(rrtype => ({ op: "query", name, rrtype, flags, ifindex: 0, failoverPolicy: 1 })); + + const both = { + "host.corp.example 1": [alias, a(10, 0, 0, 10)], + "host.corp.example 28": [alias, aaaa(0xfd00, 0x10)], + }; + + // https://github.com/oven-sh/bun/issues/44075 + test.concurrent("lookup() queries each family and allows failover to another resolver", async () => { + expect(await exchange(both, lookup("host.corp.example"))).toEqual({ + printed: [ + { address: "fd00::10", family: 6, ttl: 60 }, + { address: "10.0.0.10", family: 4, ttl: 60 }, + ], + requests: queries("host.corp.example", baseFlags | suppressUnusable), + exitCode: 0, + }); + }); + + test.concurrent("lookup() makes the same requests as getaddrinfo()", async () => { + const [system, libc] = await Promise.all( + ["system", "libc"].map(backend => exchange(both, lookup("host.corp.example", { backend }))), + ); + expect(libc.requests).toHaveLength(2); + expect(system.requests).toEqual(libc.requests); + }); + + test.concurrent("fetch() queries each family and allows failover to another resolver", async () => { + const script = ` + using server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("reached") }); + const response = await fetch("http://host.corp.example:" + server.port + "/"); + console.log(JSON.stringify(await response.text())); + `; + expect(await exchange({ "host.corp.example 1": [alias, a(127, 0, 0, 1)] }, script)).toEqual({ + printed: "reached", + requests: queries("host.corp.example", baseFlags | suppressUnusable), + exitCode: 0, + }); + }); + + test.concurrent.each([ + { family: 4, rrtype: A, address: "10.0.0.10" }, + { family: 6, rrtype: AAAA, address: "fd00::10" }, + ])("lookup() with family: $family queries that family alone", async ({ family, rrtype, address }) => { + expect(await exchange(both, lookup("host.corp.example", { family }))).toEqual({ + printed: [{ address, family, ttl: 60 }], + requests: queries("host.corp.example", baseFlags, [rrtype]), + exitCode: 0, + }); + }); + + // mDNSResponder tries the search domains for a single label only when the name has no trailing dot. + test.concurrent.each(["intranet", "host.corp.example."])("lookup(%j) sends the name as written", async name => { + expect(await exchange({ [`${name} 1`]: [a(10, 0, 0, 10)] }, lookup(name))).toEqual({ + printed: [{ address: "10.0.0.10", family: 4, ttl: 60 }], + requests: queries(name, baseFlags | suppressUnusable), + exitCode: 0, + }); + }); + + test.concurrent("lookup() scopes a link-local address to the interface it was seen on", async () => { + const answers = { "printer.local 28": [aaaa(0xfe80, 1, { ifindex: 7 }), aaaa(0xfd00, 1, { ifindex: 7 })] }; + expect(await exchange(answers, lookup("printer.local"))).toEqual({ + printed: [ + { address: "fe80::1%7", family: 6, ttl: 60 }, + { address: "fd00::1", family: 6, ttl: 60 }, + ], + requests: queries("printer.local", baseFlags | suppressUnusable), + exitCode: 0, + }); + }); + + test.concurrent("lookup() keeps only well-formed records that were added", async () => { + const removed = { ...a(10, 0, 0, 9), flags: 0 }; + const answers = { "host.corp.example 1": [a(10, 0, 0, 10), removed, a(10, 0, 0)] }; + expect(await exchange(answers, lookup("host.corp.example"))).toEqual({ + printed: [{ address: "10.0.0.10", family: 4, ttl: 60 }], + requests: queries("host.corp.example", baseFlags | suppressUnusable), + exitCode: 0, + }); + }); + + // kDNSServiceErr_NoAuth. The daemon reports a question it refuses to start with no record type. + const refused: Answer = { rrtype: 0, error: -65555 }; + + test.concurrent("lookup() waits for the other family when the daemon refuses one question", async () => { + const answers = { + "host.corp.example 1": [refused], + "host.corp.example 28": [aaaa(0xfd00, 0x10, { after: "second.test" })], + "first.test 1": [a(10, 0, 0, 1)], + "second.test 1": [a(10, 0, 0, 2)], + }; + // The answer for first.test is behind the refusal on the socket, so the refusal has been read once it resolves. + const script = ` + const host = Bun.dns.lookup("host.corp.example").catch(e => e.code); + await Bun.dns.lookup("first.test"); + await Bun.dns.lookup("second.test"); + console.log(JSON.stringify(await host)); + `; + expect(await exchange(answers, script)).toEqual({ + printed: [{ address: "fd00::10", family: 6, ttl: 60 }], + requests: ["host.corp.example", "first.test", "second.test"].flatMap(name => + queries(name, baseFlags | suppressUnusable), + ), + exitCode: 0, + }); + }); + + test.concurrent("lookup() reports ENOTFOUND when the daemon refuses both questions", async () => { + const answers = { "host.corp.example 1": [refused], "host.corp.example 28": [refused] }; + expect(await exchange(answers, lookup("host.corp.example"))).toEqual({ + printed: "DNS_ENOTFOUND", + requests: [ + ...queries("host.corp.example", baseFlags | suppressUnusable), + ...queries("host.corp.example", baseFlags), + ], + exitCode: 0, + }); + }); + + test.concurrent("lookup() asks once more without SuppressUnusable before it reports ENOTFOUND", async () => { + expect(await exchange({}, lookup("host.corp.example"))).toEqual({ + printed: "DNS_ENOTFOUND", + requests: [ + ...queries("host.corp.example", baseFlags | suppressUnusable), + ...queries("host.corp.example", baseFlags), + ], + exitCode: 0, + }); + }); + }); }); From aa8307619d8dccbda113a5a4aa1885e0cef7eaba Mon Sep 17 00:00:00 2001 From: robobun Date: Sat, 3 Oct 2026 01:09:48 -0700 Subject: [PATCH 04/15] node:buffer: reject a non-string value in the per-encoding write methods (#43452) ### Problem - The per-encoding write methods of `Buffer` (`utf8Write`, `latin1Write`, `asciiWrite`, `hexWrite`, `base64Write`, `base64urlWrite`, `ucs2Write`, `utf16leWrite`) accept a value that is not a string. `Buffer.alloc(4).utf8Write(123)` returns `3`, and an object value's `toString()` runs. Node throws `TypeError [ERR_INVALID_ARG_TYPE]: argument must be a string`. - The cause: both host function templates start with `toStringOrNull()` on the value (`src/jsc/bindings/JSBuffer.cpp:2453`, `:2487`). - No user reported this. A comparison with Node found it during work on #30138. ### Fix - The new helper `stringArgumentOrThrow` checks `isString()` and throws Node's error. Both templates call it in place of `toStringOrNull()`. - The order follows Node. `utf8Write`, `latin1Write` and `asciiWrite` check the bounds first, so `ERR_BUFFER_OUT_OF_BOUNDS` wins. The others reject the value before they read `offset` or `length`. - `Buffer.prototype.write` is not changed. #30138 changes its value check (#43451) and adds the identical helper block, so the two PRs merge in either order. - Verified: `test/js/node/buffer.test.js`, block `with a value that is not a string`: 35 tests fail on canary 1.4.3 and pass here. Also the rest of that file and the vendored `test-buffer-*.js` files. ### Background - In Node these methods are C++ (`StringWrite` and `SlowWriteString` in `src/node_buffer.cc`). Both start with `THROW_AND_RETURN_IF_NOT_STRING`: a fixed message, no conversion of the value. - Node wraps `utf8Write`, `latin1Write` and `asciiWrite` in JS that checks the bounds before the C++ code runs. That gives the two orders. - `toStringOrNull()` is JSC's `ToString`. For an object it runs user code, which can detach or resize the buffer.
Notes **Behavior change.** Code that passed a non-string value to these methods got the value's string form written. It now gets `ERR_INVALID_ARG_TYPE`, as in Node. The string path does not change: `toStringOrNull()` returned the same `JSString` for a string. **Probe, Node v26.3.0 against canary 1.4.3 (367d939d9).** `buf` is `Buffer.alloc(8)`. `A` is `ERR_INVALID_ARG_TYPE` "argument must be a string". `B` is `ERR_BUFFER_OUT_OF_BOUNDS`. | call | Node | canary | this PR | | --- | --- | --- | --- | | `utf8Write(123)` | A | `3` | A | | `utf8Write()` | A | `8` (writes `"undefine"`, the start of `"undefined"`) | A | | `utf8Write(Symbol())` | A | `TypeError`, no code | A | | `utf8Write(new String("ab"))` | A | `2` | A | | `hexWrite(123)` | A | `1` | A | | `base64Write(null)` | A | `3` | A | | `ucs2Write({})` | A | `8` | A | | `utf8Write(spy)`, `hexWrite(spy)` | A, no call into `spy` | `Symbol.toPrimitive` runs | A, no call | | `utf8Write(123, -1)`, `utf8Write(123, 0, 9)` | B | B | B | | `utf8Write(123, 8)`, `utf8Write(123, 0, 0)`, `utf8Write(123, NaN)` | A | `0` | A | | `utf8Write(123, offsetObject, lengthObject)` | A, after both `valueOf` calls | `1` | A, after both `valueOf` calls | | `hexWrite(123, -1)` | A | `ERR_OUT_OF_RANGE` | A | | `hexWrite(123, 9)` | A | B | A | | `hexWrite(123, offsetObject, lengthObject)` | A, no `valueOf` call | `1` | A, no `valueOf` call | | `utf8Write(123)` on an empty or a detached buffer | A | `0` | A | `latin1Write` and `asciiWrite` behave like `utf8Write`. `base64Write`, `base64urlWrite`, `ucs2Write` and `utf16leWrite` behave like `hexWrite`. After the fix, the 293-line probe output is identical to Node's for these methods, except for a receiver that is not a Buffer (see below). **The new tests against Node.** The block uses `bun:test`, like the blocks around it. I ran the same block under Node v26.3.0 with a small shim for `describe`, `it.each` and `expect`. 31 of 31 tests pass there. The other 4 are for `utf16leWrite`, which Node does not have. It shares the host function of `ucs2Write`. Upstream Node has no test that asserts this message for `Buffer`, so there is no upstream file to port. **No user code runs.** The test passes a Proxy whose handler is also a Proxy, so a lookup of any trap is recorded. The canary records two `get` traps and writes 1 byte. Node and this PR record nothing. This also guards against a later switch to `Bun::V::validateString`, which reads `constructor.name` from the value to build its message (#43451). **The whole class.** Node uses `THROW_AND_RETURN_IF_NOT_STRING` at four places in `src/node_buffer.cc`. Two are `StringWrite` and `SlowWriteString`, which this PR covers. The other two are `Btoa` and `Atob`. User code cannot reach them with a non-string: `btoa(123)` returns `"MTIz"` in Node and in Bun, because the JS layer converts the value first. **The `write()` sites this PR leaves**, in `jsBufferPrototypeFunction_writeBody`, with line numbers on this branch: - `JSBuffer.cpp:2565`, the `buf.write(value, encoding)` form: a bare `toString()`, so `buf.write(123, "utf8")` returns `3`. - `JSBuffer.cpp:2554` and `JSBuffer.cpp:2596`: `Bun::V::validateString` throws the right code with a different message, and `:2596` runs before `parseEncoding` (`:2605`), so `buf.write(123, 0, "bogus")` throws `ERR_INVALID_ARG_TYPE` where Node throws `ERR_UNKNOWN_ENCODING`. #30138 owns all three, and its description links #43451. At head `2edc3da077` its `writeBody` calls `stringArgumentOrThrow` at each site, after it resolves the encoding. I read that in its source. I did not build that branch. **Merge checks** with `git merge-tree --write-tree`, this branch at `5cf21732dd`: - Clean with `main` (`519963edc8`), with #30138 (`2edc3da077`) and with #43368 (`6cd01edf50`). - With #30138 in either order, the merged `JSBuffer.cpp` has one definition of `stringArgumentOrThrow` and five call sites. The helper block, with its comment, is byte-identical in the two PRs. - #43470 (`8e6c5aa3d9`) conflicts with this branch on the comment above the helper, because it has the earlier three-line comment. It conflicts with #30138 on the same lines. **Comments.** The last commit (`5cf21732dd`) shortens three comments and changes no code. The comment above `stringArgumentOrThrow` now has the text that #30138 uses, so that git merges the two identical blocks without a conflict. **Deleted guard.** In `jsBufferPrototypeFunction_StringWriteWithEncoding` the detach check was `(strValue.isObject() || offsetValue.isObject()) && isDetached()`. The value is now always a string at that point, so `strValue.isObject()` is gone. **Not changed: a receiver that is not a Buffer.** `Buffer.prototype.hexWrite.call({}, "ab")` throws a `TypeError` with no code (`Expected ArrayBufferView`). Node throws `ERR_INVALID_ARG_TYPE` "argument must be a buffer". The `Slice` methods have the same difference, so it is a separate change. #44525 tracks it. **Not changed: the conversion of `length` runs before the offset check.** With an `offset` that is out of bounds, `utf8Write`, `latin1Write` and `asciiWrite` call the `valueOf` of an object `length` before they throw `ERR_BUFFER_OUT_OF_BOUNDS`. Node throws for the offset first. The order is the same on `main`. A review of this PR found it. #44526 tracks it. **Exception scopes.** The new tests pass under `BUN_JSC_validateExceptionChecks=1 BUN_JSC_dumpSimulatedThrows=1` with the debug build. Every one of them takes the throw path. **Suites run with the debug build (ASAN).** On this branch at `5cf21732dd`, which includes `main` at `519963edc8`: - `test/js/node/buffer.test.js`: 718 pass, 1 skip, 0 fail, run with `--timeout 120000`. The file runs its `write()` tests a second time through a JS `write` that calls `utf8Write` and the other methods. - My machine was heavily loaded by other work during that run. With the default 5 s limit, 18 tests of this file failed in one run, all by timeout, and the 35 new tests passed. `main` at the same commit passed 683 of 684 with the default limit, in 78 s, a few hours earlier. - The 35 new tests under `BUN_JSC_validateExceptionChecks=1`: 35 pass. - `test-buffer-write.js` and `test-buffer-write-fast.js` in `test/js/node/test/parallel/`: exit 0. - The 293-line probe: the output differs from Node's only in the receiver lines and the `write()` lines. At the first commit of this PR (`2007e66a6e`), which has the same code before the comment change and the merge of `main`: - `test/js/node/test/parallel/test-buffer-*.js`: 62 of 63 files exit 0 when run directly. The other one, `test-buffer-resizable.js`, uses `node:test` and needs the test runner. It passes there (2 pass). - `buffer-compare-bounds`, `buffer-concat`, `buffer-copy-fill-detach`, `buffer-from-symbol-to-primitive`, `buffer-indexOf-detach`, `buffer-utf16`, `buffer-inspectmaxbytes`, `buffer-resolveObjectURL` (`test/js/node/*.test.ts`): all pass. - `test/js/node/buffer-jit.test.ts`: 11 pass, 1 fail. `differential fuzzer` exceeds its 120 s limit on my machine, a debug build with few cores. It exceeds the limit in the same way with `src/` at `origin/main`, and the file calls none of the methods in this PR. I saw this only locally, not in CI. **Self-review.** The review asked for three things in this description: name the `write()` sites that this PR leaves, name an owner for them, and say that no user reported the bug. All three are done. It also weighed a fix for `buf.write(value, encoding)` in this PR and advised against it, because #30138 and #43368 both change that line and already conflict with each other on it. The last commit changes comment lines only, so it had no new review.
--- src/jsc/bindings/JSBuffer.cpp | 29 +++++--- test/js/node/buffer.test.js | 130 ++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 9 deletions(-) diff --git a/src/jsc/bindings/JSBuffer.cpp b/src/jsc/bindings/JSBuffer.cpp index 95986d0d7b33..29fb49bb4002 100644 --- a/src/jsc/bindings/JSBuffer.cpp +++ b/src/jsc/bindings/JSBuffer.cpp @@ -2373,11 +2373,20 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_SliceWithEncoding(JSC::JSGl return jsBufferToString(lexicalGlobalObject, scope, castedThis, start, end - start, encoding); } +// Node's native writers reject a non-string value and never coerce it (THROW_AND_RETURN_IF_NOT_STRING, src/node_errors.h). +static JSString* stringArgumentOrThrow(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value) +{ + if (value.isString()) [[likely]] + return asString(value); + Bun::throwError(globalObject, scope, Bun::ErrorCode::ERR_INVALID_ARG_TYPE, "argument must be a string"_s); + return nullptr; +} + // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/buffer.js#L962-L990 // Only utf8Write/latin1Write/asciiWrite go through this strict JS wrapper in node; // the other encodings use jsBufferPrototypeFunction_StringWriteWithEncoding below. template -static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& vm, JSC::JSGlobalObject* lexicalGlobalObject, JSArrayBufferView* castedThis, JSString* str, JSValue offsetValue, JSValue lengthValue) +static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& vm, JSC::JSGlobalObject* lexicalGlobalObject, JSArrayBufferView* castedThis, JSValue stringValue, JSValue offsetValue, JSValue lengthValue) { auto scope = DECLARE_THROW_SCOPE(vm); @@ -2438,6 +2447,10 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& maxLength = std::min(byteLength - safeOffset, static_cast(intLength)); } + // Node's JS wrapper checks the bounds first, so a bounds error wins over a non-string value. + JSString* str = stringArgumentOrThrow(scope, lexicalGlobalObject, stringValue); + RETURN_IF_EXCEPTION(scope, {}); + RELEASE_AND_RETURN(scope, writeToBuffer(lexicalGlobalObject, castedThis, str, safeOffset, maxLength, encoding)); } @@ -2449,10 +2462,7 @@ static JSC::EncodedJSValue jsBufferPrototypeFunctionWriteWithEncoding(JSC::JSGlo auto* castedThis = dynamicDowncast(callFrame->thisValue()); - auto arg0 = callFrame->argument(0); - JSString* text = arg0.toStringOrNull(lexicalGlobalObject); - RETURN_IF_EXCEPTION(scope, {}); - + JSValue stringValue = callFrame->argument(0); JSValue offsetValue = callFrame->argument(1); JSValue lengthValue = callFrame->argument(2); @@ -2461,7 +2471,7 @@ static JSC::EncodedJSValue jsBufferPrototypeFunctionWriteWithEncoding(JSC::JSGlo return {}; } - RELEASE_AND_RETURN(scope, jsBufferPrototypeFunction_writeEncodingBody(vm, lexicalGlobalObject, castedThis, text, offsetValue, lengthValue)); + RELEASE_AND_RETURN(scope, jsBufferPrototypeFunction_writeEncodingBody(vm, lexicalGlobalObject, castedThis, stringValue, offsetValue, lengthValue)); } // https://github.com/nodejs/node/blob/v26.3.0/src/node_buffer.cc#L711-L741 @@ -2484,7 +2494,8 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_StringWriteWithEncoding(JSC const JSValue offsetValue = callFrame->argument(1); const JSValue lengthValue = callFrame->argument(2); - JSString* text = strValue.toStringOrNull(lexicalGlobalObject); + // The binding rejects the value before it reads offset or length. + JSString* text = stringArgumentOrThrow(scope, lexicalGlobalObject, strValue); RETURN_IF_EXCEPTION(scope, {}); size_t offset = 0; @@ -2492,9 +2503,9 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_StringWriteWithEncoding(JSC return {}; } - // toStringOrNull/toNumber only run user-overridable code for object arguments, and + // toNumber only runs user-overridable code for an object argument, and // that code can detach or resize the view, so re-validate only when it could have run. - if ((strValue.isObject() || offsetValue.isObject()) && castedThis->isDetached()) [[unlikely]] { + if (offsetValue.isObject() && castedThis->isDetached()) [[unlikely]] { throwTypeError(lexicalGlobalObject, scope, "ArrayBufferView is detached"_s); return {}; } diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js index 01ad7373d979..a2df8e1a0176 100644 --- a/test/js/node/buffer.test.js +++ b/test/js/node/buffer.test.js @@ -4527,6 +4527,136 @@ describe("raw Slice / Write bindings match Node", () => { expect(buf.toString("hex")).toBe(untouched); }); + // Node's native writers start with THROW_AND_RETURN_IF_NOT_STRING: they reject a value + // that is not a primitive string and never coerce it. + describe("with a value that is not a string", () => { + const NOT_A_STRING = expect.objectContaining({ + code: "ERR_INVALID_ARG_TYPE", + message: "argument must be a string", + }); + const nonStrings = () => [ + 123, + null, + undefined, + true, + 1n, + Symbol("s"), + {}, + [], + new String("ab"), + Buffer.from("ab"), + () => {}, + ]; + + it.each([...strict, ...clamping])("%s throws ERR_INVALID_ARG_TYPE", method => { + const buf = dest(); + for (const value of nonStrings()) { + expect(() => buf[method](value)).toThrow(NOT_A_STRING); + expect(() => buf[method](value, 0, 1)).toThrow(NOT_A_STRING); + } + expect(() => buf[method]()).toThrow(NOT_A_STRING); + expect(() => buf[method](123)).toThrow(TypeError); + expect(buf.toString("hex")).toBe(untouched); + }); + + it.each([...strict, ...clamping])("%s does not coerce an object value", method => { + const calls = []; + const value = { + toString() { + calls.push("toString"); + return source[method]; + }, + valueOf() { + calls.push("valueOf"); + return source[method]; + }, + [Symbol.toPrimitive]() { + calls.push("toPrimitive"); + return source[method]; + }, + }; + // The handler is a Proxy too, so a lookup of any trap is recorded. A message built + // from the value would show up here as a `get` of "constructor". + const proxy = new Proxy(value, new Proxy({}, { get: (_, trap) => void calls.push(`trap ${trap}`) })); + const buf = dest(); + expect(() => buf[method](value)).toThrow(NOT_A_STRING); + expect(() => buf[method](proxy)).toThrow(NOT_A_STRING); + expect(calls).toEqual([]); + expect(buf.toString("hex")).toBe(untouched); + }); + + it.each([...strict, ...clamping])("%s throws on an empty, a detached and a plain Uint8Array receiver", method => { + const detached = Buffer.from(new ArrayBuffer(9)); + structuredClone(detached.buffer, { transfer: [detached.buffer] }); + expect(() => Buffer.alloc(0)[method](123)).toThrow(NOT_A_STRING); + expect(() => detached[method](123)).toThrow(NOT_A_STRING); + expect(() => Buffer.prototype[method].call(new Uint8Array(9), 123)).toThrow(NOT_A_STRING); + }); + + // utf8Write/latin1Write/asciiWrite check the bounds in a JS wrapper first. The native + // writer sees the value only after that, so a bounds error wins. + it.each(strict)("%s reports an out-of-bounds offset or length first", method => { + const buf = dest(); + expect(() => buf[method](123, -1)).toThrow(OUT_OF_BOUNDS); + expect(() => buf[method](123, 10)).toThrow(OUT_OF_BOUNDS); + expect(() => buf[method](123, Infinity)).toThrow(OUT_OF_BOUNDS); + expect(() => buf[method](123, 0, -1)).toThrow(OUT_OF_BOUNDS); + expect(() => buf[method](123, 0, 10)).toThrow(OUT_OF_BOUNDS); + expect(() => buf[method](123, 6, 4)).toThrow(OUT_OF_BOUNDS); + // In bounds, including the ranges that leave nothing to write. + expect(() => buf[method](123, 9)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 0, 0)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, NaN)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, NaN, NaN)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, "abc")).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 1.5)).toThrow(NOT_A_STRING); + }); + + it.each(strict)("%s converts offset and length before it rejects the value", method => { + const converted = []; + const arg = (name, result) => ({ + valueOf() { + if (!converted.includes(name)) converted.push(name); + return result; + }, + }); + expect(() => dest()[method](123, arg("offset", 0), arg("length", 1))).toThrow(NOT_A_STRING); + expect(converted).toEqual(["offset", "length"]); + + const throwing = message => ({ + valueOf() { + throw new Error(message); + }, + }); + expect(() => dest()[method](123, throwing("offset valueOf"))).toThrow("offset valueOf"); + expect(() => dest()[method](123, 0, throwing("length valueOf"))).toThrow("length valueOf"); + }); + + // base64/base64url/hex/ucs2 are the raw binding. It rejects the value before it reads + // offset or length, so the value error wins and neither argument is converted. + it.each(clamping)("%s rejects the value before it reads offset and length", method => { + const buf = dest(); + expect(() => buf[method](123, -1)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 10)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, Infinity)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 0, -1)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 9, 1)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, 0, 0)).toThrow(NOT_A_STRING); + expect(() => buf[method](123, NaN, NaN)).toThrow(NOT_A_STRING); + + const converted = []; + const arg = name => ({ + valueOf() { + converted.push(name); + throw new Error(`${name} valueOf`); + }, + }); + expect(() => buf[method](123, arg("offset"), arg("length"))).toThrow(NOT_A_STRING); + expect(converted).toEqual([]); + expect(buf.toString("hex")).toBe(untouched); + }); + }); + it("the documented write() wrapper is unchanged", () => { const buf = dest(); expect(() => buf.write("hello", 6, 1000)).toThrow(expect.objectContaining({ code: "ERR_OUT_OF_RANGE" })); From b73ae471a057b796b333a561ebc90e2e42050707 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 4 Oct 2026 07:55:17 -0700 Subject: [PATCH 05/15] Bump mimalloc: faster heaps, less purge churn; release memory during Bun.sleepSync (#44560) Bumps mimalloc to oven-sh/mimalloc#59. - mi_heap_new + mi_heap_destroy (every MimallocArena) is about 7x cheaper with one allocation and 13x cheaper when unused, and no longer slows down when several threads create heaps at once. - Purging stops doing work that is undone right away: an allocation takes resident memory before purged memory, purge_delay is a minimum age while threads are busy, and a park no longer wakes the scavenger every time. - Memory that other threads free into the pages of a thread that stays blocked is given back after 30 s instead of when that thread wakes up. - The event loop only hands its heaps to the scavenger on a poll that can block. - Bun.sleepSync hands them over as well, so what the thread freed does not stay resident for the whole sleep. --- .../bun-usockets/src/eventing/epoll_kqueue.c | 7 +-- scripts/build/deps/mimalloc.ts | 2 +- src/runtime/api/BunObject.rs | 6 +++ test/js/bun/util/sleepSync.test.ts | 47 +++++++++++++++++++ test/js/web/timers/timer-gc-roots.test.ts | 3 ++ 5 files changed, 61 insertions(+), 4 deletions(-) diff --git a/packages/bun-usockets/src/eventing/epoll_kqueue.c b/packages/bun-usockets/src/eventing/epoll_kqueue.c index 0f861e510ee6..4d017cc34220 100644 --- a/packages/bun-usockets/src/eventing/epoll_kqueue.c +++ b/packages/bun-usockets/src/eventing/epoll_kqueue.c @@ -514,9 +514,10 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout /* The scavenger sweeps our heaps while we are in the kernel. Must come after * Bun__JSC_onBeforeWait, which allocates: nothing may touch our heaps until the matching * _end. mimalloc paces the sweep itself, so this costs a compare-and-swap per tick. - * With no scavenger to hand off to, fall back to sweeping inline -- but only on a tick that - * really parks, and rate-limited, because doing it between ticks is what we are avoiding. */ - const int handed_off = mi_on_thread_idle_start(); + * Only on a tick that really parks: there is no time to sweep in a poll that returns at once. + * With no scavenger to hand off to, fall back to sweeping inline, rate-limited, because doing + * it between ticks is what we are avoiding. */ + const int handed_off = will_idle_inside_event_loop && mi_on_thread_idle_start(); if (!handed_off && will_idle_inside_event_loop) { static const uint64_t idle_sweep_interval_ns = 100 * 1000000ULL; static _Thread_local uint64_t last_idle_sweep_ns = 0; diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index 2984efa98b8a..76238c1f5b7e 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "eab09015a5850ae18fc43ccfaa5bbe8272992314"; +const MIMALLOC_COMMIT = "7a828c116d96bb9a66f6c2e3417c739cbbfdb23c"; export const mimalloc: Dependency = { name: "mimalloc", diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs index d5dea22f877e..6724f2592b95 100644 --- a/src/runtime/api/BunObject.rs +++ b/src/runtime/api/BunObject.rs @@ -1025,9 +1025,15 @@ fn sleep_sync(global_object: &JSGlobalObject, callframe: &CallFrame) -> JsResult ))); } + // mimalloc's scavenger sweeps this thread's heaps while it sleeps, as it does across the event loop's poll. + // SAFETY: nothing allocates or frees on this thread until `mi_on_thread_idle_end` below. + let handed_off = unsafe { bun_alloc::mimalloc::mi_on_thread_idle_start() }; std::thread::sleep(core::time::Duration::from_millis( u64::try_from(milliseconds).expect("int cast"), )); + if handed_off { + bun_alloc::mimalloc::mi_on_thread_idle_end(); + } Ok(JSValue::UNDEFINED) } diff --git a/test/js/bun/util/sleepSync.test.ts b/test/js/bun/util/sleepSync.test.ts index 45361c2b5b36..99a9f820e943 100644 --- a/test/js/bun/util/sleepSync.test.ts +++ b/test/js/bun/util/sleepSync.test.ts @@ -1,5 +1,6 @@ import { sleepSync } from "bun"; import { expect, it } from "bun:test"; +import { bunEnv, bunExe, isASAN } from "harness"; it("sleepSync uses milliseconds", async () => { const start = performance.now(); @@ -28,3 +29,49 @@ it("sleepSync with negative number throws", async () => { it("can map with sleepSync", async () => { [1, 2, 3].map(sleepSync); }); + +// Free blocks inside pages that are still in use belong to the thread that owns the pages. They go back to the OS +// when that thread tells mimalloc that it is about to block. +it.skipIf(isASAN /* malloc is not mimalloc */)( + "sleepSync lets mimalloc release this thread's free memory", + async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const { heapStats } = require("bun:jsc"); + const purgeCalls = () => heapStats().mimalloc.purge_calls; + const spin = ms => { const start = performance.now(); while (performance.now() - start < ms); }; + + // the characters of these strings are allocated and freed by this thread + let strings = []; + for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); + // (far enough apart that whole OS pages are free in between, also where those are 16 KB) + strings = strings.filter((_, i) => i % 64 === 0); + Bun.gc(true); + + // what needs no idle thread settles first, without going idle + let before = purgeCalls(); + for (let stable = 0, tries = 0; stable < 3 && tries < 50; tries++) { + spin(60); + const now = purgeCalls(); + stable = now === before ? stable + 1 : 0; + before = now; + } + + let released = 0; + for (let i = 0; i < 20 && released < 500; i++) { + Bun.sleepSync(100); + released = purgeCalls() - before; + } + console.log(released >= 500, strings.length); + `, + ], + env: bunEnv, + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr, exitCode }).toEqual({ stdout: "true 1563\n", stderr: "", exitCode: 0 }); + }, +); diff --git a/test/js/web/timers/timer-gc-roots.test.ts b/test/js/web/timers/timer-gc-roots.test.ts index 6f5576f2c35f..a88f55d3c555 100644 --- a/test/js/web/timers/timer-gc-roots.test.ts +++ b/test/js/web/timers/timer-gc-roots.test.ts @@ -133,6 +133,9 @@ describe.concurrent("AbortSignal.timeout is released when its wrapper is collect await new Promise(r => setTimeout(r, 10)); } } + // RSS still climbs to its plateau in the first rounds + await round(); + await round(); await round(); const before = process.memoryUsage().rss; await round(); From 1878660bb47a6a87ebe518cea0581ef4bf9b71f8 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 4 Oct 2026 10:41:36 -0700 Subject: [PATCH 06/15] Bump WebKit: idle JSC threads and Atomics.wait release their free memory (#44564) Bumps WebKit to oven-sh/WebKit#768. Nothing else is in the range. - JSC's worker threads (wasm and JIT compilers, GC helpers) release the free memory of their mimalloc heap after 100 ms idle instead of when they exit after 10 s. Fixes #41438. - A thread that blocks in Atomics.wait for over 100 ms does the same. It used to keep everything it had freed for the whole wait. The wasm test is from #41449. --- scripts/build/deps/webkit.ts | 2 +- test/js/bun/wasm/compile-rss-fixture.mjs | 145 +++++++++++++++++++++++ test/js/bun/wasm/compile-rss.test.ts | 38 ++++++ test/js/web/atomics.test.ts | 116 ++++++++++++++++++ 4 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 test/js/bun/wasm/compile-rss-fixture.mjs create mode 100644 test/js/bun/wasm/compile-rss.test.ts diff --git a/scripts/build/deps/webkit.ts b/scripts/build/deps/webkit.ts index d2f910c88802..f4ffdb22eb45 100644 --- a/scripts/build/deps/webkit.ts +++ b/scripts/build/deps/webkit.ts @@ -3,7 +3,7 @@ * for local mode. Override via `--webkit-version=` to test a branch. * From https://github.com/oven-sh/WebKit releases. */ -export const WEBKIT_VERSION = "1600131e46b5af48bbda3559af8d8a3327230b6e"; +export const WEBKIT_VERSION = "5718a6ec579b98362ea7276a426deedcc6281ef5"; /** * WebKit (JavaScriptCore) — the JS engine. diff --git a/test/js/bun/wasm/compile-rss-fixture.mjs b/test/js/bun/wasm/compile-rss-fixture.mjs new file mode 100644 index 000000000000..c5ff8e8b2f12 --- /dev/null +++ b/test/js/bun/wasm/compile-rss-fixture.mjs @@ -0,0 +1,145 @@ +// Fixture for compile-rss.test.ts. Compiles a generated wasm module 6 times, discards each result, +// then polls RSS until it falls below the target (argv[2], MiB) or the deadline passes. The idle +// compiler threads exit after 10 s and release everything then, so the deadline stays well below +// that. Prints one JSON line with the lowest RSS growth seen, relative to the RSS before the first +// compile. + +class Bytes { + constructor() { + this.buf = new Uint8Array(1 << 20); + this.len = 0; + } + push(...bytes) { + for (const b of bytes) this.byte(b); + } + byte(b) { + if (this.len === this.buf.length) { + const next = new Uint8Array(this.buf.length * 2); + next.set(this.buf); + this.buf = next; + } + this.buf[this.len++] = b; + } + leb(n) { + do { + let b = n & 0x7f; + n >>>= 7; + if (n !== 0) b |= 0x80; + this.byte(b); + } while (n !== 0); + } + append(other) { + for (let i = 0; i < other.len; i++) this.byte(other.buf[i]); + } + section(id, payload) { + this.byte(id); + this.leb(payload.len); + this.append(payload); + } + bytes() { + return this.buf.subarray(0, this.len); + } +} + +// A module shaped like a tree-sitter parser: a few dozen functions of 20 to 30 KB of control-flow +// heavy code, plus one giant function. Compiling it makes each wasm compiler thread allocate and +// free tens of MB of temporaries. Uniform modules of many small functions do not show the +// retention: their pages end up fully free and mimalloc's scavenger purges them on its own. +function makeModule({ functionCount = 35, opsPerFunction = 1400, giantOps = 30000 } = {}) { + const out = new Bytes(); + out.push(0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00); + + // type 0: (i32, i32) -> i32 + const types = new Bytes(); + types.push(1, 0x60, 2, 0x7f, 0x7f, 1, 0x7f); + out.section(1, types); + + const funcs = new Bytes(); + funcs.leb(functionCount); + for (let f = 0; f < functionCount; f++) funcs.byte(0); + out.section(3, funcs); + + const exports = new Bytes(); + exports.push(1, 1, 0x66, 0x00, 0); // export "f" = func 0 + out.section(7, exports); + + const code = new Bytes(); + code.leb(functionCount); + for (let f = 0; f < functionCount; f++) { + const ops = f === 0 ? giantOps : opsPerFunction + ((f * 7919) % 400); + const body = new Bytes(); + body.push(1, 1, 0x7f); // one local group: 1 x i32 + body.push(0x20, 0); // local.get 0 + for (let i = 0; i < ops; i++) { + // block + // local.get 1; i32.const k; i32.add; local.tee 2 + // br_table {0,0,0,0} 0 + // local.get 2; local.get 0; call g; drop + // end + // local.get 2; i32.xor + body.push(0x02, 0x40, 0x20, 1, 0x41); + body.leb((f * 31 + i * 7) & 0x3f); + body.push(0x6a, 0x22, 2, 0x0e, 4, 0, 0, 0, 0, 0, 0x20, 2, 0x20, 0, 0x10); + body.leb((f + 1) % functionCount); + body.push(0x1a, 0x0b, 0x20, 2, 0x73); + } + body.byte(0x0b); // end + code.leb(body.len); + code.append(body); + } + out.section(10, code); + return out.bytes(); +} + +const targetMiB = Number(process.argv[2]); +if (!Number.isFinite(targetMiB)) throw new Error(`expected the target in MiB as argv[2], got ${process.argv[2]}`); +// On Darwin mimalloc returns memory with MADV_FREE_REUSABLE, which the kernel keeps counted in RSS +// until it reuses the pages. phys_footprint drops at once, so measure that there. memoryFootprint() +// returns undefined when task_info fails, so fall back to RSS. +const rss = () => + (process.platform === "darwin" ? Bun.unsafe.memoryFootprint?.() : undefined) ?? process.memoryUsage.rss(); +const sleep = ms => new Promise(r => setTimeout(r, ms)); +const bytes = makeModule(); + +// Building the module leaves garbage behind. Wait until RSS stops falling before taking the +// baseline: mimalloc hands freed memory back on its own schedule, and there is no signal for it. +let base = Infinity; +const baseDeadline = performance.now() + 2000; +do { + Bun.gc(true); + await sleep(50); + const now = rss(); + if (now >= base - 1048576) break; + base = now; +} while (performance.now() < baseDeadline); +// Each compile adds to what the unfixed threads keep (8 threads: 18 to 27 MiB after 3 compiles, +// 35 to 39 MiB after 6) and costs about 20 ms. The fixed build returns to the baseline either way. +for (let i = 0; i < 6; i++) { + let mod = await WebAssembly.compile(bytes); + mod = null; +} +// Read before anything is freed: the test checks that the compiles grew RSS at all. +const peak = rss() - base; +// Free the modules. The main thread frees their metadata into the compiler threads' pages, and only +// those threads can hand that memory back. They do so once they have worked and gone idle again, so +// give each of them one trivial function to compile after the modules are gone. +Bun.gc(true); +await WebAssembly.compile(makeModule({ functionCount: 16, opsPerFunction: 1, giantOps: 1 })); +Bun.gc(true); +const afterCompiles = rss() - base; + +const deadline = performance.now() + 3000; +let min = Infinity; +do { + await sleep(100); + Bun.gc(true); + min = Math.min(min, rss() - base); +} while (performance.now() < deadline && min / 1048576 >= targetMiB); +console.log( + JSON.stringify({ + moduleMiB: bytes.length / 1048576, + peakDeltaMiB: peak / 1048576, + afterCompilesDeltaMiB: afterCompiles / 1048576, + idleDeltaMiB: min / 1048576, + }), +); diff --git a/test/js/bun/wasm/compile-rss.test.ts b/test/js/bun/wasm/compile-rss.test.ts new file mode 100644 index 000000000000..8987962ce8d3 --- /dev/null +++ b/test/js/bun/wasm/compile-rss.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { bunEnv, bunExe, isASAN, isDebug } from "harness"; +import path from "node:path"; + +// Unfixed, the 8 idle wasm compiler threads keep 35 to 39 MiB of freed compile temporaries until +// they exit after 10 s. Fixed, they release it about 100 ms after the last compile and RSS returns +// to where it started. +const idleTargetMiB = 10; + +// Debug and ASAN builds link a JavaScriptCore that does not allocate through mimalloc, so the +// per-thread retention this test checks for does not exist there. +test.skipIf(isDebug || isASAN)( + "WebAssembly.compile does not retain memory in idle compiler threads (#41438)", + async () => { + await using proc = Bun.spawn({ + cmd: [bunExe(), path.join(import.meta.dir, "compile-rss-fixture.mjs"), String(idleTargetMiB)], + env: { + ...bunEnv, + // The retained amount scales with the compiler thread count. Pin it so the test does not + // depend on the core count of the machine. + BUN_JSC_numberOfWasmCompilerThreads: "8", + }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + const result = JSON.parse(stdout.trim().split("\n").at(-1)!); + // The whole measurement is in the object so a failure prints it. The compiles have to grow RSS + // well past the target first, or the idle check means nothing. + expect({ + ...result, + grew: result.peakDeltaMiB > idleTargetMiB * 2, + released: result.idleDeltaMiB < idleTargetMiB, + }).toMatchObject({ grew: true, released: true }); + expect(exitCode).toBe(0); + }, +); diff --git a/test/js/web/atomics.test.ts b/test/js/web/atomics.test.ts index a36b5cf80cd2..673821d3fc28 100644 --- a/test/js/web/atomics.test.ts +++ b/test/js/web/atomics.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, isASAN } from "harness"; describe("Atomics", () => { describe("basic operations", () => { @@ -307,3 +308,118 @@ describe("Atomics", () => { }); }); }); + +// Free blocks inside pages that are still in use belong to the thread that owns the pages. They go back to the OS +// when that thread tells mimalloc that it is idle, which a wait that takes a while does. +test.skipIf(isASAN /* malloc is not mimalloc */)( + "Atomics.wait lets mimalloc release this thread's free memory", + async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const { heapStats } = require("bun:jsc"); + const purgeCalls = () => heapStats().mimalloc.purge_calls; + const spin = ms => { const start = performance.now(); while (performance.now() - start < ms); }; + + // the characters of these strings are allocated and freed by this thread + let strings = []; + for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); + // (far enough apart that whole OS pages are free in between, also where those are 16 KB) + strings = strings.filter((_, i) => i % 64 === 0); + Bun.gc(true); + + // what needs no idle thread settles first, without going idle + let before = purgeCalls(); + for (let stable = 0, tries = 0; stable < 3 && tries < 50; tries++) { + spin(60); + const now = purgeCalls(); + stable = now === before ? stable + 1 : 0; + before = now; + } + + const view = new Int32Array(new SharedArrayBuffer(4)); + let released = 0; + for (let i = 0; i < 10 && released < 500; i++) { + if (Atomics.wait(view, 0, 0, 250) !== "timed-out") throw new Error("unexpected result"); + released = purgeCalls() - before; + } + console.log(released >= 500, strings.length); + `, + ], + env: bunEnv, + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr, exitCode }).toEqual({ stdout: "true 1563\n", stderr: "", exitCode: 0 }); + }, +); + +// 100 ms into a wait the waiter drops the lock of the waiter list to release its memory, and takes it again. +test.skipIf(isASAN /* malloc is not mimalloc */)( + "a notify that arrives while Atomics.wait releases this thread's free memory is not lost", + async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const view = new Int32Array(new SharedArrayBuffer(16)); + const [VALUE, WAITING, LAST] = [0, 1, 2]; + const worker = new Worker( + URL.createObjectURL( + new Blob( + [ + \` + self.onmessage = event => { + const view = new Int32Array(event.data); + const results = []; + for (;;) { + // Free memory in between what is in use, so that the release has something to do and takes a while. + let strings = []; + for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); + strings = strings.filter((_, i) => i % 64 === 0); + Bun.gc(true); + Atomics.store(view, 1, 1); + // A waiter that misses its notification is off the list already: it sleeps for the whole + // timeout and then still answers "ok". + const start = performance.now(); + const result = Atomics.wait(view, 0, 0, 10000); + results.push(performance.now() - start > 5000 ? "late" : result); + Atomics.store(view, 0, 0); + if (Atomics.load(view, 2)) return postMessage(results); + } + }; + \`, + ], + { type: "application/javascript" }, + ), + ), + ); + worker.postMessage(view.buffer); + const results = new Promise(resolve => (worker.onmessage = event => resolve(event.data))); + const delays = []; + for (let delay = 100; delay <= 107; delay += 1) delays.push(delay); + for (const delay of delays) { + while (Atomics.load(view, WAITING) !== 1); + Atomics.store(view, WAITING, 0); + const start = performance.now(); + while (performance.now() - start < delay); + if (delay === delays.at(-1)) Atomics.store(view, LAST, 1); + Atomics.store(view, VALUE, 1); + Atomics.notify(view, VALUE); + } + // ("not-equal" if the worker was held up for that long before it got to wait) + console.log(JSON.stringify((await results).filter(result => result !== "ok" && result !== "not-equal")), delays.length); + process.exit(0); + `, + ], + env: bunEnv, + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr, exitCode }).toEqual({ stdout: "[] 8\n", stderr: "", exitCode: 0 }); + }, + 30_000, +); From b335bbea18defc531453f0875812d76d769b3468 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 4 Oct 2026 16:21:17 -0700 Subject: [PATCH 07/15] fix(module): preserve package interop in async CJS imports Keep the parser format hint separate from the package-scope source tag. Adapts oven-sh/bun#40940 for the async runtime loader. --- CHANGELOG.md | 2 ++ docs/runtime/nodejs-compat.mdx | 2 ++ src/jsc/RuntimeTranspilerStore.rs | 33 +++++++++---------- src/runtime/jsc_hooks.rs | 1 + .../bun/resolve/esModule-annotation.test.js | 32 +++++++++++++++++- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf69a85e091b..c1b64535f5df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Preserve package-scope CommonJS interop in async imports without replacing the file's parser format. Adapts [oven-sh/bun#40940](https://github.com/oven-sh/bun/pull/40940). Thanks @robobun! + - Align WebKit and VM source-position assertions with Node's constructor locations and select these regressions for native stack-formatting changes in both fork CI lanes. - Preserve the native-context count in the fast `v8.getHeapStatistics()` adapter and avoid appending a second strict code-generation flag to inherited worker arguments. diff --git a/docs/runtime/nodejs-compat.mdx b/docs/runtime/nodejs-compat.mdx index f242366c62f1..852a99a22b82 100644 --- a/docs/runtime/nodejs-compat.mdx +++ b/docs/runtime/nodejs-compat.mdx @@ -171,6 +171,8 @@ Unreadable selected package metadata also throws `ERR_INVALID_PACKAGE_CONFIG`, r `import.meta.resolve()` validates the package scope of existing `.js`, `.ts`, and extensionless files. Missing-file URLs and `require.resolve()` defer that scope validation. Loading an already-resolved CommonJS file through the ESM loader preserves the original resolution mode. +Async imports of CommonJS files preserve package-scope interop. A `.cjs` file inside a `"type": "module"` package keeps `module.exports` as its default export and exposes its `__esModule` property, while the file still executes as CommonJS. + `registerHooks()` supports synchronous resolve/load chaining, source transformation, and deregistration. `nextResolve` honors parent URLs and custom conditions, and hooks preserve query/fragment module identities. Dynamic-import attributes, including attributes returned by resolve hooks, reach the matching load hook. Runtime `Bun.plugin()` hooks retry missing native resolutions with the filesystem refresh policy used by `registerHooks()`. Delegated resolution can load package-import targets created after their directory was cached, including files created inside an `onResolve` callback that returns `undefined`. diff --git a/src/jsc/RuntimeTranspilerStore.rs b/src/jsc/RuntimeTranspilerStore.rs index 0d33b1a75903..444c86517389 100644 --- a/src/jsc/RuntimeTranspilerStore.rs +++ b/src/jsc/RuntimeTranspilerStore.rs @@ -23,7 +23,7 @@ use bun_ptr::BackRef; use bun_resolve_builtins::{Alias as HardcodedAlias, Cfg as HardcodedAliasCfg}; use bun_resolver::fs as Fs; use bun_resolver::node_fallbacks; -use bun_resolver::package_json::MacroMap as MacroRemap; +use bun_resolver::package_json::{MacroMap as MacroRemap, PackageJSON}; use bun_sys::{self, Dir, Fd, FdExt as _, File, OpenDirOptions}; use bun_threading::Guarded; use bun_threading::unbounded_queue::{self, UnboundedQueue}; @@ -327,6 +327,7 @@ impl RuntimeTranspilerStore { referrer: String, loader: Loader, module_type: ModuleType, + package_json: Option<&PackageJSON>, module_loader: JSValue, ) -> *mut c_void { // The path text is heap-duplicated here and freed in `reset_for_pool` via @@ -342,17 +343,16 @@ impl RuntimeTranspilerStore { }; let promise: *mut JSInternalPromise = JSInternalPromise::create(global_object); - // NOTE: DirInfo should already be cached since module loading happens - // after module resolution, so this should be cheap - let mut resolved_source = ResolvedSource::default(); - match module_type { - ModuleType::Cjs => { - resolved_source.tag = ResolvedSourceTag::PackageJsonTypeCommonjs; - resolved_source.is_commonjs_module = true; - } - ModuleType::Esm => resolved_source.tag = ResolvedSourceTag::PackageJsonTypeModule, - ModuleType::Unknown => {} - } + // The package tag selects CJS interop; parsing uses the file's own format hint. + let resolved_source = ResolvedSource { + tag: match package_json.map(|pkg| pkg.module_type) { + Some(ModuleType::Cjs) => ResolvedSourceTag::PackageJsonTypeCommonjs, + Some(ModuleType::Esm) => ResolvedSourceTag::PackageJsonTypeModule, + _ => ResolvedSourceTag::Javascript, + }, + is_commonjs_module: module_type == ModuleType::Cjs, + ..ResolvedSource::default() + }; // Build the job by value and `get_init` it into the hive — the `Box` // alloc, `JSInternalPromise::create`, and `StrongOptional::create` @@ -371,6 +371,7 @@ impl RuntimeTranspilerStore { ticket: None, log: bun_ast::Log::init(), loader, + module_type, promise: StrongOptional::create(JSValue::from_cell(promise), global_object), module_loader: if module_loader.is_empty() { StrongOptional::empty() @@ -422,6 +423,7 @@ pub struct TranspilerJob { pub(crate) non_threadsafe_input_specifier: bun_core::String, pub(crate) non_threadsafe_referrer: bun_core::String, pub(crate) loader: Loader, + pub(crate) module_type: ModuleType, pub(crate) promise: StrongOptional, /// The `JSModuleLoader` that is fetching, when it is not the global object's (a /// `Bun.ModuleGraph`'s): handed back with the result. Empty otherwise. @@ -669,6 +671,7 @@ impl TranspilerJob { let path = self.path; let specifier = self.path.text; let loader = self.loader; + let module_type = self.module_type; let this_tag = self.resolved_source.tag; // RuntimeTranspilerCache has no per-allocator fields (Box<[u8]> + global mimalloc). @@ -802,12 +805,6 @@ impl TranspilerJob { && vm_main_hash == hash && strings::eql_long(vm_main, path.text, false); - let module_type: ModuleType = match this_tag { - ResolvedSourceTag::PackageJsonTypeCommonjs => ModuleType::Cjs, - ResolvedSourceTag::PackageJsonTypeModule => ModuleType::Esm, - _ => ModuleType::Unknown, - }; - let mut parse_options = ParseOptions { arena: &arena, path, diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 94d50e1d17fe..e2bc81449ba4 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -4468,6 +4468,7 @@ pub(crate) unsafe extern "C" fn Bun__transpileFile( referrer.clone(), concurrent_loader, module_type, + lr.package_json, module_loader, ) }; diff --git a/test/js/bun/resolve/esModule-annotation.test.js b/test/js/bun/resolve/esModule-annotation.test.js index 01c5fb3b40a8..3de84cb94974 100644 --- a/test/js/bun/resolve/esModule-annotation.test.js +++ b/test/js/bun/resolve/esModule-annotation.test.js @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { tempDir } from "harness"; +import { bunEnv, bunExe, tempDir } from "harness"; import { join } from "path"; import * as WithTypeModuleExportEsModuleAnnotationMissingDefault from "./with-type-module/export-esModule-annotation-empty.cjs"; import * as WithTypeModuleExportEsModuleAnnotationNoDefault from "./with-type-module/export-esModule-annotation-no-default.cjs"; @@ -10,6 +10,36 @@ import * as WithoutTypeModuleExportEsModuleAnnotationNoDefault from "./without-t import * as WithoutTypeModuleExportEsModuleAnnotation from "./without-type-module/export-esModule-annotation.cjs"; import * as WithoutTypeModuleExportEsModuleNoAnnotation from "./without-type-module/export-esModule-no-annotation.cjs"; +test.concurrent("async CJS imports preserve package interop without changing their format", async () => { + using dir = tempDir("cjs-package-interop", { + "package.json": JSON.stringify({ type: "module" }), + "annotated.cjs": "exports.default = 42; exports.__esModule = true;", + "plain.cjs": "globalThis.hasCommonJSThis = this !== undefined && this !== null;", + "entry.mjs": ` + import * as annotated from "./annotated.cjs"; + import "./plain.cjs"; + console.log(JSON.stringify({ + default: annotated.default, + marker: annotated.__esModule, + hasCommonJSThis: globalThis.hasCommonJSThis, + })); + `, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "entry.mjs"], + cwd: String(dir), + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ result: JSON.parse(stdout), stderr, exitCode }).toEqual({ + result: { default: { default: 42, __esModule: true }, marker: true, hasCommonJSThis: true }, + stderr: "", + exitCode: 0, + }); +}); + describe('without type: "module"', () => { test("module.exports = {}", () => { expect(WithoutTypeModuleExportEsModuleAnnotationMissingDefault.default).toEqual({}); From 9c3439ae55818a95b78581fbd4b014602f7635e4 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 4 Oct 2026 16:32:11 -0700 Subject: [PATCH 08/15] Bump mimalloc: upstream v3.5.3 (#44575) Bumps mimalloc to oven-sh/mimalloc#60, which merges upstream v3.5.3 into the fork (it was at v3.5.2). - Fixes a crash when the first mimalloc call on a new thread is mi_realloc(NULL, n), mi_reallocf, mi_strdup, mi_mallocn or mi_new, with a TLS model where the thread local heap of a new thread starts out as NULL (macOS and Windows). - Fixes mi_free_size_aligned of a small block with an alignment above its size. Bun's build (MI_FREE_USE_PAGEMAP=1) was not affected. - No function is removed from mimalloc.h, and no signature that src/mimalloc_sys binds changes. Instruction counts for the allocation and free paths are the same before and after. --- scripts/build/deps/mimalloc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index 76238c1f5b7e..f162fdf97830 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "7a828c116d96bb9a66f6c2e3417c739cbbfdb23c"; +const MIMALLOC_COMMIT = "92ef6587c57b262f6e6abf513f7776a2f8d12aeb"; export const mimalloc: Dependency = { name: "mimalloc", From c7b06d94bac19817ba34b6677bb1099fb4f6d2be Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 4 Oct 2026 23:42:12 +0000 Subject: [PATCH 09/15] docker: inline the release signing key instead of asking a keyserver for it Every Dockerfile fetched the key that signs SHASUMS256.txt from keys.openpgp.org at build time, falling back to keyserver.ubuntu.com. keys.openpgp.org is resetting connections ("keyserver receive failed: End of file"), and keyserver.ubuntu.com has never had this key ("No data"), so every image build fails. That fails the four test-docker-build-*.ts tests on every linux x64 lane and the "Release to Dockerhub" jobs. The public key is now written into the Dockerfile, so verifying a release needs nothing but github.com, which the build already downloads from. The fingerprint stays pinned: --recv-keys only accepted a key with the fingerprint it asked for, and --assert-signer now requires the signature to be made by it, whatever else is in the keyring. --assert-signer fails when a file named like the signature without ".asc" exists, which gpg's own --output created, so the verified checksums are written to SHASUMS256.verified.txt. That also removes the "file 'SHASUMS256.txt' was NOT verified!" warning every build printed. dirmngr was only there to reach keyservers. --- dockerhub/alpine/Dockerfile | 32 ++++++++++++++++++++++---------- dockerhub/debian-slim/Dockerfile | 31 +++++++++++++++++++++---------- dockerhub/debian/Dockerfile | 32 ++++++++++++++++++++++---------- dockerhub/distroless/Dockerfile | 31 +++++++++++++++++++++---------- 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/dockerhub/alpine/Dockerfile b/dockerhub/alpine/Dockerfile index eb983806ab1e..63c9f2d5410e 100644 --- a/dockerhub/alpine/Dockerfile +++ b/dockerhub/alpine/Dockerfile @@ -3,7 +3,9 @@ FROM alpine:3.24 AS build # https://github.com/oven-sh/bun/releases ARG BUN_VERSION=latest -RUN apk --no-cache add ca-certificates curl dirmngr gpg gpg-agent unzip \ +# The signing key is inlined so that a keyserver outage cannot fail the build. +# --assert-signer fails if --output is the signature's name without ".asc", hence ".verified.txt". +RUN apk --no-cache add ca-certificates curl gpg gpg-agent unzip \ && arch="$(apk --print-arch)" \ && case "${arch##*-}" in \ x86_64) build="x64-musl-baseline";; \ @@ -25,23 +27,33 @@ RUN apk --no-cache add ca-certificates curl dirmngr gpg gpg-agent unzip \ --compressed \ --retry 5 \ || (echo "error: failed to download: $tag" && exit 1) \ - && for key in \ - "F3DCC08A8572C0749B3E18888EAB4D40A7B22B59" \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" \ - || gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ + && printf '%s\n' \ + '-----BEGIN PGP PUBLIC KEY BLOCK-----' \ + '' \ + 'mDMEY9GQFhYJKwYBBAHaRw8BAQdAkppAqaXl0RkROz6NfdvlwYd2UuUVfHLk2NNY' \ + 'IzEdnT+0GVJvYm9idW4gPHJvYm9idW5Ab3Zlbi5zaD6IkwQTFgoAOxYhBPPcwIqF' \ + 'csB0mz4YiI6rTUCnsitZBQJj0ZAWAhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4H' \ + 'AheAAAoJEI6rTUCnsitZ96UBAMvwCLD6Ud1RZkpvvnGUU+idHt2hcNmYU2d0XxDI' \ + 'HQ0rAQCA9VFOtjZefQkfhDzgIgnEEdSFXaMiyY+D+LP8awNMCrg4BGPRkBYSCisG' \ + 'AQQBl1UBBQEBB0BpfePYSOEx8PihYkNXjlK5YT89CGHjEK5etleaB9i6OwMBCAeI' \ + 'eAQYFgoAIBYhBPPcwIqFcsB0mz4YiI6rTUCnsitZBQJj0ZAWAhsMAAoJEI6rTUCn' \ + 'sitZvhAA/j4SQxOCLheRG86A2181WAP4qLS1qxSw+fCf28DgiPfbAQCL0kcel+M9' \ + 'qbRIlMnwn6TwlQgN9w1qqlSnA9CbKXT9Aw==' \ + '=dGV6' \ + '-----END PGP PUBLIC KEY BLOCK-----' \ + | gpg --batch --import \ && curl "https://github.com/oven-sh/bun/releases/$release/SHASUMS256.txt.asc" \ -fsSLO \ --compressed \ --retry 5 \ - && gpg --batch --verify --output SHASUMS256.txt SHASUMS256.txt.asc \ + && gpg --batch --assert-signer F3DCC08A8572C0749B3E18888EAB4D40A7B22B59 \ + --verify --output SHASUMS256.verified.txt SHASUMS256.txt.asc \ || (echo "error: failed to verify: $tag" && exit 1) \ - && grep " bun-linux-$build.zip\$" SHASUMS256.txt | sha256sum -c - \ + && grep " bun-linux-$build.zip\$" SHASUMS256.verified.txt | sha256sum -c - \ || (echo "error: failed to verify: $tag" && exit 1) \ && unzip "bun-linux-$build.zip" \ && mv "bun-linux-$build/bun" /usr/local/bin/bun \ - && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.txt \ + && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.verified.txt \ && chmod +x /usr/local/bin/bun FROM alpine:3.24 diff --git a/dockerhub/debian-slim/Dockerfile b/dockerhub/debian-slim/Dockerfile index f510bf8220a0..ab45830acfcd 100644 --- a/dockerhub/debian-slim/Dockerfile +++ b/dockerhub/debian-slim/Dockerfile @@ -3,11 +3,12 @@ FROM debian:trixie-slim AS build # https://github.com/oven-sh/bun/releases ARG BUN_VERSION=latest +# The signing key is inlined so that a keyserver outage cannot fail the build. +# --assert-signer fails if --output is the signature's name without ".asc", hence ".verified.txt". RUN apt-get update -qq \ && apt-get install -qq --no-install-recommends \ ca-certificates \ curl \ - dirmngr \ gpg \ gpg-agent \ unzip \ @@ -34,23 +35,33 @@ RUN apt-get update -qq \ --compressed \ --retry 5 \ || (echo "error: failed to download: $tag" && exit 1) \ - && for key in \ - "F3DCC08A8572C0749B3E18888EAB4D40A7B22B59" \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" \ - || gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ + && printf '%s\n' \ + '-----BEGIN PGP PUBLIC KEY BLOCK-----' \ + '' \ + 'mDMEY9GQFhYJKwYBBAHaRw8BAQdAkppAqaXl0RkROz6NfdvlwYd2UuUVfHLk2NNY' \ + 'IzEdnT+0GVJvYm9idW4gPHJvYm9idW5Ab3Zlbi5zaD6IkwQTFgoAOxYhBPPcwIqF' \ + 'csB0mz4YiI6rTUCnsitZBQJj0ZAWAhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4H' \ + 'AheAAAoJEI6rTUCnsitZ96UBAMvwCLD6Ud1RZkpvvnGUU+idHt2hcNmYU2d0XxDI' \ + 'HQ0rAQCA9VFOtjZefQkfhDzgIgnEEdSFXaMiyY+D+LP8awNMCrg4BGPRkBYSCisG' \ + 'AQQBl1UBBQEBB0BpfePYSOEx8PihYkNXjlK5YT89CGHjEK5etleaB9i6OwMBCAeI' \ + 'eAQYFgoAIBYhBPPcwIqFcsB0mz4YiI6rTUCnsitZBQJj0ZAWAhsMAAoJEI6rTUCn' \ + 'sitZvhAA/j4SQxOCLheRG86A2181WAP4qLS1qxSw+fCf28DgiPfbAQCL0kcel+M9' \ + 'qbRIlMnwn6TwlQgN9w1qqlSnA9CbKXT9Aw==' \ + '=dGV6' \ + '-----END PGP PUBLIC KEY BLOCK-----' \ + | gpg --batch --import \ && curl "https://github.com/oven-sh/bun/releases/$release/SHASUMS256.txt.asc" \ -fsSLO \ --compressed \ --retry 5 \ - && gpg --batch --verify --output SHASUMS256.txt SHASUMS256.txt.asc \ + && gpg --batch --assert-signer F3DCC08A8572C0749B3E18888EAB4D40A7B22B59 \ + --verify --output SHASUMS256.verified.txt SHASUMS256.txt.asc \ || (echo "error: failed to verify: $tag" && exit 1) \ - && grep " bun-linux-$build.zip\$" SHASUMS256.txt | sha256sum -c - \ + && grep " bun-linux-$build.zip\$" SHASUMS256.verified.txt | sha256sum -c - \ || (echo "error: failed to verify: $tag" && exit 1) \ && unzip "bun-linux-$build.zip" \ && mv "bun-linux-$build/bun" /usr/local/bin/bun \ - && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.txt \ + && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.verified.txt \ && chmod +x /usr/local/bin/bun \ && which bun \ && bun --version diff --git a/dockerhub/debian/Dockerfile b/dockerhub/debian/Dockerfile index eb5c4a9e93c5..af6f21e3bef3 100644 --- a/dockerhub/debian/Dockerfile +++ b/dockerhub/debian/Dockerfile @@ -3,13 +3,15 @@ FROM debian:trixie-slim AS build # https://github.com/oven-sh/bun/releases ARG BUN_VERSION=latest +# The signing key is inlined so that a keyserver outage cannot fail the build. +# --assert-signer fails if --output is the signature's name without ".asc", hence ".verified.txt". + # Node.js includes python3 for node-gyp, see https://github.com/oven-sh/bun/issues/9807 # Though, not on slim and alpine images. RUN apt-get update -qq \ && apt-get install -qq --no-install-recommends \ ca-certificates \ curl \ - dirmngr \ gpg \ gpg-agent \ unzip \ @@ -37,23 +39,33 @@ RUN apt-get update -qq \ --compressed \ --retry 5 \ || (echo "error: failed to download: $tag" && exit 1) \ - && for key in \ - "F3DCC08A8572C0749B3E18888EAB4D40A7B22B59" \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" \ - || gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ + && printf '%s\n' \ + '-----BEGIN PGP PUBLIC KEY BLOCK-----' \ + '' \ + 'mDMEY9GQFhYJKwYBBAHaRw8BAQdAkppAqaXl0RkROz6NfdvlwYd2UuUVfHLk2NNY' \ + 'IzEdnT+0GVJvYm9idW4gPHJvYm9idW5Ab3Zlbi5zaD6IkwQTFgoAOxYhBPPcwIqF' \ + 'csB0mz4YiI6rTUCnsitZBQJj0ZAWAhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4H' \ + 'AheAAAoJEI6rTUCnsitZ96UBAMvwCLD6Ud1RZkpvvnGUU+idHt2hcNmYU2d0XxDI' \ + 'HQ0rAQCA9VFOtjZefQkfhDzgIgnEEdSFXaMiyY+D+LP8awNMCrg4BGPRkBYSCisG' \ + 'AQQBl1UBBQEBB0BpfePYSOEx8PihYkNXjlK5YT89CGHjEK5etleaB9i6OwMBCAeI' \ + 'eAQYFgoAIBYhBPPcwIqFcsB0mz4YiI6rTUCnsitZBQJj0ZAWAhsMAAoJEI6rTUCn' \ + 'sitZvhAA/j4SQxOCLheRG86A2181WAP4qLS1qxSw+fCf28DgiPfbAQCL0kcel+M9' \ + 'qbRIlMnwn6TwlQgN9w1qqlSnA9CbKXT9Aw==' \ + '=dGV6' \ + '-----END PGP PUBLIC KEY BLOCK-----' \ + | gpg --batch --import \ && curl "https://github.com/oven-sh/bun/releases/$release/SHASUMS256.txt.asc" \ -fsSLO \ --compressed \ --retry 5 \ - && gpg --batch --verify --output SHASUMS256.txt SHASUMS256.txt.asc \ + && gpg --batch --assert-signer F3DCC08A8572C0749B3E18888EAB4D40A7B22B59 \ + --verify --output SHASUMS256.verified.txt SHASUMS256.txt.asc \ || (echo "error: failed to verify: $tag" && exit 1) \ - && grep " bun-linux-$build.zip\$" SHASUMS256.txt | sha256sum -c - \ + && grep " bun-linux-$build.zip\$" SHASUMS256.verified.txt | sha256sum -c - \ || (echo "error: failed to verify: $tag" && exit 1) \ && unzip "bun-linux-$build.zip" \ && mv "bun-linux-$build/bun" /usr/local/bin/bun \ - && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.txt \ + && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.verified.txt \ && chmod +x /usr/local/bin/bun FROM debian:trixie diff --git a/dockerhub/distroless/Dockerfile b/dockerhub/distroless/Dockerfile index fec474bf7b42..6f8df44f0d5e 100644 --- a/dockerhub/distroless/Dockerfile +++ b/dockerhub/distroless/Dockerfile @@ -3,11 +3,12 @@ FROM debian:trixie-slim AS build # https://github.com/oven-sh/bun/releases ARG BUN_VERSION=latest +# The signing key is inlined so that a keyserver outage cannot fail the build. +# --assert-signer fails if --output is the signature's name without ".asc", hence ".verified.txt". RUN apt-get update -qq \ && apt-get install -qq --no-install-recommends \ ca-certificates \ curl \ - dirmngr \ gpg \ gpg-agent \ unzip \ @@ -34,23 +35,33 @@ RUN apt-get update -qq \ --compressed \ --retry 5 \ || (echo "error: failed to download: $tag" && exit 1) \ - && for key in \ - "F3DCC08A8572C0749B3E18888EAB4D40A7B22B59" \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" \ - || gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ + && printf '%s\n' \ + '-----BEGIN PGP PUBLIC KEY BLOCK-----' \ + '' \ + 'mDMEY9GQFhYJKwYBBAHaRw8BAQdAkppAqaXl0RkROz6NfdvlwYd2UuUVfHLk2NNY' \ + 'IzEdnT+0GVJvYm9idW4gPHJvYm9idW5Ab3Zlbi5zaD6IkwQTFgoAOxYhBPPcwIqF' \ + 'csB0mz4YiI6rTUCnsitZBQJj0ZAWAhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4H' \ + 'AheAAAoJEI6rTUCnsitZ96UBAMvwCLD6Ud1RZkpvvnGUU+idHt2hcNmYU2d0XxDI' \ + 'HQ0rAQCA9VFOtjZefQkfhDzgIgnEEdSFXaMiyY+D+LP8awNMCrg4BGPRkBYSCisG' \ + 'AQQBl1UBBQEBB0BpfePYSOEx8PihYkNXjlK5YT89CGHjEK5etleaB9i6OwMBCAeI' \ + 'eAQYFgoAIBYhBPPcwIqFcsB0mz4YiI6rTUCnsitZBQJj0ZAWAhsMAAoJEI6rTUCn' \ + 'sitZvhAA/j4SQxOCLheRG86A2181WAP4qLS1qxSw+fCf28DgiPfbAQCL0kcel+M9' \ + 'qbRIlMnwn6TwlQgN9w1qqlSnA9CbKXT9Aw==' \ + '=dGV6' \ + '-----END PGP PUBLIC KEY BLOCK-----' \ + | gpg --batch --import \ && curl "https://github.com/oven-sh/bun/releases/$release/SHASUMS256.txt.asc" \ -fsSLO \ --compressed \ --retry 5 \ - && gpg --batch --verify --output SHASUMS256.txt SHASUMS256.txt.asc \ + && gpg --batch --assert-signer F3DCC08A8572C0749B3E18888EAB4D40A7B22B59 \ + --verify --output SHASUMS256.verified.txt SHASUMS256.txt.asc \ || (echo "error: failed to verify: $tag" && exit 1) \ - && grep " bun-linux-$build.zip\$" SHASUMS256.txt | sha256sum -c - \ + && grep " bun-linux-$build.zip\$" SHASUMS256.verified.txt | sha256sum -c - \ || (echo "error: failed to verify: $tag" && exit 1) \ && unzip "bun-linux-$build.zip" \ && mv "bun-linux-$build/bun" /usr/local/bin/bun \ - && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.txt \ + && rm -f "bun-linux-$build.zip" SHASUMS256.txt.asc SHASUMS256.verified.txt \ && chmod +x /usr/local/bin/bun \ && which bun \ && bun --version From d4928764f23213ecf3cd61fa0b5b4a44369a5096 Mon Sep 17 00:00:00 2001 From: robobun Date: Sun, 4 Oct 2026 20:21:44 -0700 Subject: [PATCH 10/15] fetch: set up TLS once per connection, not once per request (#44507) ### Problem - A TLS 1.2 server can kill a bun process that calls `fetch()` twice. It renegotiates on the idle keep-alive connection, and the next request that takes the pooled socket dies: `panic: abort() called`, stack `SSL_set_session` <- `session_cache::install` <- `HTTPClient::on_open`. Regression in 1.4.0. - `on_open` (`src/http/lib.rs`) runs per request. Its TLS setup was guarded by `SSL_is_init_finished(ssl) == 0`, also true during a renegotiation, so it offered the cached session again. BoringSSL aborts on that. ### Fix - The TLS setup (SNI, ALPN, inline reject, session offer) becomes `configure_tls`. Only `on_connect` calls it, the open callback of a new connection. The guard is gone. - Correct because uSockets starts the handshake after that callback returns, and only a new connection takes that path. - Verified: `test/js/web/fetch/fetch.tls.test.ts` (new test, fails on 1.4.3-canary). Also ran the fetch-keepalive, fetch-session, proxy and tls renegotiation suites. ### Background - fetch parks idle connections in a pool and caches one TLS session per origin (#36598). `SSL_set_session` offers it, legal only before a handshake starts. - A renegotiation is a second handshake on a live TLS 1.2 connection, started by the server. - Considered a guard in `session_cache::install` and a check at pool pickup. Both leave per-connection work on the per-request path. ### Downsides - A request that takes a socket mid-renegotiation still stalls until its timeout: uSockets does not retry the parked write (#37094). A `todo` test records it. - Cost, release builds: a new TLS connection goes from 1,427 to 1,415 instructions, a pooled request from 5,146 to 5,123. Text grows by 1,024 bytes. The stripped binary keeps its size.
Notes **How to reach it.** No user code is needed beyond two `fetch()` calls to one HTTPS origin with verification on (the default). The server decides the rest. It does not have to withhold anything: a server that answers and then calls `renegotiate()` is enough when the second request starts inside the renegotiation. A server that never finishes the renegotiation keeps the window open for as long as it likes. **Why the guard was wrong.** `SSL_in_init()` is true whenever a handshake object exists and is not finalized. That covers the time before the first handshake step and the whole of a renegotiation. uSockets allows a client renegotiation (`ssl_renegotiate_explicit`) and runs it from its `SSL_read` loop, so the pool's idle data handler never sees the HelloRequest and the socket stays parked while it renegotiates. **What else the old block did to a renegotiating pooled socket.** It set SNI and ALPN again. `session_cache::install` took the cached session out of the cache before it offered it, so the next new connection could not resume. It replaced the armed session sink with an unarmed one. The new test checks the second point: after the pooled pickup, a new connection still resumes the cached session. **The test.** `fetch.tls.renegotiation-peer-fixture.mjs` runs under Node, because BoringSSL cannot send a HelloRequest. It is a TLS 1.2 keep-alive origin behind a TCP relay that can hold the client's bytes, plus a plain HTTP control server. The client fixture parks a connection, asks the origin to renegotiate, and gets the answer only when the relay holds the client's renegotiation ClientHello. Then it starts the request that takes the pooled socket. A control request queued behind it on the HTTP thread proves that the pickup ran. The test uses no timer. **Measured, linux x64.** - 1.4.3-canary.1 (`367d939d9`), release: the new test fails. The client prints `panic: abort() called` at the pooled pickup. - Debug build with ASAN of main `468efacace` without the change: the client exits with SIGABRT at the pooled pickup. - This branch, debug build with ASAN: the new test passes. `fetch.tls.test.ts` (62 pass, 1 todo), `fetch-keepalive.test.ts` (49), `fetch-session.test.ts` (34), `test/js/bun/http/proxy.test.ts` (97) and `test/js/node/tls/renegotiation.test.ts` (21) pass. With `--todo`, the todo test times out, as expected. `fetch-keepalive.test.ts` needed a second run with a longer per-test timeout: four tests hit the 5 s limit on the loaded build machine. - Without a relay (the origin answers, then renegotiates, and the client waits 1, 2 or 3 ms and fetches again), release builds, 5 runs per wait: main `80de08a1db` aborts in 13 of 15 runs. This branch aborts in 0 of 15. **What stays.** In 12 of those 15 runs on this branch the second request does not get an answer. The server reports that the renegotiation ended, and the request ends in a timeout. (Of the other 3, one got its answer and two failed with `EPROTO`, which main also shows in this cell.) The request write parks while the handshake runs, and uSockets dispatches the writable event only in its `HANDSHAKE_COMPLETED` state, which a renegotiation leaves until application data arrives. That is a separate defect in `packages/bun-usockets/src/crypto/openssl.c`, and #37094 has a fix for it. It does not come from the session cache: it also occurs with `BUN_FEATURE_FLAG_DISABLE_FETCH_TLS_SESSION_CACHE=1`. The todo test next to the new test awaits that request, so it flips when the write is retried. **Cost numbers.** Release builds of main `80de08a1db` and of this branch's source on the same commit. - Instructions from entry to return, nested calls included, counted with gdb single-steps. 3 runs, 3 consecutive calls per run. - Open callback of a new TLS connection (`Trampolines>::on_open`): 1,427 on main, 1,415 here. All 9 calls of each build agree. - `HTTPContext::connect` for a pooled TLS request, sixth request of the process: 5,146, 5,160, 5,146 on main and 5,123, 5,118, 5,123 here. Later requests vary more from run to run: 6,617 to 6,762 on main, 6,585 to 6,723 here. - `session_cache::install` calls: 1 per 1,000 keep-alive requests and 100 per 100 new connections, on both builds. - Size: `bun-profile` text is 80,727,274 bytes on main and 80,728,298 here (+1,024). The stripped `bun` is 80,868,936 bytes on both. Symbols: `HTTPClient::on_open::` 1,474 to 280 bytes, `HTTPContext::connect` 5,283 to 5,798, the open callback 375 to 1,582. The TLS block is now part of the open callback, and the compiler inlines the smaller `on_open` at the three pool reuse sites. - Tools: gdb 16.3, `llvm-nm`, `llvm-size`. `perf`, `valgrind` and `bloaty` are not installed on the machine. **Related.** #41671 covers the other way into the same BoringSSL abort, `socket.setSession()` after the handshake started. This change does not depend on it. **Other designs.** - A guard in `session_cache::install`: it stops the abort, but the cached session is still taken out of the cache and the sink is still replaced. - A stricter check in `on_open`: BoringSSL has no query for "the handshake has not started", so it needs a uSockets accessor or new per-socket state, and one branch stays on every pooled request. - Skip a socket that is mid-handshake at pool pickup: the abort stops only because `on_open` is not reached with such a socket. It also changes which requests reuse a connection. - Pass SNI, ALPN and the session to uSockets at connect time: a larger change across every TLS client. It is not needed to remove this defect. **Not run locally.** Windows and macOS.
--- src/http/ProxyTunnel.rs | 2 +- src/http/lib.rs | 29 ++++--- .../fetch.tls.renegotiation-client-fixture.ts | 38 ++++++++ .../fetch.tls.renegotiation-peer-fixture.mjs | 86 +++++++++++++++++++ test/js/web/fetch/fetch.tls.test.ts | 70 ++++++++++++++- 5 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 test/js/web/fetch/fetch.tls.renegotiation-client-fixture.ts create mode 100644 test/js/web/fetch/fetch.tls.renegotiation-peer-fixture.mjs diff --git a/src/http/ProxyTunnel.rs b/src/http/ProxyTunnel.rs index abdb981d4541..2f4e6f990116 100644 --- a/src/http/ProxyTunnel.rs +++ b/src/http/ProxyTunnel.rs @@ -625,7 +625,7 @@ impl ProxyTunnel { return; } }; - // The inner connection's form of the `set_inline_reject` call in `HTTPClient::on_open`. + // The inner connection's form of the `set_inline_reject` call in `HTTPClient::configure_tls`. if this.flags.reject_unauthorized { wrapper.set_inline_reject(); } diff --git a/src/http/lib.rs b/src/http/lib.rs index 265ba9db804e..36df14880bfa 100644 --- a/src/http/lib.rs +++ b/src/http/lib.rs @@ -1099,7 +1099,7 @@ pub enum AlpnOffer { /// Sets SNI (when `hostname` is non-empty), the legacy-server-connect option, /// the ALPN protocol list for `offer`, and enables SCT/OCSP stapling. Called -/// from `on_open` for every TLS socket — must run even when the hostname is an +/// before a TLS socket's first handshake. It must run even when the hostname is an /// IP literal (with empty SNI) so ALPN is still advertised. /// // `ssl` is the live SSL handle for a just-opened socket (BoringSSL never @@ -1844,14 +1844,21 @@ impl<'a> HTTPClient<'a> { self.state.request_stage = RequestStage::Opened; } + if !IS_SSL { + self.first_call::(socket); + } + Ok(()) + } + + /// Valid only before the handshake starts, so [`Self::on_connect`] is the only caller. + fn configure_tls(&mut self, socket: HttpSocket) { if IS_SSL { // SAFETY: socket.get_native_handle() returns a valid *mut SSL on TLS sockets let ssl_ptr: *mut boringssl::c::SSL = socket .get_native_handle() .map(|p| p.cast()) .unwrap_or(core::ptr::null_mut()); - // SAFETY: ssl_ptr is a live *mut SSL for the just-opened TLS socket - if !ssl_ptr.is_null() && unsafe { boringssl::c::SSL_is_init_finished(ssl_ptr) } == 0 { + if !ssl_ptr.is_null() { let raw_hostname = get_tls_hostname(self, self.http_proxy.is_some()); // Build a NUL-terminated SNI string only when the hostname is not an @@ -1895,9 +1902,9 @@ impl<'a> HTTPClient<'a> { .filter(|_| crate::session_cache::eligible(self)) { let want_tunnel = self.http_proxy.is_some() && self.url.is_https(); - // SAFETY: `ssl_ptr` is live and pre-handshake (guarded by - // `SSL_is_init_finished == 0` above); `get_ssl_ctx` returns - // the static `https_context` or the heap context this + // SAFETY: `ssl_ptr` is live and pre-handshake (uSockets starts + // the handshake when the open callback returns); `get_ssl_ctx` + // returns the static `https_context` or the heap context this // client holds a strong ref on, both of which outlive // every SSL attached to their socket group. unsafe { @@ -1917,10 +1924,7 @@ impl<'a> HTTPClient<'a> { } } } - } else { - self.first_call::(socket); } - Ok(()) } /// Runs once per connection, from the uSockets open callback. A socket @@ -1954,7 +1958,12 @@ impl<'a> HTTPClient<'a> { let _ = socket.set_keep_alive(true, 60); } - self.on_open::(socket) + self.on_open::(socket)?; + // `on_open` starts the request on a plain socket, which can free `self`. + if IS_SSL { + self.configure_tls::(socket); + } + Ok(()) } /// Whether to advertise "h2" in the TLS ALPN list. Restricted to request diff --git a/test/js/web/fetch/fetch.tls.renegotiation-client-fixture.ts b/test/js/web/fetch/fetch.tls.renegotiation-client-fixture.ts new file mode 100644 index 000000000000..57dbcf99e721 --- /dev/null +++ b/test/js/web/fetch/fetch.tls.renegotiation-client-fixture.ts @@ -0,0 +1,38 @@ +// Bun client for the "pooled socket mid-renegotiation" tests in +// fetch.tls.test.ts. +// argv: the of fetch.tls.renegotiation-peer-fixture.mjs, +// then "await-second" to wait for the request that takes the pooled socket +// env: CA_CERT, the certificate of the origin +const [relayPort, controlPort, mode] = process.argv.slice(2); +const origin = `https://127.0.0.1:${relayPort}/`; +const control = (path: string) => fetch(`http://127.0.0.1:${controlPort}${path}`).then(res => res.text()); +// Verification stays on: fetch keeps no TLS session for a client without it. +const tls = { ca: process.env.CA_CERT }; + +// Parks the connection in the keep-alive pool and its TLS 1.2 session in +// fetch's session cache. +console.log("first", await (await fetch(origin, { tls })).text()); + +// The origin asks for a renegotiation and the relay holds the client's +// ClientHello, so the pooled socket stays mid-handshake until "/release". +console.log("held record type", await control("/renegotiate")); + +// The pooled pickup. It is queued on the HTTP thread before the control +// request below, so once that one is answered the pickup has run. +const second = fetch(origin, { tls }).then(res => res.text()); +// Only "await-second" reads the outcome of that request. +second.catch(() => {}); +console.log("after the pooled pickup", await control("/ping")); + +await control("/release"); + +if (mode === "await-second") { + console.log("second", await second); + process.exit(0); +} + +// `keepalive: false` takes a connection of its own, so the origin reports +// whether the cached session is still there to offer. +console.log("third", await (await fetch(origin, { tls, keepalive: false })).text()); +console.log("resumed", await control("/resumed")); +process.exit(0); diff --git a/test/js/web/fetch/fetch.tls.renegotiation-peer-fixture.mjs b/test/js/web/fetch/fetch.tls.renegotiation-peer-fixture.mjs new file mode 100644 index 000000000000..5ccb2da7de6a --- /dev/null +++ b/test/js/web/fetch/fetch.tls.renegotiation-peer-fixture.mjs @@ -0,0 +1,86 @@ +// Node peer for the "pooled socket that is mid-renegotiation" test in +// fetch.tls.test.ts. It has to be Node: BoringSSL cannot start a renegotiation +// from the server side, so a Bun server cannot send a HelloRequest. +// +// origin: a TLS 1.2 keep-alive https server +// relay: a TCP proxy in front of the origin that can hold the client's bytes +// control: a plain http server through which the client fixture drives both +// +// The first line on stdout is " ". +import http from "node:http"; +import https from "node:https"; +import net from "node:net"; + +const listen = server => new Promise(resolve => server.listen(0, "127.0.0.1", () => resolve(server.address().port))); + +// One entry per TLS connection: did the client resume a cached session? +// A renegotiation adds no entry, it runs on the connection it belongs to. +const resumed = []; +let lastSecure; +const origin = https.createServer( + { key: process.env.SERVER_KEY, cert: process.env.SERVER_CERT, minVersion: "TLSv1.2", maxVersion: "TLSv1.2" }, + (req, res) => res.end("ok"), +); +// The idle connection has to stay open however slow the machine is. +origin.keepAliveTimeout = 0; +origin.on("secureConnection", socket => { + lastSecure = socket; + resumed.push(socket.isSessionReused()); + socket.on("error", () => {}); +}); +origin.on("tlsClientError", () => {}); +const originPort = await listen(origin); + +let holding = false; +let onHeld; +const releases = []; +const relay = net.createServer(client => { + const upstream = net.connect(originPort, "127.0.0.1"); + const held = []; + releases.push(() => { + for (const chunk of held.splice(0)) upstream.write(chunk); + }); + client.on("data", chunk => { + if (!holding) return void upstream.write(chunk); + held.push(chunk); + // The connection is idle, so the first held chunk is the client's + // renegotiation ClientHello. Its first byte is the TLS record type. + onHeld?.(chunk[0]); + onHeld = undefined; + }); + upstream.on("data", chunk => client.write(chunk)); + for (const socket of [client, upstream]) { + socket.on("error", () => {}); + socket.on("close", () => { + client.destroy(); + upstream.destroy(); + }); + } +}); +const relayPort = await listen(relay); + +const control = http.createServer((req, res) => { + switch (req.url) { + case "/renegotiate": { + // Answers once the relay holds the client's ClientHello. From then on + // the client's SSL is mid-handshake, until "/release". + holding = true; + onHeld = recordType => res.end(String(recordType)); + // A renegotiation that does not start must not leave the client waiting. + const fail = reason => void (res.writableEnded || res.end("no renegotiation: " + reason)); + lastSecure.once("close", () => fail("the connection closed")); + if (!lastSecure.renegotiate({}, error => error && fail(error.code))) fail("refused"); + return; + } + case "/release": + holding = false; + for (const release of releases) release(); + return void res.end("released"); + case "/resumed": + return void res.end(JSON.stringify(resumed)); + default: + return void res.end("pong"); + } +}); +const controlPort = await listen(control); +console.log(relayPort, controlPort); diff --git a/test/js/web/fetch/fetch.tls.test.ts b/test/js/web/fetch/fetch.tls.test.ts index 3cf57b86d0cc..695c1c8097a9 100644 --- a/test/js/web/fetch/fetch.tls.test.ts +++ b/test/js/web/fetch/fetch.tls.test.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, describe, expect, it } from "bun:test"; -import { bunEnv, bunExe, isASAN, isIPv6, isWindows, tmpdirSync } from "harness"; +import { bunEnv, bunExe, isASAN, isIPv6, isWindows, nodeExe, tmpdirSync } from "harness"; import { once } from "node:events"; import { readFileSync } from "node:fs"; import net from "node:net"; @@ -551,6 +551,74 @@ describe.concurrent("fetch-tls", () => { timeout, ); } + + // A pooled keep-alive socket is mid-handshake while its server + // renegotiates. The peer holds the client's renegotiation ClientHello, so + // the client's next request picks the socket up in that state. Node is the + // peer because BoringSSL cannot send a HelloRequest. + async function pickUpMidRenegotiation(...mode: string[]) { + await using peer = Bun.spawn({ + cmd: [nodeExe()!, join(import.meta.dir, "fetch.tls.renegotiation-peer-fixture.mjs")], + env: { ...bunEnv, SERVER_CERT: validTls.cert, SERVER_KEY: validTls.key }, + stdout: "pipe", + stderr: "inherit", + }); + // The peer prints " " when its servers listen. + let line = ""; + for await (const chunk of peer.stdout) { + line += new TextDecoder().decode(chunk); + if (line.includes("\n")) break; + } + const ports = line.trim().split(" "); + expect(ports).toHaveLength(2); + + await using client = Bun.spawn({ + cmd: [bunExe(), join(import.meta.dir, "fetch.tls.renegotiation-client-fixture.ts"), ...ports, ...mode], + env: { ...bunEnv, CA_CERT: validTls.cert }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([client.stdout.text(), client.stderr.text(), client.exited]); + return { stdout: stdout.trim().split("\n"), stderr, exitCode }; + } + + // The TLS setup of a connection (SNI, ALPN, the cached session) ran again + // for the request that picked the socket up. BoringSSL aborts the process + // when a session is offered after the handshake has begun. The setup also + // took the session out of the cache. + it.skipIf(!nodeExe())( + "a request that reuses a pooled socket mid-renegotiation does not set up TLS again", + async () => { + const { stdout, stderr, exitCode } = await pickUpMidRenegotiation(); + expect(stderr).toBe(""); + expect(stdout).toEqual([ + "first ok", + // 22 is a handshake record: the client's renegotiation ClientHello. + "held record type 22", + // The process died in the pooled pickup. + "after the pooled pickup pong", + "third ok", + // The third request is on a connection of its own, and it resumes + // the session that the first one cached. + "resumed [false,true]", + ]); + expect(exitCode).toBe(0); + }, + timeout, + ); + + // Observed: the request gets no answer, and the test times out. Its write + // parks while the renegotiation runs, and uSockets does not retry it when + // the renegotiation ends. + it.todo( + "a request that reuses a pooled socket mid-renegotiation gets its answer", + async () => { + const { stdout, exitCode } = await pickUpMidRenegotiation("await-second"); + expect(stdout).toEqual(["first ok", "held record type 22", "after the pooled pickup pong", "second ok"]); + expect(exitCode).toBe(0); + }, + timeout, + ); }); // Covers a family of HTTP-thread crashes (sentry BUN-2WC6 and siblings) where From a7c2bb616e49193b542081ba74c0fcb7abced15d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 4 Oct 2026 19:15:20 -0700 Subject: [PATCH 11/15] fix(module): pair namespace markers with the OpenClaw engine Carry openclaw/bun#106 as the source prerequisite for the rebased OpenClaw WebKit release. Keep import namespaces unmarked and give require(esm) its own live-export facade; preserve forwarded native namespaces and cached require markers. Remove the former prototype/private-field workaround. Preserve the work from ea461c8037a870df4a817b25e47fe95793ff28fa, fba679a18c7ac197a67c8cd80b91e006653965ea and 72f230bb93bac2fad29f511985ab1f21e11ca793. Adapts upstream Bun #33894 and #39888 by robobun; original attribution and regression coverage are retained. This prerequisite lands together with the engine manifest and upstream sync. --- CHANGELOG.md | 3 + docs/runtime/module-resolution.mdx | 2 + docs/runtime/nodejs-compat.mdx | 2 + src/js/builtins/CommonJS.ts | 36 +--- src/jsc/bindings/JSCommonJSModule.cpp | 27 ++- src/jsc/bindings/NodeVMModule.cpp | 5 - src/jsc/bindings/ZigGlobalObject.cpp | 50 ++---- .../import-attributes.test.ts | 10 +- .../bun/namespace-prototype-pollution.test.ts | 9 +- .../bun/resolve/esModule-annotation.test.js | 10 +- test/js/bun/resolve/esModule.test.ts | 164 +++++++++++++++--- .../require-esm-evaluating-cycle.test.ts | 7 +- .../js/node/module/require-extensions.test.ts | 2 +- 13 files changed, 207 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61cdd5c168d1..7a1f101fb5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -183,3 +183,6 @@ - Fix Intl.Segments.containing() at both halves of surrogate pairs with the pinned OpenClaw WebKit; cover grapheme, word, and sentence boundaries in both lookup directions. Ports [oven-sh/WebKit#753](https://github.com/oven-sh/WebKit/pull/753). Thanks @robobun! - Reduce kernel CPU during cold module imports on macOS by serializing transpiler-cache writes while preserving parallel parsing and cache reads. + +- Keep ESM namespaces free of inherited `__esModule` markers and preserve the own marker and live exports for `require(esm)`, fixing Vite/tsx namespace interop. Adapts [oven-sh/bun#33894](https://github.com/oven-sh/bun/pull/33894) and [oven-sh/WebKit#279](https://github.com/oven-sh/WebKit/pull/279). Thanks @robobun! + diff --git a/docs/runtime/module-resolution.mdx b/docs/runtime/module-resolution.mdx index 77760cb4b9ad..8f6b999a1fbc 100644 --- a/docs/runtime/module-resolution.mdx +++ b/docs/runtime/module-resolution.mdx @@ -161,6 +161,8 @@ import Stuff from "./my-commonjs.cjs"; const myStuff = require("./my-commonjs.cjs"); ``` +When `require()` loads an ES module with a default export and no explicit `__esModule` export, it returns a namespace with an own, enumerable `__esModule: true` property. Its exports remain live. Importing the same module returns the original namespace without that added marker. Both namespace objects have a null prototype and reject property assignments. + ### Top level await The only exception to this rule is top-level await. You can't `require()` a file that uses top-level await, since the `require()` function is inherently synchronous. diff --git a/docs/runtime/nodejs-compat.mdx b/docs/runtime/nodejs-compat.mdx index 80551bb15142..358bd613f993 100644 --- a/docs/runtime/nodejs-compat.mdx +++ b/docs/runtime/nodejs-compat.mdx @@ -15,6 +15,8 @@ Stdin and eval scripts report malformed ancestor package metadata only when an i The OpenClaw fork reports missing runtime imports without downloading npm packages. This default covers scripts, workers, and child processes running the fork. You can opt in with `--install=auto|fallback|force` or an explicit bunfig `install.auto` setting. Package-manager commands (`bun install`, `bun add`, `bun x`, and `bunx`) are unchanged. See [Auto-install](/runtime/auto-install). +Module namespaces have a null prototype. `require(esm)` adds an own enumerable `__esModule` marker only for modules with a default export and no explicit marker, without modifying the imported namespace. CommonJS namespaces preserve explicit markers and expose the original exports as `module.exports`. + ## Built-in Node.js modules `Error.prepareStackTrace` supports deletion, descriptor restoration, and accessor replacement. Nested stack formatting and throwing formatters leave later captures usable. diff --git a/src/js/builtins/CommonJS.ts b/src/js/builtins/CommonJS.ts index d4ea37ea1a46..92299d629b59 100644 --- a/src/js/builtins/CommonJS.ts +++ b/src/js/builtins/CommonJS.ts @@ -124,26 +124,11 @@ export function overridableRequire(this: JSCommonJSModule, originalId: string, o const namespace = out; // In a require cycle the namespace is live while the module body is still - // running, so an export named `__esModule` / `module.exports` may be in TDZ. - let esModule, moduleExports; + // running, so an export named `module.exports` may be in TDZ. + let moduleExports; try { - esModule = namespace.__esModule; moduleExports = namespace["module.exports"]; } catch {} - // In Bun, when __esModule is not defined, it's a CustomAccessor on the prototype. - // Various libraries expect __esModule to be set when using ESM from require(). - // We don't want to always inject the __esModule export into every module, - // And creating an Object wrapper causes the actual exports to not be own properties. - // So instead of either of those, we make it so that the __esModule property can be set at runtime. - // It only supports "true" and undefined. Anything non-truthy is treated as undefined. - // https://github.com/oven-sh/bun/issues/14411 - if (esModule === undefined) { - try { - namespace.__esModule = true; - } catch { - // https://github.com/oven-sh/bun/issues/17816 - } - } return (mod.exports = moduleExports ?? namespace); } @@ -217,25 +202,10 @@ export function requireESMFromHijackedExtension(this: JSCommonJSModule, id: stri const namespace = $requireESM(id, this); // See `overridableRequire`: TDZ-safe reads for the require-cycle case. - let esModule, moduleExports; + let moduleExports; try { - esModule = namespace.__esModule; moduleExports = namespace["module.exports"]; } catch {} - // In Bun, when __esModule is not defined, it's a CustomAccessor on the prototype. - // Various libraries expect __esModule to be set when using ESM from require(). - // We don't want to always inject the __esModule export into every module, - // And creating an Object wrapper causes the actual exports to not be own properties. - // So instead of either of those, we make it so that the __esModule property can be set at runtime. - // It only supports "true" and undefined. Anything non-truthy is treated as undefined. - // https://github.com/oven-sh/bun/issues/14411 - if (esModule === undefined) { - try { - namespace.__esModule = true; - } catch { - // https://github.com/oven-sh/bun/issues/17816 - } - } this.exports = moduleExports ?? namespace; } diff --git a/src/jsc/bindings/JSCommonJSModule.cpp b/src/jsc/bindings/JSCommonJSModule.cpp index a043ef3cd3d3..a3d891874ed0 100644 --- a/src/jsc/bindings/JSCommonJSModule.cpp +++ b/src/jsc/bindings/JSCommonJSModule.cpp @@ -1188,7 +1188,8 @@ void populateESMExports( if (auto* exports = result.getObject()) { bool hasESModuleMarker = false; - if (!ignoreESModuleAnnotation) { + // A native namespace forwarded by CJS keeps its default binding inside that namespace. + if (!ignoreESModuleAnnotation && !dynamicDowncast(exports)) { PropertySlot slot(exports, PropertySlot::InternalMethodType::VMInquiry, &vm); auto has = exports->getPropertySlot(globalObject, esModuleMarker, slot); RETURN_IF_EXCEPTION(scope, ); @@ -1213,7 +1214,7 @@ void populateESMExports( if (canPerformFastEnumeration(structure)) { exports->structure()->forEachProperty(vm, [&](const PropertyTableEntry& entry) -> bool { auto key = entry.key(); - if (key->isSymbol() || key == esModuleMarker) + if (key->isSymbol()) return true; needsToAssignDefault = needsToAssignDefault && key != vm.propertyNames->defaultKeyword; @@ -1230,7 +1231,7 @@ void populateESMExports( RETURN_IF_EXCEPTION(scope, ); for (auto property : properties) { - if (property.isEmpty() || property.isNull() || property == esModuleMarker || property.isPrivateName() || property.isSymbol()) [[unlikely]] + if (property.isEmpty() || property.isNull() || property.isPrivateName() || property.isSymbol()) [[unlikely]] continue; // ignore constructor @@ -1336,7 +1337,25 @@ void JSCommonJSModule::toSyntheticSource(JSC::JSGlobalObject* globalObject, auto result = this->exportsObject(); RETURN_IF_EXCEPTION(scope, ); - RELEASE_AND_RETURN(scope, populateESMExports(globalObject, result, exportNames, exportValues, this->ignoreESModuleAnnotation)); + populateESMExports(globalObject, result, exportNames, exportValues, this->ignoreESModuleAnnotation); + RETURN_IF_EXCEPTION(scope, ); + + auto& vm = globalObject->vm(); + const auto moduleExports = Identifier::fromString(vm, "module.exports"_s); + exportValues.append(result); + if (exportValues.hasOverflowed()) { + throwOutOfMemoryError(globalObject, scope); + return; + } + // Node's CJS namespace always exposes the original exports through this name. + for (size_t i = 0; i < exportNames.size(); ++i) { + if (exportNames[i] == moduleExports) { + exportValues[i] = exportValues.last(); + exportValues.removeLast(); + return; + } + } + exportNames.append(moduleExports); } void JSCommonJSModule::setExportsObject(JSC::JSValue exportsObject) diff --git a/src/jsc/bindings/NodeVMModule.cpp b/src/jsc/bindings/NodeVMModule.cpp index ef0440c50982..2570d1cda5f7 100644 --- a/src/jsc/bindings/NodeVMModule.cpp +++ b/src/jsc/bindings/NodeVMModule.cpp @@ -348,13 +348,8 @@ JSModuleNamespaceObject* NodeVMModule::namespaceObject(JSC::JSGlobalObject* glob object = amr->getModuleNamespace(globalObject); RETURN_IF_EXCEPTION(scope, {}); if (object) { - // The shared module namespace structure carries an __esModule accessor for - // CJS interop, but a Module Namespace Exotic Object is specified to have a - // null [[Prototype]], which is what vm modules must expose. - object->setPrototypeDirect(vm, jsNull()); namespaceObject(vm, object); } - RETURN_IF_EXCEPTION(scope, {}); return object; } diff --git a/src/jsc/bindings/ZigGlobalObject.cpp b/src/jsc/bindings/ZigGlobalObject.cpp index 2c61e485f1c8..5e47c42b2d71 100644 --- a/src/jsc/bindings/ZigGlobalObject.cpp +++ b/src/jsc/bindings/ZigGlobalObject.cpp @@ -811,9 +811,11 @@ JSC_DEFINE_HOST_FUNCTION(functionEsmNamespaceForCjs, (JSC::JSGlobalObject * glob RETURN_IF_EXCEPTION(scope, {}); if (!record) return JSValue::encode(jsUndefined()); - auto* ns = record->getModuleNamespace(globalObject, false); + auto* ns = record->getModuleNamespace(globalObject); RETURN_IF_EXCEPTION(scope, {}); - return JSValue::encode(ns); + auto* required = ns->createNamespaceForRequire(globalObject); + RETURN_IF_EXCEPTION(scope, {}); + return JSValue::encode(required); } JSC_DEFINE_HOST_FUNCTION(functionEsmRegistryHasEvaluated, (JSC::JSGlobalObject * globalObject, JSC::CallFrame* callFrame)) @@ -883,9 +885,11 @@ JSC_DEFINE_HOST_FUNCTION(functionEsmLoadSync, (JSC::JSGlobalObject * lexicalGlob if (auto* entry = loader->registryEntry(key)) { entryExistedBefore = true; if (isModuleEvaluated(entry->record()) || isModuleEvaluatingSync(entry->record())) { - auto* ns = entry->record()->getModuleNamespace(globalObject, false); + auto* ns = entry->record()->getModuleNamespace(globalObject); + RETURN_IF_EXCEPTION(scope, {}); + auto* required = ns->createNamespaceForRequire(globalObject); RETURN_IF_EXCEPTION(scope, {}); - return JSValue::encode(ns); + return JSValue::encode(required); } // Any other Evaluating record (one with top-level await) must not // reach loadModuleSync: Link() and Evaluate() reject that status. @@ -958,9 +962,11 @@ JSC_DEFINE_HOST_FUNCTION(functionEsmLoadSync, (JSC::JSGlobalObject * lexicalGlob } } - auto* ns = record->getModuleNamespace(globalObject, false); + auto* ns = record->getModuleNamespace(globalObject); + RETURN_IF_EXCEPTION(scope, {}); + auto* required = ns->createNamespaceForRequire(globalObject); RETURN_IF_EXCEPTION(scope, {}); - return JSValue::encode(ns); + return JSValue::encode(required); } #define WEBCORE_GENERATED_CONSTRUCTOR_GETTER(ConstructorName) \ @@ -1990,31 +1996,6 @@ extern "C" napi_env ZigGlobalObject__makeNapiEnvForFFI(Zig::GlobalObject* global } extern "C" JSC::EncodedJSValue CryptoObject__create(JSGlobalObject*); -JSC_DEFINE_CUSTOM_GETTER(moduleNamespacePrototypeGetESModuleMarker, (JSGlobalObject * globalObject, JSC::EncodedJSValue encodedThisValue, PropertyName)) -{ - JSValue thisValue = JSValue::decode(encodedThisValue); - JSModuleNamespaceObject* moduleNamespaceObject = dynamicDowncast(thisValue); - if (!moduleNamespaceObject || moduleNamespaceObject->m_hasESModuleMarker != WTF::TriState::True) { - return JSC::JSValue::encode(jsUndefined()); - } - - return JSC::JSValue::encode(jsBoolean(true)); -} - -JSC_DEFINE_CUSTOM_SETTER(moduleNamespacePrototypeSetESModuleMarker, (JSGlobalObject * globalObject, JSC::EncodedJSValue encodedThisValue, JSC::EncodedJSValue encodedValue, PropertyName)) -{ - auto& vm = JSC::getVM(globalObject); - JSValue thisValue = JSValue::decode(encodedThisValue); - JSModuleNamespaceObject* moduleNamespaceObject = dynamicDowncast(thisValue); - if (!moduleNamespaceObject) { - return false; - } - auto scope = DECLARE_THROW_SCOPE(vm); - JSValue value = JSValue::decode(encodedValue); - WTF::TriState triState = value.toBoolean(globalObject) ? WTF::TriState::True : WTF::TriState::False; - moduleNamespaceObject->m_hasESModuleMarker = triState; - return true; -} namespace { @@ -2347,11 +2328,6 @@ void GlobalObject::finishCreation(VM& vm) createMemoryFootprintStructure( init.vm, static_cast(init.owner))); } }, - { OBJECT_OFFSETOF(GlobalObject, m_moduleNamespaceObjectStructure), [](const LazyProperty::Initializer& init) { - JSObject* moduleNamespacePrototype = JSC::constructEmptyObject(init.vm, init.owner->nullPrototypeObjectStructure()); - moduleNamespacePrototype->putDirectCustomAccessor(init.vm, init.vm.propertyNames->__esModule, CustomGetterSetter::create(init.vm, moduleNamespacePrototypeGetESModuleMarker, moduleNamespacePrototypeSetESModuleMarker), PropertyAttribute::DontEnum | PropertyAttribute::DontDelete | PropertyAttribute::CustomAccessor | 0); - init.set(JSModuleNamespaceObject::createStructure(init.vm, init.owner, moduleNamespacePrototype)); - } }, { OBJECT_OFFSETOF(GlobalObject, m_JSBufferSubclassStructure), [](const LazyProperty::Initializer& init) { auto scope = DECLARE_TOP_EXCEPTION_SCOPE(init.vm); auto* globalObject = static_cast(init.owner); @@ -2684,8 +2660,6 @@ void GlobalObject::finishCreation(VM& vm) dynamicDowncast(init.owner))); }); - // Change prototype from null to object for synthetic modules. - m_vmModuleContextMap.initLater( [](const Initializer& init) { init.set(JSWeakMap::create(init.vm, init.owner->weakMapStructure())); diff --git a/test/js/bun/import-attributes/import-attributes.test.ts b/test/js/bun/import-attributes/import-attributes.test.ts index 2c1888662f74..00eef072d557 100644 --- a/test/js/bun/import-attributes/import-attributes.test.ts +++ b/test/js/bun/import-attributes/import-attributes.test.ts @@ -344,11 +344,11 @@ describe("other loaders do not crash", () => { }); describe("?raw", () => { - for (const [name, fn] of [ - ["bun run", testBunRun], + for (const [name, fn, expected] of [ + ["bun run", testBunRun, {}], // ["bun build", testBunBuild], // TODO: bun.build doesn't support query params at all yet - ["bun run await import", testBunRunAwaitImport], - ["require", testBunRunRequire], + ["bun run await import", testBunRunAwaitImport, {}], + ["require", testBunRunRequire, { __esModule: true }], // ["bun build require", testBunBuildRequire], // TODO: bun.build doesn't support query params at all yet ] as const) { test(name, async () => { @@ -357,7 +357,7 @@ describe("?raw", () => { await using question_raw = tempDir("import-attributes", { [filename]: code, }); - expect(await fn(question_raw, null, filename + "?raw")).toEqual({ default: code }); + expect(await fn(question_raw, null, filename + "?raw")).toEqual({ ...expected, default: code }); }); } }); diff --git a/test/js/bun/namespace-prototype-pollution.test.ts b/test/js/bun/namespace-prototype-pollution.test.ts index 36c25f43652a..bb49b55c983e 100644 --- a/test/js/bun/namespace-prototype-pollution.test.ts +++ b/test/js/bun/namespace-prototype-pollution.test.ts @@ -19,8 +19,9 @@ test("namespace imports should not inherit from Object.prototype", async () => { console.log("PASS: prototype pollution prevented"); } - // Verify __esModule still works - console.log("__esModule settable:", (mod.__esModule = true, mod.__esModule === true)); + console.log("Null prototype:", Object.getPrototypeOf(mod) === null); + console.log("__esModule absent:", !("__esModule" in mod)); + console.log("__esModule immutable:", !Reflect.set(mod, "__esModule", true)); // Original exports should work console.log("Original export:", mod.value); @@ -38,6 +39,8 @@ test("namespace imports should not inherit from Object.prototype", async () => { expect(exitCode).toBe(0); expect(stdout).toContain("PASS: prototype pollution prevented"); - expect(stdout).toContain("__esModule settable: true"); + expect(stdout).toContain("Null prototype: true"); + expect(stdout).toContain("__esModule absent: true"); + expect(stdout).toContain("__esModule immutable: true"); expect(stdout).toContain("Original export: original"); }); diff --git a/test/js/bun/resolve/esModule-annotation.test.js b/test/js/bun/resolve/esModule-annotation.test.js index 01c5fb3b40a8..c4bfe3c993b2 100644 --- a/test/js/bun/resolve/esModule-annotation.test.js +++ b/test/js/bun/resolve/esModule-annotation.test.js @@ -21,20 +21,20 @@ describe('without type: "module"', () => { __esModule: true, }); - // The module namespace object will not have the __esModule property. - expect(WithoutTypeModuleExportEsModuleAnnotationNoDefault.__esModule).toBeUndefined(); + expect(Object.hasOwn(WithoutTypeModuleExportEsModuleAnnotationNoDefault, "__esModule")).toBe(true); + expect(WithoutTypeModuleExportEsModuleAnnotationNoDefault.__esModule).toBeTrue(); }); test("exports.default = true; exports.__esModule = true;", () => { expect(WithoutTypeModuleExportEsModuleAnnotation.default).toBeTrue(); - expect(WithoutTypeModuleExportEsModuleAnnotation.__esModule).toBeUndefined(); + expect(WithoutTypeModuleExportEsModuleAnnotation.__esModule).toBeTrue(); }); test("exports.default = true;", () => { expect(WithoutTypeModuleExportEsModuleNoAnnotation.default).toEqual({ default: true, }); - expect(WithoutTypeModuleExportEsModuleAnnotation.__esModule).toBeUndefined(); + expect(WithoutTypeModuleExportEsModuleNoAnnotation.__esModule).toBeUndefined(); }); }); @@ -65,7 +65,7 @@ describe('with type: "module"', () => { expect(WithTypeModuleExportEsModuleNoAnnotation.default).toEqual({ default: true, }); - expect(WithTypeModuleExportEsModuleAnnotation.__esModule).toBeTrue(); + expect(WithTypeModuleExportEsModuleNoAnnotation.__esModule).toBeUndefined(); }); }); diff --git a/test/js/bun/resolve/esModule.test.ts b/test/js/bun/resolve/esModule.test.ts index 1d66d9eded4c..0f3bc3cbf891 100644 --- a/test/js/bun/resolve/esModule.test.ts +++ b/test/js/bun/resolve/esModule.test.ts @@ -1,31 +1,149 @@ import { expect, test } from "bun:test"; - -// Top-level `await import(self)` is a spec-level deadlock under the new -// pure-C++ module loader (Node prints an "unsettled top-level await" warning -// and exits). A static self-import yields the same namespace object without -// blocking evaluation on itself. +import { bunEnv, bunExe, tempDir } from "harness"; import * as Self from "./esModule.test.ts"; -test("__esModule defaults to undefined", () => { +test("an imported namespace has no inherited or writable __esModule marker", () => { + expect(Object.getPrototypeOf(Self)).toBeNull(); + expect("__esModule" in Self).toBe(false); + expect(Object.hasOwn(Self, "__esModule")).toBe(false); + expect(Object.getOwnPropertyDescriptor(Self, "__esModule")).toBeUndefined(); + expect(Reflect.ownKeys(Self)).toEqual([Symbol.toStringTag]); + expect(Reflect.set(Self, "__esModule", true)).toBe(false); + expect(Reflect.defineProperty(Self, "__esModule", { value: true })).toBe(false); + expect(require("./esModule.test.ts")).toBe(Self); expect(Self.__esModule).toBeUndefined(); }); -test("__esModule is settable", () => { - Self.__esModule = true; - expect(Self.__esModule).toBe(true); - Self.__esModule = false; - expect(Self.__esModule).toBe(undefined); - Self.__esModule = true; - expect(Self.__esModule).toBe(true); - Self.__esModule = undefined; -}); - -test("require of self sets __esModule", () => { - expect(Self.__esModule).toBeUndefined(); - { - const Self = require("./esModule.test.ts"); - expect(Self.__esModule).toBe(true); +for (const order of ["import-first", "require-first", "cache-first"]) { + test.concurrent(`require(esm) marker reflection and live bindings (${order})`, async () => { + using dir = tempDir("namespace-marker", { + "default.mjs": "export default {}; export let value = 1; export function bump() { value++; }", + "none.mjs": "export const value = 1;", + "false.mjs": "export const __esModule = false; export default 1;", + "undefined.mjs": "export const __esModule = undefined; export default 1;", + "true.mjs": "export const __esModule = true; export default 1;", + "sorted.mjs": "export default 1; export const Alpha = 1, _a = 2, zzz = 3;", + "leaf.mjs": "export const value = 'synthetic-ok';", + "alias.mjs": "import * as z from './leaf.mjs'; export * from './leaf.mjs'; export { z, z as default };", + "entry.mjs": ` +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +const require = createRequire(import.meta.url); +function marker(ns, present, value) { + assert.equal(Object.getPrototypeOf(ns), null); + assert.equal(Object.isExtensible(ns), false); + assert.equal('__esModule' in ns, present); + assert.equal(Object.hasOwn(ns, '__esModule'), present); + assert.equal(Object.prototype.hasOwnProperty.call(ns, '__esModule'), present); + assert.deepEqual(Object.getOwnPropertyDescriptor(ns, '__esModule'), present ? { + value, writable: true, enumerable: true, configurable: false, + } : undefined); + assert.equal(Reflect.set(ns, '__esModule', true), false); + assert.equal(Reflect.set(ns, '__esModule', value), false); + assert.equal(Reflect.defineProperty(ns, '__esModule', { value }), present); + for (const descriptor of [{ value: {} }, { writable: false }, { enumerable: false }, { configurable: true }, { get() {} }]) + assert.equal(Reflect.defineProperty(ns, '__esModule', descriptor), false); + assert.equal(Reflect.deleteProperty(ns, '__esModule'), !present); +} +for (const [name, hasDefault, explicit, value] of [ + ['default', true, false, undefined], ['none', false, false, undefined], + ['false', true, true, false], ['undefined', true, true, undefined], ['true', true, true, true], +]) { + let imported, required; + if (${JSON.stringify(order)} !== 'require-first') { + imported = await import('./' + name + '.mjs'); + marker(imported, explicit, value); + if (${JSON.stringify(order)} === 'cache-first') void require.cache[require.resolve('./' + name + '.mjs')]; + required = require('./' + name + '.mjs'); + } else { + required = require('./' + name + '.mjs'); + imported = await import('./' + name + '.mjs'); + } + marker(imported, explicit, value); + marker(required, explicit || hasDefault, explicit ? value : hasDefault ? true : undefined); + assert.equal(imported === required, explicit || !hasDefault); + assert.equal(require('./' + name + '.mjs'), required); + delete require.cache[require.resolve('./' + name + '.mjs')]; + const reloaded = require('./' + name + '.mjs'); + // Bun also evicts the ESM registry on cache deletion; only check the added facade's identity. + if (hasDefault && !explicit) assert.notEqual(reloaded, required); + marker(reloaded, explicit || hasDefault, explicit ? value : hasDefault ? true : undefined); + if (name === 'default') { + imported.bump(); + assert.equal(required.value, 2); + assert.equal(({ ...required }).__esModule, true); + assert.equal(Object.assign({}, required).__esModule, true); + assert.equal(JSON.parse(JSON.stringify(required)).__esModule, true); + globalThis.Bun?.gc(true); + assert.equal(required.value, imported.value); } - expect(Self.__esModule).toBe(true); - expect(Object.getOwnPropertyNames(Self)).toBeEmpty(); +} +const sorted = require('./sorted.mjs'); +assert.deepEqual(Reflect.ownKeys(sorted), ['Alpha', '__esModule', '_a', 'default', 'zzz', Symbol.toStringTag]); +assert.deepEqual(Object.keys(sorted), ['Alpha', '__esModule', '_a', 'default', 'zzz']); +const alias = await import('./alias.mjs'); +const interop = alias.default && typeof alias.default === 'object' && '__esModule' in alias.default ? alias.default : alias; +assert.equal(interop.z.value, 'synthetic-ok'); +assert.equal(require('./alias.mjs').__esModule, true); +assert.equal('__esModule' in alias, false); +console.log('ok'); +`, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "entry.mjs"], + cwd: String(dir), + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr, exitCode }).toEqual({ stdout: "ok\n", stderr: "", exitCode: 0 }); + }); +} + +test.concurrent("CommonJS-as-ESM namespaces preserve explicit markers and own-key reflection", async () => { + using dir = tempDir("cjs-namespace-marker", { + "plain.cjs": "exports.value = 1;", + "true.cjs": "Object.defineProperty(exports, '__esModule', { value: true }); exports.value = 1;", + "false.cjs": "exports.__esModule = false; exports.value = 1;", + "undefined.cjs": "exports.__esModule = undefined; exports.value = 1;", + "collision.cjs": "exports['module.exports'] = 3; exports.value = 1;", + "forwarded.mjs": "export default 'kept'; export const value = 1;", + "forwarder.cjs": "module.exports = require('./forwarded.mjs');", + "entry.mjs": ` +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +const require = createRequire(import.meta.url); +for (const [name, present, value] of [ + ['plain', false], ['true', true, true], ['false', true, false], ['undefined', true, undefined], ['collision', false], +]) { + const ns = await import('./' + name + '.cjs'); + assert.equal(Object.getPrototypeOf(ns), null); + assert.equal(Object.isExtensible(ns), false); + assert.equal('__esModule' in ns, present); + assert.equal(Object.hasOwn(ns, '__esModule'), present); + assert.equal(Object.prototype.hasOwnProperty.call(ns, '__esModule'), present); + assert.deepEqual(Object.getOwnPropertyDescriptor(ns, '__esModule'), present ? { + value, writable: true, enumerable: true, configurable: false, + } : undefined); + assert.deepEqual(Reflect.ownKeys(ns), [...(present ? ['__esModule'] : []), 'default', 'module.exports', 'value', Symbol.toStringTag]); + assert.equal(ns['module.exports'], require('./' + name + '.cjs')); + assert.equal(Reflect.set(ns, '__esModule', true), false); +} +const forwarded = await import('./forwarder.cjs'); +assert.equal(forwarded.default, require('./forwarder.cjs')); +assert.equal(forwarded.default.default, 'kept'); +assert.equal(forwarded['module.exports'], forwarded.default); +console.log('ok'); +`, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "entry.mjs"], + cwd: String(dir), + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr, exitCode }).toEqual({ stdout: "ok\n", stderr: "", exitCode: 0 }); }); diff --git a/test/js/bun/resolve/require-esm-evaluating-cycle.test.ts b/test/js/bun/resolve/require-esm-evaluating-cycle.test.ts index 4002df11c3ce..ec50c1d7f70a 100644 --- a/test/js/bun/resolve/require-esm-evaluating-cycle.test.ts +++ b/test/js/bun/resolve/require-esm-evaluating-cycle.test.ts @@ -44,9 +44,10 @@ test("require(esm) inside a require cycle returns the live namespace, not an emp expect(exitCode).toBe(0); }); -// The CommonJS side reads `__esModule` / `"module.exports"` off that live -// namespace; when the module happens to export a binding by that name that is -// still in TDZ, require() itself must not throw (reading it afterwards does). +// The CommonJS side reads `"module.exports"` off that live namespace. Marker +// selection checks export names without reading the bindings. When either +// binding is still in TDZ, require() itself must not throw +// (reading it afterwards does). test("require(esm) inside a require cycle tolerates a TDZ `__esModule` export", async () => { using dir = tempDir("require-esm-evaluating-cycle-tdz", { "a.mjs": ` diff --git a/test/js/node/module/require-extensions.test.ts b/test/js/node/module/require-extensions.test.ts index 86dfa1d98429..70f3514d979e 100644 --- a/test/js/node/module/require-extensions.test.ts +++ b/test/js/node/module/require-extensions.test.ts @@ -156,7 +156,7 @@ test("wrapping an existing extension but it's secretly sync esm", () => { original(module, filename); })); const mod = require("./extensions-fixture/secretly_esm"); - expect(mod).toEqual({ default: 1 }); + expect({ ...mod }).toEqual({ __esModule: true, default: 1 }); expect(mocked).toBeCalled(); } finally { require.extensions[".cjs"] = original; From 951a8b593af7d01d97ba838ba88e7ce2882e6bdb Mon Sep 17 00:00:00 2001 From: robobun Date: Sun, 4 Oct 2026 23:07:28 -0700 Subject: [PATCH 12/15] sql(postgres): keep a failed query in flight until its ReadyForQuery, and write nothing past a statement being parsed (#44235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Problem - `Bun.SQL` (postgres): after a pipelined query fails, two queries issued together on that connection resolve with each other's rows. Equal column types: no error. Different column types: `ERR_POSTGRES_UNSUPPORTED_INTEGER_SIZE` or wrong values. - The `ErrorResponse` arm of `PostgresSQLConnection::on` released the failed request before its `ReadyForQuery`. A statement issued in that gap wrote its Parse at once. The late `ReadyForQuery` then made `advance()` write the next request's Bind first. - Behind pgdog 0.1.60, a second `ReadyForQuery` for one Sync does the same on every run. ### Fix - The `ErrorResponse` arm rejects through `reject_in_flight` (`on_undecodable_row` of #43187, renamed). The promise rejects at once. The request keeps its counter and the head of the queue until its `ReadyForQuery`. - `advance()` stops at a request whose statement is being parsed, as in #20986, and a debug assertion states that rule. A reply carries no request id: it goes to the head of the queue. - Verified: `test/js/sql/postgres-prepared-pipeline-reorder.test.ts` (24 new tests, 19 fail on main), `sql.test.ts`, every `test/js/sql/postgres-*.test.ts`. - Self-reviewed: 7 concerns, 6 addressed, 1 in part (Notes). ### Background - Bun writes several prepared queries before the first reply. The server answers each Sync with one `ReadyForQuery`, also after an error. - `advance()` walks the request queue. It pops finished requests and writes queued ones. - Considered a count of Syncs: two counter updates per query, no result changes. Considered only the clear of `IS_READY_FOR_QUERY` from #35120: the step-over in `advance()` stays. ### Downsides - A new statement or simple query issued between an `ErrorResponse` and its `ReadyForQuery` waits for that `ReadyForQuery`. - Release binary: `.text` +256 B.
Notes **Reproduction.** One connection (`max: 1`, also `sql.reserve()` and a pool connection that carries more than one query), a real PostgreSQL, default options: ```js const sql = new SQL({ url, max: 1 }); const ofBob = () => sql`select owner, secret from swap_b where id = ${1}`; const check = n => sql`select 10 / ${n}::int as q`; await ofBob(); await check(1); await Promise.all([ofBob(), check(0)]).catch(() => {}); const [a, b] = await Promise.all([sql`select owner, secret from swap_a where id = ${1}`, ofBob()]); // a is the row of swap_b, b is the row of swap_a ``` The `ErrorResponse` of `check(0)` and its `ReadyForQuery` must arrive in two reads. On loopback that happens in 35 to 86 of 100 rounds. A proxy that forwards every byte and holds the `ReadyForQuery` until the two reads are issued makes it 50 of 50. The tests use that proxy (`pgHoldingProxy` in `test/js/sql/wire-frames.ts`), with no timer. Frames after the rejection, main: `Parse(swap_a) Sync`, the held `ReadyForQuery`, `Bind(swap_b) Sync`, `Bind(swap_a) Sync`. The queue order is swap_a, swap_b. This branch: the held `ReadyForQuery`, `Parse(swap_a) Sync`, `Bind(swap_a) Sync`, `Bind(swap_b) Sync`. The queries in the gap do not have to come from the rejection handler. Issued by a timer that was armed before the failing query, through the same proxy, they are wrong in 10 of 10 rounds on 1.4.2 and in 0 of 10 here. **pgdog.** pgdog 0.1.60 with its query parser on (one primary and one replica entry for the same PostgreSQL 17) rejects a statement with a syntax error itself. It answers the batch `Parse Describe Bind Execute Flush Sync` with `ErrorResponse ReadyForQuery`, and then with one more `ReadyForQuery`. The second one arrives when the Parse of the next request is on the wire. ```js const bad = sql`selec 2 as x`; // rejected by pgdog const a = sql`select 'alice' as owner, ${1}::int as id`; // new to the connection const b = sql`select 'bob' as owner, ${1}::int as id`; // prepared await Promise.allSettled([bad, a, b]); ``` Official 1.4.2: `a` resolves with bob and `b` with alice in 50 of 50 rounds, with no proxy and no mock. This branch: 0 of 50. After the two `ReadyForQuery`, 1.4.2 writes `Parse(alice) Describe Sync`, `Bind(bob)`, and later `Bind(alice)`. This branch writes `Parse(alice) Describe Sync`, and after its reply `Bind(alice)`, `Bind(bob)`. The stop in `advance()` is the half that fixes this case. **Why the flag was wrong.** Every `ReadyForQuery` sets `IS_READY_FOR_QUERY`, whichever Sync it answers. With two pipelined queries in flight, the first `ReadyForQuery` sets it while the second Sync is not answered. `pipelined_requests > 0` is the guard for that state. The `ErrorResponse` arm called `finish_request`, so the count went to 0 with a `ReadyForQuery` owed. A query that fails alone twice in a row reaches the same state: the second one is written behind the first in the gap of the first. **Tests.** `postgres-prepared-pipeline-reorder.test.ts` has 28 tests, 24 of them new. - Real server through the proxy, 17 tests that fail on main: 8 shapes of queries issued in the gap (a new statement and a prepared one, with equal and with different column types, through `execute()`, with a simple query, two new statements, three queries), each after a query that failed behind another one and after a query that failed alone twice in a row. One more on a connection reserved from a pool of 10. The shape with two new statements does not settle on main. - Real server, 5 tests that pass on main too: a transaction that begins in the gap, and 4 other kinds of failing query (a simple query, a Parse that fails with and without a parameter, a first execution without parameters). They pin that each kind leaves the queue at its `ReadyForQuery` and that the connection continues. - Mock, 2 tests that fail on main. PostgreSQL sends nothing but the `ReadyForQuery` after an `ErrorResponse`, and one `ReadyForQuery` for each Sync. With such a server, each half of the fix alone gives every query its own rows. So each half has a test with a peer that sends more. - The hold: the mock sends a row after the `ErrorResponse`. With only the hold reverted, 1 of 28 fails: the row goes to the query issued in the gap. No real peer that does this is known. - The stop: the mock replays the frames of pgdog. With only the stop reverted, 1 of 28 fails, and a debug build stops at the assertion `advance() passed a request that is not written yet`. **Measurements.** Base is main `a4f1429148`. PostgreSQL 17.11 on loopback, linux x64. - Edited arms on the success path: 0 executions in 1,703 fulfilled queries over 7 workloads (gdb counting breakpoints, debug build. Base 0 in 1,703). The calls of `advance()` from the `ReadyForQuery` arm and the Binds written at enqueue are equal on base and branch for the 6 workloads with a fixed order. The workload with a pool of 10 differs from run to run on both. - Steady state, 200 queries in bursts of 10: socket writes 20 (base 20), `advance()` entries 200 (base 200), reads 23 to 25 (base 23). - Instructions for each pipelined query: not measured. `perf` and `valgrind` are not in the container and have no package there. In place of that, gdb `stepi` on the release build with symbols, 20 calls in the middle of a burst: `advance()` 108 instructions for each call (base 108), `finish_request` 17 (base 17). - Release binary: `.text` +256 B. `advance()` +357 B, `on_result` +29 B, the two instances of `on()` -22 B each (`size`, `nm -S`). The debug assertion and the comments came after this measurement. They change no byte: `.text` and these symbol sizes are equal at `28aeaddf7` and `21c547a52`. - `size_of` `PostgresSQLQuery` 64 B (was 64), `PostgresSQLConnection` 672 B (was 672), `Flags` 7 B (was 7). - For each failing query: `finish_request` at the `ErrorResponse` 0 (was 1), at the `ReadyForQuery` 1 (was 0). The shape of the report: `advance()` entries 15 (was 15), socket writes 11 (was 12). - Counters when the connection closes, after 7 workloads with failing queries (50 connections): `pipelined_requests` 0, `nonpipelinable_requests` 0, `pending_requests` 0. - 12 shapes of queries in the gap, with the wire recorded: 0 wrong (was 6). Shapes that write before the held `ReadyForQuery`: 2 (was 12). The 2 are a prepared execute, which is still pipelined behind the failed request. After a failed simple query a prepared execute is written after the `ReadyForQuery` (was before). - The script of the report, release build: 0 of 100 rounds wrong in 3 runs (base 35, 37, 39. Official 1.4.2: 86, 84, 62). With the proxy: 0 of 50 (base 50, 1.4.2 50). - `close()` between the `ErrorResponse` and the held `ReadyForQuery`, 5 kinds of failing query and 6 ways to close: all 30 settle the same on 1.4.2 and here. A FATAL `ErrorResponse`, which the server follows with a close and no `ReadyForQuery`, also settles the same. **Existing suites**, debug build, local PostgreSQL: every `test/js/sql/postgres-*.test.ts`, `sql-prepare-false`, `sql-pool-transaction-isolation`, `sql-reserve-abort`, `sql-onconnect-onclose-throw`, `sql-close-pending-connection`, `wire-frames`. `postgres-string-leak` exceeds the 5 s timeout under the debug build, on base too. In the last run the container was under load, and subprocess tests in 4 other files passed 5 s. Each of those that I ran again alone passed. `sql.test.ts` against the local server: 18 to 20 of 854 fail, base 23. Each case that fails on the branch fails on base too. They need roles and an encoding that the local server does not have, or they depend on time under the debug build. The case for #22395 passes. No run stopped at the new assertion. **A request that fails in its Parse** stays `Pending` with its place in `pending_requests`. The `discard_response` branch of the `ReadyForQuery` arm calls `finish_request`, whose `Pending` arm releases that place. `finish_request` has no edit. **Covered by reading only.** The failure path of `on_result` needs a command tag that fails to convert. It now keeps the request in flight when the `ReadyForQuery` is still to come. Before, the counter of that request was never released. **Not changed, named for later.** - A peer that sends a `ReadyForQuery` that answers no Sync releases a held request one message early. The rows stay in order (the stop). - The setter of `IS_READY_FOR_QUERY` and the early clear of `WAITING_TO_PREPARE` at `ParseComplete` and `ParameterDescription`. The held counters are the guard. A count of Syncs is the follow-up if the flag must be exact. It needs a rule for a `ReadyForQuery` that answers no Sync first: pgdog sends one, and 6 cases of `postgres-infinity-date.test.ts` answer one Sync with two. - Parse+Describe+Sync of a named statement and the first execution without parameters hold no counter. The flag pair guards them. - The arm of `advance()` for a request with no statement. It cannot be reached: `do_run` sets the statement before each push. - The MySQL adapter releases a request at a row it cannot decode, in the middle of its response (`JSMySQLConnection.rs`). #43323 is open for that. **Open pull requests on the same code.** - #35120 (re-prepare of invalidated statements) clears `IS_READY_FOR_QUERY` in the `ErrorResponse` arm for every error. That line alone closes the gap for a server that sends one `ReadyForQuery` for each Sync. It leaves the early release and the step-over. By reading, the pgdog case stays with that line alone. The two changes merge with no conflict. The retry branch of #35120 releases and queues the request again at the `ErrorResponse`, so it needs the same hold after a rebase. - #33370 (`Query.cancel()`) releases a `Pending` request whose Parse is on the wire and marks it `Fail`. That is the same early release from a second place. - #43918 rewrites `encode_request`. This change does not touch it. **Self-review.** The concerns taken up: 1. The first text said that no open pull request covers this gap. #35120 covers a part of it. The text above now says so. 2. #33370 was not named. It is named above. 3. The sites of the same class that stay were not listed. They are listed above. 4. Each half was pinned only by a mock of a peer that does not exist. Addressed in part: the stop test now replays the frames of pgdog, and the run against the real pgdog is above. The hold test is still a mock. 5. `src/` had comments of more than one line. Each is one line now. 6. The reserved case ran on a pool of 1. It now reserves from a pool of 10: 10 sockets open and one carries the queries. 7. Nothing stated the write order in a debug build. `advance()` now asserts that it passes no request that is not written yet. **Credit.** The hold is the mechanism of #43187. The stop is the arm of #20986.
--- **[human-review]** gate passed · iteration 0 · 4 files touched
fails on main (without fix) ```console ASAN without fix: 19 FAILED $ BUN_DEBUG_QUIET_LOGS=1 bun scripts/build.ts --profile=debug --quiet test "--reporter=junit" "--reporter-outfile=/tmp/pr_gate.xml" test/js/sql/postgres-prepared-pipeline-reorder.test.ts bun test v1.4.3 (367d939d9) test/js/sql/postgres-prepared-pipeline-reorder.test.ts: Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 (pass) postgres > a prepared-statement execute does not jump a queued unwritten prepare [950.35ms] (pass) postgres > prepared executes do not jump multiple queued unwritten prepares [371.61ms] (pass) postgres > mixed burst of prepared and new statements returns every row in order [369.38ms] (pass) postgres > a prepared-statement execute does not jump a queued unwritten simple query [128.05ms] Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 204 | return `{${values.map(arrayValueSerializer.bind(this, type, isPostgresNumericType(type), isPostgresJsonType(type))).join(delimiter)}}`; 205 | } 206 | function wrapPostgresError(error) { 207 | if (Error.isError(error)) { 208 | return error; 209 | return new PostgresError(error.message, error); ^ PostgresError: Failed to read data hint: "The quer ... (truncated) release without fix: all passed bun test v1.4.3-canary.1 (21c547a52) test/js/sql/postgres-prepared-pipeline-reorder.test.ts: Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 (pass) postgres > a prepared-statement execute does not jump a queued unwritten prepare [54.98ms] (pass) postgres > prepared executes do not jump multiple queued unwritten prepares [48.48ms] (pass) postgres > mixed burst of prepared and new statements returns every row in order [24.94ms] (pass) postgres > a prepared-statement execute does not jump a queued unwritten simple query [12.04ms] Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 (pass) postgres > before the ReadyForQuery of a query that failed behind another query > a new statement with other column types, a prepared one resolve with their own rows [131.62ms] (pass) postgres > before the ReadyForQuery of a query that failed behind another query > a new statement, a prepared one, through execute() resolve with their own rows [141.14ms] (pass) postgres > before the ReadyForQuery of a query that failed behind another query > a new statement, a simple query resolve with their own rows [142.52ms] (pass) postgres > before the ReadyFor ... (truncated) ```
passes on PR (with fix) ```console ASAN with fix: all passed $ BUN_DEBUG_QUIET_LOGS=1 bun scripts/build.ts --profile=debug --quiet test "--reporter=junit" "--reporter-outfile=/tmp/pr_gate.xml" test/js/sql/postgres-prepared-pipeline-reorder.test.ts bun test v1.4.3 (367d939d9) test/js/sql/postgres-prepared-pipeline-reorder.test.ts: Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 (pass) postgres > a prepared-statement execute does not jump a queued unwritten prepare [609.58ms] (pass) postgres > prepared executes do not jump multiple queued unwritten prepares [492.29ms] (pass) postgres > mixed burst of prepared and new statements returns every row in order [253.06ms] (pass) postgres > a prepared-statement execute does not jump a queued unwritten simple query [157.56ms] Container ready via docker-compose: postgres_plain at 127.0.0.1:5432 (pass) postgres > before the ReadyForQuery of a query that failed behind another query > a new statement, a prepared one resolve with their own rows [1856.81ms] (pass) postgres > before the ReadyForQuery of a query that failed behind another query > a new statement with other column types, a prepared one resolve with their own rows [1664.26ms] (pass) postgres > before the Rea ... (truncated) release with fix: all passed $ bun scripts/build.ts --profile=release [configured] bun-profile → bun (stripped) in 7581ms (unchanged) ninja: Entering directory `/workspace/bun/build/release' [1/186] gen cpp.rs (cppbind) [2/186] gen generated_host_exports.rs generated_host_exports.rs: 121 exports (host=5, lazy=10, generic=106, rust=0); 248 extern-C blocks audited [3/186] gen JS modules (bundle-modules) Preprocess modules (11769ms) Bundle modules (1818ms) Postprocesss modules (245ms) Bundle Functions (1012ms) Generate Code (63ms) [14.93s] Bundled "src/js" for production 2613 kb 198 internal modules 13 native modules 50 internal functions across 16 files [4/74] cxx obj/unified/UnifiedSource-src_jsc_bindings-1.cpp.o [5/74] cxx obj/unified/UnifiedSource-src_jsc_bindings-3.cpp.o [6/74] cxx obj/unified/UnifiedSource-src_jsc_bindings-0.cpp.o [7/74] cxx obj/unified/UnifiedSource-src_jsc_bindings-5.cpp.o [8/74] cxx obj/unified/UnifiedSource-src_jsc_bindings-4.cpp.o [9/74] cxx obj/src/jsc/bindings/bindings.cpp.o [10/74] cc obj/packages/bun-usockets/src/loop.c.o [11/74] cxx obj/unified/UnifiedSource-src_jsc_bindings_node-0.cpp.o [12/74] cc obj/packages/bun-usockets/src/bsd.c.o [13/74] cxx obj/un ... (truncated) ```
diff hotspot ``` src/sql_jsc/postgres/PostgresSQLConnection.rs | 17 +- src/sql_jsc/postgres/PostgresSQLQuery.rs | 16 +- .../sql/postgres-prepared-pipeline-reorder.test.ts | 278 ++++++++++++++++++++- test/js/sql/wire-frames.ts | 56 +++++ 4 files changed, 355 insertions(+), 12 deletions(-) ```
**gate history** · 1 passed · 0 rejected · iteration 0
evidence per changed file ``` file reads edits tests src/sql_jsc/postgres/PostgresSQLConnection.rs 5 0 39 src/sql_jsc/postgres/PostgresSQLQuery.rs 3 0 39 test/js/sql/postgres-prepared-pipeline-reorder.test.ts 2 0 38 test/js/sql/wire-frames.ts 2 2 40 ```
--- src/sql_jsc/postgres/PostgresSQLConnection.rs | 17 +- src/sql_jsc/postgres/PostgresSQLQuery.rs | 16 +- ...postgres-prepared-pipeline-reorder.test.ts | 278 +++++++++++++++++- test/js/sql/wire-frames.ts | 56 ++++ 4 files changed, 355 insertions(+), 12 deletions(-) diff --git a/src/sql_jsc/postgres/PostgresSQLConnection.rs b/src/sql_jsc/postgres/PostgresSQLConnection.rs index d6b87a535944..8191ee47866c 100644 --- a/src/sql_jsc/postgres/PostgresSQLConnection.rs +++ b/src/sql_jsc/postgres/PostgresSQLConnection.rs @@ -1893,6 +1893,11 @@ impl PostgresSQLConnection { let req = ParentRef::from(self.requests.get()[offset].as_non_null()); match req.status.get() { QueryStatus::Pending => { + debug_assert!( + offset == 0 + || self.requests.get()[offset - 1].status.get() != QueryStatus::Pending, + "advance() passed a request that is not written yet" + ); // Optimistically account for this request leaving Pending; the // few paths below that keep it Pending (can't execute yet / // Parse written but not Bind / statement still Parsing) undo @@ -2250,10 +2255,9 @@ impl PostgresSQLConnection { return; } StatementStatus::Parsing => { - // we are still parsing, lets wait for it to be prepared or failed + // Replies go to the requests in queue order: write nothing past this one. self.note_request_pending(); - offset += 1; - continue; + break; } } } else { @@ -2431,7 +2435,7 @@ impl PostgresSQLConnection { return Err(err); } let js_err = self.undecodable_row_error(err)?; - request.on_undecodable_row(js_err, self.global()); + request.reject_in_flight(js_err, self.global()); return Ok(()); } @@ -2455,7 +2459,7 @@ impl PostgresSQLConnection { Ok(result) => result, Err(err) => { let js_err = self.undecodable_row_error(err)?; - request.on_undecodable_row(js_err, self.global()); + request.reject_in_flight(js_err, self.global()); return Ok(()); } }; @@ -2995,9 +2999,8 @@ impl PostgresSQLConnection { } // If `err` was not moved into stmt above, it drops here automatically. - self.finish_request(&request); self.update_ref(); - request.on_js_error(js_err, self.global()); + request.reject_in_flight(js_err, self.global()); } MessageType::PortalSuspended => { reader.skip_message()?; diff --git a/src/sql_jsc/postgres/PostgresSQLQuery.rs b/src/sql_jsc/postgres/PostgresSQLQuery.rs index 31590b249bab..e62470a0c244 100644 --- a/src/sql_jsc/postgres/PostgresSQLQuery.rs +++ b/src/sql_jsc/postgres/PostgresSQLQuery.rs @@ -78,7 +78,7 @@ pub struct Flags { pub(crate) binary: bool, pub(crate) bigint: bool, pub(crate) simple: bool, - /// Rejected for an undecodable row: in flight, its response skipped, until `ReadyForQuery`. + /// Rejected while the server still answers it: its response is skipped until `ReadyForQuery`. pub(crate) discard_response: bool, /// Which connection counter this request's dispatch incremented; reset to /// `None` when `finish_request` consumes that contribution, so the @@ -219,13 +219,14 @@ impl PostgresSQLQuery { ); } + /// For a request that is owed no `ReadyForQuery`: `Fail` lets `advance()` drop it at once. pub(crate) fn on_js_error(&self, err: JSValue, global_object: &JSGlobalObject) { self.status.set(Status::Fail); self.reject(err, global_object); } - /// Rejects now, but `status` stays in flight: the server is still answering this query. - pub(crate) fn on_undecodable_row(&self, err: JSValue, global_object: &JSGlobalObject) { + /// Rejects now. `status`, the counter and the queue head stay until its `ReadyForQuery`. + pub(crate) fn reject_in_flight(&self, err: JSValue, global_object: &JSGlobalObject) { self.update_flags(|f| f.discard_response = true); self.reject(err, global_object); } @@ -302,7 +303,14 @@ impl PostgresSQLQuery { let tag = CommandTag::init(command_tag_str); let js_tag: JSValue = match tag.to_js_tag(global_object) { Ok(v) => v, - Err(e) => return self.on_js_error(global_object.take_exception(e), global_object), + Err(e) => { + let err = global_object.take_exception(e); + return if is_last { + self.on_js_error(err, global_object) + } else { + self.reject_in_flight(err, global_object) + }; + } }; js_tag.ensure_still_alive(); diff --git a/test/js/sql/postgres-prepared-pipeline-reorder.test.ts b/test/js/sql/postgres-prepared-pipeline-reorder.test.ts index 0f2b89f8a5d1..3c82e733d6a8 100644 --- a/test/js/sql/postgres-prepared-pipeline-reorder.test.ts +++ b/test/js/sql/postgres-prepared-pipeline-reorder.test.ts @@ -9,8 +9,22 @@ // query's rows (decoded under an empty column list), and the next query never // settles, wedging the connection. import { SQL } from "bun"; -import { expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; import { describeWithContainer } from "harness"; +import { + pgBindComplete, + pgBindParameters, + pgCommandComplete, + pgDataRow, + pgErrorResponse, + pgHold, + pgHoldingProxy, + pgMockServer, + pgParameterDescription, + pgParseComplete, + pgReadyForQuery, + pgRowDescription, +} from "./wire-frames"; describeWithContainer("postgres", { image: "postgres_plain" }, container => { const url = () => `postgres://bun_sql_test@${container.host}:${container.port}/bun_sql_test`; @@ -98,3 +112,265 @@ describeWithContainer("postgres", { image: "postgres_plain" }, container => { expect(await Promise.all([a, b, c])).toEqual([[{ v: "A" }], [{ v: "B" }], [{ v: "C" }]]); }); }); + +// A query that fails is answered with ErrorResponse, then with the +// ReadyForQuery of its Sync. The two can arrive in two reads, and the rejection +// runs user code between them. The failed request released its place in the +// connection at the ErrorResponse, so a statement issued in that gap wrote its +// Parse at once, and the late ReadyForQuery was taken for the answer to that +// Parse: advance() then wrote the Bind of the next request first. Replies go to +// the requests in queue order, so each query resolved with the rows of another +// one. With equal column types nothing fails to decode and nothing rejects. +// +// Now the failed request keeps the head of the queue until its ReadyForQuery, +// and advance() writes nothing past a request whose statement is being parsed. +const alice = { owner: "alice", secret: "alice-secret" }; +const bob = { owner: "bob", secret: "bob-secret" }; +const carol = { owner: "carol", secret: "carol-secret" }; +const errno = (err: any) => err.errno; + +/** `sql`, a reserved connection or a transaction. */ +type Handle = InstanceType; +const ofBob = (q: Handle) => q`select owner, secret from swap_b where id = ${1}`; +const divide = (q: Handle, by: number) => q`select 10 / ${by}::int as q`; +/** A statement with a parameter that is new to the connection: Parse, then Bind in a second batch. */ +const fresh = (q: Handle, table: string, columns = "owner, secret") => + q.unsafe(`select ${columns} from swap_${table} where id = $1`, [1]); +/** A statement without parameters that is new to the connection: Parse and Bind in one batch. */ +const freshOfCarol = (q: Handle) => q`select owner, secret from swap_c where id = 1`; +const simpleOfBob = (q: Handle) => q.unsafe(`select owner, secret from swap_b where id = 1`); + +type Proxy = Awaited>; +/** Resolves when the query has rejected. The proxy then holds the ReadyForQuery of that query. */ +type Failure = (q: Handle, proxy: Proxy) => Promise; +/** What is issued before the proxy lets that ReadyForQuery through. */ +type Gap = (q: Handle) => PromiseLike[]; + +// The statements of ofBob and divide are prepared, so these are written when +// they are issued, and the first ReadyForQuery arrives with the second owed. +const failsBehindAnother: Failure = (q, proxy) => { + proxy.holdAfterError(); + return Promise.all([ofBob(q), divide(q, 0)]).catch(errno); +}; +const failsAloneTwice: Failure = async (q, proxy) => { + proxy.holdAfterError(); + const first = await divide(q, 0).catch(errno); + const second = divide(q, 0).catch(errno); + proxy.holdAfterError(); + proxy.release(); + return [first, await second]; +}; + +const gaps: [name: string, issue: Gap, rows: unknown[]][] = [ + ["a new statement, a prepared one", q => [fresh(q, "a"), ofBob(q)], [[alice], [bob]]], + [ + "a new statement with other column types, a prepared one", + q => [fresh(q, "a", "id, owner"), ofBob(q)], + [[{ id: 1, owner: "alice" }], [bob]], + ], + [ + "a new statement, a prepared one, through execute()", + q => [fresh(q, "a").execute(), ofBob(q).execute()], + [[alice], [bob]], + ], + ["a new statement, a simple query", q => [fresh(q, "a"), simpleOfBob(q)], [[alice], [bob]]], + ["two new statements", q => [fresh(q, "a"), fresh(q, "c")], [[alice], [carol]]], + ["two new statements, a prepared one", q => [fresh(q, "a"), fresh(q, "c"), ofBob(q)], [[alice], [carol], [bob]]], + [ + "a new statement without parameters, a new statement, a prepared one", + q => [freshOfCarol(q), fresh(q, "a"), ofBob(q)], + [[carol], [alice], [bob]], + ], + [ + "a new statement without parameters, a new statement, a simple query", + q => [freshOfCarol(q), fresh(q, "a"), simpleOfBob(q)], + [[carol], [alice], [bob]], + ], +]; + +describeWithContainer("postgres", { image: "postgres_plain", concurrent: true }, container => { + async function run(failure: Failure, gap: Gap, reserve = false) { + await container.ready; + const proxy = await pgHoldingProxy(container.host, container.port); + try { + await using sql = new SQL({ + url: `postgres://bun_sql_test@127.0.0.1:${proxy.port}/bun_sql_test`, + max: reserve ? 10 : 1, + idleTimeout: 30, + }); + await using reserved = reserve ? await sql.reserve() : undefined; + const q: Handle = reserved ?? sql; + await q.unsafe(` + create temp table swap_a (id int, owner text, secret text); + create temp table swap_b (id int, owner text, secret text); + create temp table swap_c (id int, owner text, secret text); + insert into swap_a values (1, 'alice', 'alice-secret'); + insert into swap_b values (1, 'bob', 'bob-secret'); + insert into swap_c values (1, 'carol', 'carol-secret'); + `); + const warm = await Promise.all([ofBob(q), divide(q, 1)]); + + const failed = await failure(q, proxy); + const reads = Promise.all(gap(q)); + proxy.release(); + return { warm, failed, reads: await reads, after: await ofBob(q) }; + } finally { + proxy.server.close(); + } + } + const ran = (failed: unknown, reads: unknown[]) => ({ warm: [[bob], [{ q: 10 }]], failed, reads, after: [bob] }); + + describe.each([ + ["behind another query", failsBehindAnother, "22012"], + ["alone, twice in a row", failsAloneTwice, ["22012", "22012"]], + ] as const)("before the ReadyForQuery of a query that failed %s", (_, failure, failed) => { + test.each(gaps)("%s resolve with their own rows", async (_, gap, rows) => { + expect(await run(failure, gap)).toEqual(ran(failed, rows)); + }); + }); + + // A reserved handle keeps its queries on one connection, whatever the size of the pool. + test("on a connection reserved from a pool of 10, a new statement and a prepared one resolve with their own rows", async () => { + expect(await run(failsBehindAnother, gaps[0][1], true)).toEqual(ran("22012", [[alice], [bob]])); + }); + + test("a transaction that begins before that ReadyForQuery reads its own rows", async () => { + const gap: Gap = q => [q.begin(tx => Promise.all([fresh(tx, "a"), ofBob(tx)]))]; + expect(await run(failsBehindAnother, gap)).toEqual(ran("22012", [[[alice], [bob]]])); + }); + + // These failing queries are written with nothing else in flight. Each one + // keeps the head of the queue until its ReadyForQuery in its own way: a + // simple query holds nonpipelinable_requests, a statement that fails in its + // Parse stays Pending, a first execution without parameters holds no counter. + test.each([ + ["a simple query", q => q.unsafe(`select 10 / 0 as q`), "22012"], + ["a new statement that fails in its Parse", q => fresh(q, "none"), "42P01"], + ["a new statement without parameters that fails in its Parse", q => q`select owner from swap_none`, "42P01"], + ["a new statement without parameters that fails after its Parse", q => q`select 10 / 0 as q`, "22012"], + ] as [string, (q: Handle) => PromiseLike, string][])( + "before the ReadyForQuery of %s, a new statement and a prepared one resolve with their own rows", + async (_, fails, failed) => { + const failure: Failure = (q, proxy) => { + proxy.holdAfterError(); + return fails(q).then(undefined, errno); + }; + expect(await run(failure, gaps[0][1])).toEqual(ran(failed, [[alice], [bob]])); + }, + ); +}); + +// PostgreSQL sends nothing but the ReadyForQuery after an ErrorResponse, and one +// ReadyForQuery for each Sync. With such a server, either of the two rules alone +// gives each query its own rows. Each mock below sends what PostgreSQL does not, +// so each rule has a test that fails without it. +describe.concurrent("postgres mock", () => { + type Reply = (Buffer | typeof pgHold)[] | undefined; + /** Answers `select $1 as v` with the bound text as the row. `quirk` can replace the reply to a message. */ + async function echoServer(quirk: (type: string, body: Buffer, bound: string | undefined) => Reply) { + const bound = new WeakMap(); + const binds: string[] = []; + const mock = await pgMockServer((type, body, socket) => { + const reply = quirk(type, body, bound.get(socket)); + if (reply) return reply; + switch (type) { + case "P": + return pgParseComplete(); + case "D": + return [pgParameterDescription([25 /* text */]), pgRowDescription([{ name: "v", typeOid: 25 }])]; + case "B": + bound.set(socket, pgBindParameters(body)[0]!.toString()); + binds.push(bound.get(socket)!); + return pgBindComplete(); + case "E": + return [pgDataRow([Buffer.from(bound.get(socket)!)]), pgCommandComplete("SELECT 1")]; + case "S": + return pgReadyForQuery(); + case "X": + socket.end(); + } + }); + return { ...mock, binds }; + } + const settle = async (queries: PromiseLike[]) => + (await Promise.allSettled(queries)).map(result => + result.status === "fulfilled" ? [...result.value] : { code: result.reason.code, errno: result.reason.errno }, + ); + + // The failed request keeps the head of the queue until its ReadyForQuery. If + // it left at the ErrorResponse, the enqueue of `fresh` would write a Parse at + // once, and the held row would go to `fresh`. + test("frames that arrive after an ErrorResponse stay with the failed query when it failed behind another one", async () => { + const mock = await echoServer((type, _, bound) => + type === "E" && bound === "fail" + ? [ + pgErrorResponse({ S: "ERROR", C: "22012", M: "division by zero" }), + pgHold, + pgDataRow([Buffer.from("late")]), + pgCommandComplete("SELECT 1"), + ] + : undefined, + ); + try { + await using sql = new SQL({ url: `postgres://u@127.0.0.1:${mock.port}/db`, max: 1 }); + const echo = (text: string) => sql`select ${text} as v`; + await echo("warm"); + + // execute() sends a query at once, so these two are written before the first reply. + const first = echo("first").execute(); + const failed = echo("fail").execute(); + let fresh!: PromiseLike; + const rejection = await failed.catch((err: any) => { + fresh = sql`select ${"fresh"} as v -- not prepared yet`.execute(); + mock.release(); + return err.errno; + }); + + expect({ rejection, results: await settle([first, fresh, echo("after")]), binds: mock.binds }).toEqual({ + rejection: "22012", + results: [[{ v: "first" }], [{ v: "fresh" }], [{ v: "after" }]], + binds: ["warm", "first", "fail", "fresh", "after"], + }); + } finally { + mock.server.close(); + } + }); + + // These are the frames of pgdog 0.1.60 with its query parser on, for a + // statement that pgdog rejects itself: ErrorResponse and ReadyForQuery at the + // Flush, then one more ReadyForQuery at the Sync. The second one arrives when + // the Parse of `fresh` is on the wire. advance() must stop at `fresh`. If it + // stepped over it, the Bind of `prepared` would be written first. + test("a second ReadyForQuery for one Sync lets nothing pass a statement that is being parsed", async () => { + let rejecting = false; + const mock = await echoServer((type, body) => { + if (type === "P" && body.includes("rejected by the pooler")) rejecting = true; + if (!rejecting) return; + if (type === "H") return [pgErrorResponse({ S: "ERROR", C: "42601", M: "syntax error" }), pgReadyForQuery()]; + if (type !== "S") return []; + rejecting = false; + return [pgReadyForQuery()]; + }); + try { + await using sql = new SQL({ url: `postgres://u@127.0.0.1:${mock.port}/db`, max: 1 }); + const echo = (text: string) => sql`select ${text} as v`; + await echo("warm"); + + const rejected = sql`select 1 as v -- rejected by the pooler`.execute(); + const fresh = sql`select ${"fresh"} as v -- not prepared yet`.execute(); + const prepared = echo("prepared").execute(); + + expect({ results: await settle([rejected, fresh, prepared, echo("after")]), binds: mock.binds }).toEqual({ + results: [ + { code: "ERR_POSTGRES_SYNTAX_ERROR", errno: "42601" }, + [{ v: "fresh" }], + [{ v: "prepared" }], + [{ v: "after" }], + ], + binds: ["warm", "fresh", "prepared", "after"], + }); + } finally { + mock.server.close(); + } + }); +}); diff --git a/test/js/sql/wire-frames.ts b/test/js/sql/wire-frames.ts index af2030885250..22364f1d161d 100644 --- a/test/js/sql/wire-frames.ts +++ b/test/js/sql/wire-frames.ts @@ -408,6 +408,62 @@ export async function pgMockServer( return { port, server, release: () => releases.forEach(release => release()) }; } +/** + * TCP proxy in front of a real PostgreSQL (no TLS). Every byte goes through + * unchanged and in order; only the segmentation of the backend stream differs. + * After `holdAfterError()`, the proxy forwards the backend stream up to the end + * of the next ErrorResponse and keeps back what follows, the ReadyForQuery of + * the failed query first, until `release()`. + */ +export async function pgHoldingProxy( + upstreamHost: string, + upstreamPort: number, +): Promise<{ port: number; server: net.Server; holdAfterError(): void; release(): void }> { + let armed = false; + const releases = new Set<() => void>(); + const { port, server } = await listeningServer(client => { + const upstream = net.connect(upstreamPort, upstreamHost); + let buffered = Buffer.alloc(0); + let held: Buffer[] | undefined; + const release = () => { + if (held?.length) client.write(Buffer.concat(held)); + held = undefined; + }; + releases.add(release); + client.on("data", chunk => upstream.write(chunk)); + upstream.on("data", chunk => { + buffered = Buffer.concat([buffered, chunk]); + const out: Buffer[] = []; + let end = 0; + while (buffered.length - end >= 5) { + const frameEnd = end + 1 + buffered.readInt32BE(end + 1); + if (buffered.length < frameEnd) break; + (held ?? out).push(buffered.subarray(end, frameEnd)); + if (armed && buffered[end] === 0x45 /* 'E' ErrorResponse */) { + armed = false; + held = []; + } + end = frameEnd; + } + buffered = buffered.subarray(end); + if (out.length) client.write(Buffer.concat(out)); + }); + client.on("close", () => { + releases.delete(release); + upstream.destroy(); + }); + upstream.on("close", () => client.destroy()); + client.on("error", () => {}); + upstream.on("error", () => {}); + }); + return { + port, + server, + holdAfterError: () => void (armed = true), + release: () => releases.forEach(release => release()), + }; +} + // --------------------------------------------------------------------------- // MySQL client/server protocol — https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_packets.html // --------------------------------------------------------------------------- From 9bd19c98eacc01530a4e7609bc427abffa87d77e Mon Sep 17 00:00:00 2001 From: robobun Date: Sun, 4 Oct 2026 23:07:56 -0700 Subject: [PATCH 13/15] sql(mysql): reject only the query whose row the client cannot decode (#43323) ### Problem - `Bun.SQL` (MySQL): when the client cannot decode a row, its query rejects and the rest of the response goes to the next query. After a `CALL` with two result sets, the next query resolves with `[{"leftover":777}]`, or the connection fails with `ERR_MYSQL_UNEXPECTED_PACKET`. - `JSMySQLConnection::on_result_row` (`src/sql_jsc/mysql/JSMySQLConnection.rs`) marked the query `Fail` at the bad row. `advance()` pops a `Fail` head at the next terminator, while the server still answers it. - A JSON column that `JSON.parse` refuses fails in `to_js`, which returned `JSError`. That closes the connection: every queued query rejects with that `SyntaxError`. A default MariaDB stores such text (`{"name":"O\'Brien"}` passes `JSON_VALID`). ### Fix - The query rejects at once but stays at the queue head, because the server still sends the rest of its response. The new `DISCARD_RESPONSE` flag skips those rows. The last terminator, a server error, or a close marks it `Fail`, with no second rejection. - A `to_js` error takes the same path. A pending termination exception still fails the connection. - Verified: `test/js/sql/sql-mysql-row-decode-error.test.ts` (9 mock tests, 1 MariaDB test, all fail without the fix). Also `test/js/sql/sql-mysql*.test.ts`. - Self-reviewed: 9 concerns raised, 7 addressed. Not taken: a merge into #43187, the same fix for Postgres (linked instead), and issues for the "Not covered" sites. ### Background - `Bun.SQL` pipelines: it writes several queries before the first reply. Each reply packet goes to the head of a FIFO queue. - A response is one or more result sets. Each ends with a terminator. Its `SERVER_MORE_RESULTS_EXISTS` flag says that another follows. - `advance()` pops finished requests from the queue head and writes queued ones.
Notes **Reproduction.** Mock server, 1.4.3 (`b52d51348`), `max: 1`, before the fix: ``` A (decode error, two result sets) bad query : REJECTED ERR_MYSQL_INVALID_BINARY_VALUE: Failed to bind query neighbour : [{"leftover":777}] B (to_js error) bad query : REJECTED SyntaxError: JSON Parse error: Expected '}' neighbour : REJECTED SyntaxError: JSON Parse error: Expected '}' ``` **MariaDB.** A JSON column is `LONGTEXT` with a `JSON_VALID` check constraint. On MariaDB 11.8.6 with default settings, `JSON_VALID` accepts `\'`, `\x41`, `\q` and a Windows path with single backslashes (`{"path":"C:\Users\qux"}`). `JSON.parse` refuses all of them. Before the fix, one such row closed the connection: a pipelined query that the server had answered rejected with `SyntaxError: JSON Parse error: Invalid escape character '`, and a `TEMPORARY` table was gone for the next query. Before v1.4.0 such a column came back as a string (#37130 began to parse it). The MariaDB test in the new file covers this. It runs where `mariadb_plain` is provided (the same gate as `sql-mariadb-json.test.ts`). MySQL has a native JSON type and sends normalized text, so only a mock reaches this path there. The decode error (`row.decode`) needs a mock: a healthy server does not send a binary `DATE` of 5 bytes. MySQL frames each row in a packet, and `process_packets` skips the packet whole, so the stream stays in step after any row decode error. **Not covered, on purpose.** - A column definition that cannot be decoded, at prepare time (`handle_prepared_statement`) or at execute time (`handle_result_set`), still fails the connection. A column type that `FieldType` does not know reaches these sites with `ERR_MYSQL_UNSUPPORTED_COLUMN_TYPE`, for example `MYSQL_TYPE_VECTOR` (0xf2) from MySQL 9. The fix there is to decode an unknown type as bytes, not to reject the query, so it is a separate change. - A row whose lengths run past its packet still reads as `ShortRead`, which means "wait for more data". #35118 bounds reads to the packet. After it, such a row takes the path of this PR. - A result set header, `OK`, `EOF` or `ERR` packet that cannot be decoded still fails the connection. The stream is out of step then. **Overlap with open PRs.** #33190 adds `mysqlBinaryResultSetRow` and `mysqlBinaryResultSet` to `test/js/sql/wire-frames.ts` under the same names. The versions here take one more optional argument (`statusFlags`). The PR that lands second drops its copy. `settle()` in the new test file is the helper from the test file of #43187. The PR that lands second can share it. **Accounting.** `mark_current_request_as_finished` is no longer called at the bad row. The last terminator, or the `ERR` packet, calls it while the request still counts as running, so `pipelined_requests` or `nonpipelinable_requests` drops once. A build that skips this release for a discarding request makes 8 of the 9 mock tests time out: a statement that is not prepared yet, and a `.simple()` query, can only start after the counter drops. **Other shapes I ran by hand on the ASAN debug build.** - Real MariaDB: a 200000 row result whose first row is not JSON, with `Bun.gc(true)` in the rejection handler while the rest streams. The same with the bad row second to last. - Real MariaDB: `sql.begin()`. The callback throws on the rejection and `ROLLBACK` runs on the same connection. A callback that catches the rejection continues in the same transaction. - Real MariaDB: `.values()` rejects the same way. `.raw()` does not decode and resolves. - Mock: the legacy framing without `CLIENT_DEPRECATE_EOF`. - Mock: the server drops the connection, or `sql.close({ timeout })` fires, while the response is discarded. The query rejects once and the next one rejects with `ERR_MYSQL_CONNECTION_CLOSED`. **Test runs** on the debug build against a local MariaDB: `sql-mysql.test.ts` (plain image only) gives 97 pass and 7 fail, `sql-mysql.transactions.test.ts` gives 12 pass and 2 fail. The same tests fail on the build without the fix. They compare MySQL error texts that MariaDB words differently, or they exceed their timeout under ASAN. `BUN_JSC_validateExceptionChecks=1` passes on the new file.
--- src/sql_jsc/mysql/JSMySQLConnection.rs | 53 +-- src/sql_jsc/mysql/JSMySQLQuery.rs | 46 ++- src/sql_jsc/mysql/MySQLQuery.rs | 46 ++- .../js/sql/sql-mysql-row-decode-error.test.ts | 353 ++++++++++++++++++ test/js/sql/wire-frames.ts | 71 +++- 5 files changed, 535 insertions(+), 34 deletions(-) create mode 100644 test/js/sql/sql-mysql-row-decode-error.test.ts diff --git a/src/sql_jsc/mysql/JSMySQLConnection.rs b/src/sql_jsc/mysql/JSMySQLConnection.rs index f754ec0ba085..8ed2da29c633 100644 --- a/src/sql_jsc/mysql/JSMySQLConnection.rs +++ b/src/sql_jsc/mysql/JSMySQLConnection.rs @@ -734,12 +734,32 @@ impl JSMySQLConnection { request.resolve(self.get_queries_array(), result); } + /// The client cannot turn one of `request`'s rows into a JS value. The + /// packet header frames the row and the packet is skipped whole, so the + /// stream is still in step: that fails the request alone, not the + /// connection and the requests queued behind it. A VM that is stopping + /// fails the connection instead. + fn on_undecodable_row( + &self, + request: &JSMySQLQuery, + err: AnyMySQLErrorT, + ) -> Result<(), OnResultRowError> { + if self.global_object.has_pending_termination_exception() { + return Err(OnResultRowError::JSError); + } + request.reject_undecodable_row(self.get_queries_array(), err); + Ok(()) + } + pub(crate) fn on_result_row( &self, request: &JSMySQLQuery, statement: &mut MySQLStatement, reader: NewReader, ) -> Result<(), OnResultRowError> { + if request.is_discarding_response() { + return Ok(()); + } let result_mode = request.get_result_mode(); let mut structure: JSValue = JSValue::UNDEFINED; // `MySQLStatement::structure(&mut self) -> &CachedStructure` @@ -774,37 +794,26 @@ impl JSMySQLConnection { if e == AnyMySQLErrorT::ShortRead { return Err(OnResultRowError::ShortRead); } - self.connection_mut() - .queue - .mark_current_request_as_finished(request); - request.reject(self.get_queries_array(), e); - return Ok(()); + return self.on_undecodable_row(request, e); } let pending_value = request.get_pending_value().unwrap_or(JSValue::UNDEFINED); // `ParentRef::Deref` recovers `&CachedStructure`; `*statement` is live // and not mutably borrowed for the duration of this `to_js` call. let cached_structure = cached_structure.as_deref(); // Process row data - let row_value = row - .to_js( - &self.global_object, - pending_value, - structure, - fields_flags, - result_mode, - cached_structure, - ) - .map_err(|_| OnResultRowError::JSError)?; + let Ok(row_value) = row.to_js( + &self.global_object, + pending_value, + structure, + fields_flags, + result_mode, + cached_structure, + ) else { + return self.on_undecodable_row(request, AnyMySQLErrorT::JSError); + }; // `Row<'_>` has a Drop impl, so its `&statement.columns` borrow lives to // end-of-scope; drop it now so `statement.result_count += 1` may take `&mut`. drop(row); - if let Some(err) = self.global_object.try_take_exception() { - self.connection_mut() - .queue - .mark_current_request_as_finished(request); - request.reject_with_js_value(self.get_queries_array(), err); - return Ok(()); - } statement.result_count += 1; if pending_value.is_empty_or_undefined_or_null() { diff --git a/src/sql_jsc/mysql/JSMySQLQuery.rs b/src/sql_jsc/mysql/JSMySQLQuery.rs index 191eb98c607c..a89dba4e8e8c 100644 --- a/src/sql_jsc/mysql/JSMySQLQuery.rs +++ b/src/sql_jsc/mysql/JSMySQLQuery.rs @@ -309,16 +309,42 @@ impl JSMySQLQuery { } pub(crate) fn reject(&self, queries_array: JSValue, err: AnyMySQLError::Error) { - if let Some(err_) = self.global_object().try_take_exception() { - self.reject_with_js_value(queries_array, err_); - } else { - let instance = mysql_error_to_js(self.global_object(), "Failed to bind query", err); - instance.ensure_still_alive(); - self.reject_with_js_value(queries_array, instance); - } + self.reject_with_js_value(queries_array, self.error_to_js(err)); + } + + /// The client cannot decode a row of this query's result. Reject the query + /// now, but leave it in flight (`MySQLQuery::discard_response`): the + /// connection keeps it at the queue head and skips the rest of its + /// response. + pub(crate) fn reject_undecodable_row(&self, queries_array: JSValue, err: AnyMySQLError::Error) { + self.reject_when( + MySQLQuery::discard_response, + queries_array, + self.error_to_js(err), + ); } pub(crate) fn reject_with_js_value(&self, queries_array: JSValue, err: JSValue) { + self.reject_when(MySQLQuery::fail, queries_array, err); + } + + fn error_to_js(&self, err: AnyMySQLError::Error) -> JSValue { + if let Some(err_) = self.global_object().try_take_exception() { + return err_; + } + let instance = mysql_error_to_js(self.global_object(), "Failed to bind query", err); + instance.ensure_still_alive(); + instance + } + + /// `transition` moves the query to its rejected state, and returns whether + /// a rejection is still to be delivered to JS. + fn reject_when( + &self, + transition: fn(&mut MySQLQuery) -> bool, + queries_array: JSValue, + err: JSValue, + ) { // `ref_guard` brackets re-entry; drops *after* `_downgrade` so the // allocation outlives the closure body. let _guard = self.ref_guard(); @@ -330,7 +356,7 @@ impl JSMySQLQuery { } }); - if !self.query.with_mut(|q| q.fail()) { + if !self.query.with_mut(transition) { return; } @@ -447,6 +473,10 @@ impl JSMySQLQuery { self.query.get().is_pipelined() } #[inline] + pub(crate) fn is_discarding_response(&self) -> bool { + self.query.get().is_discarding_response() + } + #[inline] pub(crate) fn is_simple(&self) -> bool { self.query.get().is_simple() } diff --git a/src/sql_jsc/mysql/MySQLQuery.rs b/src/sql_jsc/mysql/MySQLQuery.rs index a80bc954a2b6..05b0e95e64e0 100644 --- a/src/sql_jsc/mysql/MySQLQuery.rs +++ b/src/sql_jsc/mysql/MySQLQuery.rs @@ -44,7 +44,12 @@ impl Flags { const SIMPLE: u8 = 1 << 1; const PIPELINED: u8 = 1 << 2; const RESULT_MODE_SHIFT: u8 = 3; - const RESULT_MODE_MASK: u8 = 0b11 << Self::RESULT_MODE_SHIFT; // SQLQueryResultMode is 2 bits (3 bool + 2 + 3 pad = 8) + const RESULT_MODE_MASK: u8 = 0b11 << Self::RESULT_MODE_SHIFT; // SQLQueryResultMode is 2 bits (4 bool + 2 + 2 pad = 8) + /// Set by [`MySQLQuery::discard_response`]: the query is already rejected, + /// but the server is still answering it. It stays in flight at the queue + /// head, and the rest of its response is skipped, until the terminator of + /// its last result set. + const DISCARD_RESPONSE: u8 = 1 << 5; #[inline] fn bigint(self) -> bool { @@ -67,6 +72,14 @@ impl Flags { } } #[inline] + fn discard_response(self) -> bool { + self.0 & Self::DISCARD_RESPONSE != 0 + } + #[inline] + fn set_discard_response(&mut self) { + self.0 |= Self::DISCARD_RESPONSE; + } + #[inline] fn result_mode(self) -> SQLQueryResultMode { // result_mode bits were written from a valid SQLQueryResultMode // discriminant (`set_result_mode`); the unreachable 4th bit-state @@ -435,11 +448,18 @@ impl MySQLQuery { self.flags.set_result_mode(result_mode); } + /// Returns whether the caller has a result to deliver. #[inline] pub(crate) fn result(&mut self, is_last_result: bool) -> bool { if self.status == Status::Success || self.status == Status::Fail { return false; } + if self.flags.discard_response() { + if is_last_result { + self.status = Status::Fail; + } + return false; + } self.status = if is_last_result { Status::Success } else { @@ -449,15 +469,39 @@ impl MySQLQuery { true } + /// Returns whether the caller has a rejection to deliver. pub(crate) fn fail(&mut self) -> bool { if self.status == Status::Fail || self.status == Status::Success { return false; } self.status = Status::Fail; + !self.flags.discard_response() + } + + /// The client cannot decode a row of this query's result. The caller + /// rejects the query now, but the status stays in flight: the server is + /// still answering, and a `Fail` head would let `advance()` pop it and hand + /// the rest of its response to the next request. [`Self::result`] ends it + /// at its last terminator. Returns whether the caller has a rejection to + /// deliver. + pub(crate) fn discard_response(&mut self) -> bool { + if !self.is_running() { + return self.fail(); + } + if self.flags.discard_response() { + return false; + } + self.flags.set_discard_response(); + true } + #[inline] + pub(crate) fn is_discarding_response(&self) -> bool { + self.flags.discard_response() + } + #[inline] pub(crate) fn is_completed(&self) -> bool { self.status == Status::Success || self.status == Status::Fail diff --git a/test/js/sql/sql-mysql-row-decode-error.test.ts b/test/js/sql/sql-mysql-row-decode-error.test.ts new file mode 100644 index 000000000000..a66e6d43e508 --- /dev/null +++ b/test/js/sql/sql-mysql-row-decode-error.test.ts @@ -0,0 +1,353 @@ +// A result row can be framed correctly and still hold a value this client +// cannot turn into a JS value: a JSON column whose text `JSON.parse` refuses +// (MariaDB accepts some), a binary DATE with a length no server sends. That is +// the failure of the query that asked for the row, and the server keeps +// answering that query: more rows, the terminator, further result sets of a +// CALL or a multi-statement string. +// +// The connection marked the query failed at once. The request queue then +// dropped it at the next terminator, and the rest of its response went to the +// query queued behind it. When the value failed only while it was turned into a +// JS value (the JSON case), the connection closed instead, and every query on it +// rejected with that one query's error. +import { SQL, randomUUIDv7 } from "bun"; +import { describe, expect, test } from "bun:test"; +import { describeWithContainer, isCI, isDockerEnabled } from "harness"; +import { + MYSQL_SERVER_MORE_RESULTS_EXISTS, + MYSQL_SERVER_STATUS_AUTOCOMMIT, + listeningServer, + mysqlAckSessionSetup, + mysqlBinaryDate, + mysqlBinaryLong, + mysqlBinaryResultSet, + mysqlErrPacket, + mysqlHandshakeV10, + mysqlLenencStr, + mysqlOkPacket, + mysqlReadPackets, + mysqlStmtPrepareOk, + mysqlTextResultSet, +} from "./wire-frames"; + +/** One entry per query, in order: its rows, or the error it rejected with. */ +async function settle(queries: PromiseLike[]) { + return (await Promise.allSettled(queries)).map(result => + result.status === "fulfilled" + ? [...result.value] + : { name: result.reason.name, code: result.reason.code, message: result.reason.message }, + ); +} + +// Fault-injection tests: a healthy server does not send a DATE of 5 bytes, and +// it does not stop in the middle of a response on demand. DO NOT COPY THIS +// PATTERN: anything a real server can produce belongs in describeWithContainer. +// All wire-protocol bytes come from test/js/sql/wire-frames.ts. +// +// The mock answers each statement text with its entry in `responses`, over both +// protocols. An entry is a list of chunks. The first chunk is sent at once. The +// mock keeps back the other chunks, and everything it sends after them, until +// `release()`. +async function mockServer(responses: Record) { + let connections = 0; + let release = () => {}; + const { port, server } = await listeningServer(socket => { + connections++; + let buffered = Buffer.alloc(0); + let authed = false; + let held: Buffer[] | undefined; + const prepared: string[] = []; + const send = (chunks: Buffer[]) => { + chunks.forEach((chunk, i) => { + if (i > 0) held ??= []; + if (held) held.push(chunk); + else socket.write(chunk); + }); + }; + release = () => { + if (!held) return; + socket.write(Buffer.concat(held)); + held = undefined; + }; + socket.write(mysqlHandshakeV10()); + socket.on("error", () => {}); + socket.on("data", chunk => { + buffered = mysqlReadPackets(Buffer.concat([buffered, chunk]), (seq, payload) => { + if (!authed) { + authed = true; + socket.write(mysqlOkPacket(seq + 1)); + return; + } + if (mysqlAckSessionSetup(socket, payload)) return; + switch (payload[0]) { + case 0x16: // COM_STMT_PREPARE: Int<1>(0x16) String(query) + // No columns at prepare time, which is what a server says for a + // CALL. The client takes the columns from each result set. + prepared.push(payload.subarray(1).toString()); + return send([mysqlStmtPrepareOk(1, prepared.length, 0, 0)]); + case 0x17: // COM_STMT_EXECUTE: Int<1>(0x17) Int<4>(statement_id) ... + return send(responses[prepared[payload.readUInt32LE(1) - 1]]); + case 0x03: // COM_QUERY: Int<1>(0x03) String(query) + return send(responses[payload.subarray(1).toString()]); + } + }); + }); + }); + return { port, server, release: () => release(), connections: () => connections }; +} + +const MYSQL_TYPE_LONG = 0x03; +const MYSQL_TYPE_DATE = 0x0a; +const MYSQL_TYPE_JSON = 0xf5; +const moreResults = MYSQL_SERVER_STATUS_AUTOCOMMIT | MYSQL_SERVER_MORE_RESULTS_EXISTS; + +/** One result set with one column: its name, then one cell per row. A number is a LONG, a string is the text of a JSON column. */ +type ResultSet = [column: string, ...cells: (number | string)[]]; + +/** + * One response: result sets back to back, over the text protocol (`simple`) or + * the binary one. Every terminator but the last says that another result set + * follows. With `serverError` they all say so, and an ERR packet ends the + * response, as when a later statement of a CALL fails. + */ +function response(simple: boolean, sets: ResultSet[], serverError = false): Buffer { + let seq = 1; + const frames = sets.map(([name, ...cells], i) => { + const columns = [{ name, type: typeof cells[0] === "number" ? MYSQL_TYPE_LONG : MYSQL_TYPE_JSON }]; + const statusFlags = serverError || i + 1 < sets.length ? moreResults : undefined; + const startSeq = seq; + seq += 3 + cells.length; // the column count, the column, the rows, the terminator + return simple + ? mysqlTextResultSet( + startSeq, + columns, + cells.map(cell => [String(cell)]), + statusFlags, + ) + : mysqlBinaryResultSet( + startSeq, + columns, + cells.map(cell => [typeof cell === "number" ? mysqlBinaryLong(cell) : mysqlLenencStr(cell)]), + statusFlags, + ); + }); + if (serverError) frames.push(mysqlErrPacket(seq, 1146, "42S02", "Table 'db.missing' doesn't exist")); + return Buffer.concat(frames); +} + +/** The response to `select as n`. */ +const numberRow = (n: number, simple = false) => [response(simple, [["n", n]])]; + +const date = [{ name: "d", type: MYSQL_TYPE_DATE }]; +// The length byte of a binary DATE is 0, 4, 7 or 11. +const undecodableDate = mysqlBinaryDate(2024, 6, 15, 5); +// A CALL whose first result set has a row the client cannot decode between two +// rows it can. A second result set follows: 3 rows + 3 sequence ids later. +const callBad = Buffer.concat([ + mysqlBinaryResultSet( + 1, + date, + [[mysqlBinaryDate(2024, 6, 14)], [undecodableDate], [mysqlBinaryDate(2024, 6, 16)]], + moreResults, + ), + mysqlBinaryResultSet(7, [{ name: "leftover", type: MYSQL_TYPE_LONG }], [[mysqlBinaryLong(777)]]), +]); +const undecodable = { name: "MySQLError", code: "ERR_MYSQL_INVALID_BINARY_VALUE", message: expect.any(String) }; +const notJson = { name: "SyntaxError", message: "JSON Parse error: Expected '}'" }; + +describe.concurrent("mysql mock", () => { + test("a pipelined query with a row the client cannot decode rejects alone", async () => { + const mock = await mockServer({ "call bad()": [callBad], "select 1 as n": numberRow(1) }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + // After this first run the statement is prepared, so its second run is + // written right behind `call bad()`, before the first reply arrives. + await sql`select 1 as n`; + + const results = await settle([sql`call bad()`, sql`select 1 as n`]); + results.push([...(await sql`select 1 as n`)]); + expect({ results, connections: mock.connections() }).toEqual({ + results: [undecodable, [{ n: 1 }], [{ n: 1 }]], + connections: 1, + }); + } finally { + mock.server.close(); + } + }); + + test("a statement that is not prepared yet runs after the rejected query's response", async () => { + const mock = await mockServer({ "call bad()": [callBad], "select 2 as n": numberRow(2) }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + + // A prepare cannot be pipelined. `select 2 as n` waits until the + // connection no longer counts `call bad()` as in flight. + expect({ + results: await settle([sql`call bad()`, sql`select 2 as n`]), + connections: mock.connections(), + }).toEqual({ results: [undecodable, [{ n: 2 }]], connections: 1 }); + } finally { + mock.server.close(); + } + }); + + test("result sets that arrive after the rejection stay with the rejected query", async () => { + const mock = await mockServer({ + "call bad()": [ + mysqlBinaryResultSet(1, date, [[undecodableDate]], moreResults), + response(false, [["leftover", 777]]), + ], + "select 3 as n": numberRow(3), + "select 4 as n": numberRow(4), + }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + await sql`select 3 as n`; + + // execute() sends a query at once, so these two are pipelined in this order. + const bad = sql`call bad()`.execute(); + const next = sql`select 3 as n`.execute(); + // `bad` rejects while the server still holds its second result set. The + // handler queues a statement that is not prepared yet, then lets the + // server continue. The held result set must not go to `next`, which is + // the request behind the rejected one. + let unprepared!: PromiseLike; + const rejection = await bad.catch((err: any) => { + unprepared = sql`select 4 as n`.execute(); + mock.release(); + return err.code; + }); + + expect({ + rejection, + results: await settle([next, unprepared]), + connections: mock.connections(), + }).toEqual({ + rejection: "ERR_MYSQL_INVALID_BINARY_VALUE", + results: [[{ n: 3 }], [{ n: 4 }]], + connections: 1, + }); + } finally { + mock.server.close(); + } + }); + + // The binary protocol (a prepared statement) and the text protocol (`.simple()`) + // decode a row in different code, and count a running request differently. + describe.each([ + ["prepared", false], + ["simple", true], + ] as const)("%s query", (_, simple) => { + const run = (query: any) => (simple ? query.simple() : query); + // Three queries behind `call docs()`. The first is already prepared, so a + // prepared `call docs()` has it pipelined right behind. The other two can + // only start when the connection no longer counts a request as in flight. + const responses = { + "select 5 as n": numberRow(5), + "select 6 as n": numberRow(6), + "select 7 as n": numberRow(7, true), + }; + const others = (sql: SQL) => [sql`select 5 as n`, sql`select 6 as n`, sql`select 7 as n`.simple()]; + const ownRows = [[{ n: 5 }], [{ n: 6 }], [{ n: 7 }]]; + + test("a JSON column that is not JSON rejects alone", async () => { + const docs = response(simple, [ + ["doc", '{"a":1}', "{not json", '{"b":2}'], + ["leftover", 777], + ]); + const mock = await mockServer({ "call docs()": [docs], ...responses }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + await sql`select 5 as n`; + + const results = await settle([run(sql`call docs()`), ...others(sql)]); + results.push([...(await sql`select 5 as n`)]); + expect({ results, connections: mock.connections() }).toEqual({ + results: [notJson, ...ownRows, [{ n: 5 }]], + connections: 1, + }); + } finally { + mock.server.close(); + } + }); + + test("a bad row in the second of three result sets rejects alone", async () => { + const docs = response(simple, [ + ["first", 1], + ["doc", "{not json"], + ["leftover", 777], + ]); + const mock = await mockServer({ "call docs()": [docs], ...responses }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + await sql`select 5 as n`; + + expect({ + results: await settle([run(sql`call docs()`), ...others(sql)]), + connections: mock.connections(), + }).toEqual({ results: [notJson, ...ownRows], connections: 1 }); + } finally { + mock.server.close(); + } + }); + + test("a server error that ends the rejected query's response is not delivered", async () => { + const mock = await mockServer({ "call docs()": [response(simple, [["doc", "{not json"]], true)], ...responses }); + try { + await using sql = new SQL({ url: `mysql://root@127.0.0.1:${mock.port}/db`, max: 1 }); + await sql`select 5 as n`; + + expect({ + results: await settle([run(sql`call docs()`), ...others(sql)]), + connections: mock.connections(), + }).toEqual({ results: [notJson, ...ownRows], connections: 1 }); + } finally { + mock.server.close(); + } + }); + }); +}); + +// MariaDB has no JSON type. A JSON column is LONGTEXT with a JSON_VALID check +// constraint, and JSON_VALID accepts escapes that JSON.parse refuses. So a +// healthy MariaDB with default settings can send a JSON column that this client +// cannot parse. The gate on the container is the one in sql-mariadb-json.test.ts. +const mariadbProvided = + !!process.env.BUN_TEST_SERVICE_mariadb_plain || !!process.env.BUN_DOCKER_COORDINATOR || (isCI && isDockerEnabled()); +if (!mariadbProvided) { + describe.todo("mariadb"); +} else { + describeWithContainer("mariadb", { image: "mariadb_plain" }, container => { + test("a JSON column that JSON.parse refuses rejects only the query that reads it", async () => { + await container.ready; + await using sql = new SQL({ url: `mysql://root@${container.host}:${container.port}/bun_sql_test`, max: 1 }); + const table = ("t_" + randomUUIDv7("hex").replaceAll("-", "")).toLowerCase(); + await sql`CREATE TEMPORARY TABLE ${sql(table)} (id INT, doc JSON)`; + for (const [id, doc] of ['{"a":1}', `{"name":"O\\'Brien"}`, '{"b":2}'].entries()) { + await sql`INSERT INTO ${sql(table)} VALUES (${id + 1}, ${doc})`; + } + const refused = { name: "SyntaxError", message: "JSON Parse error: Invalid escape character '" }; + + // One statement text. After this first run it is prepared, so the three + // executions below are all written before the first reply arrives. + const from = (id: number) => sql`SELECT id, doc FROM ${sql(table)} WHERE id >= ${id} ORDER BY id`; + await from(3); + expect(await settle([from(3), from(1), from(3)])).toEqual([ + [{ id: 3, doc: { b: 2 } }], + refused, + [{ id: 3, doc: { b: 2 } }], + ]); + + // The same over the text protocol, where nothing is pipelined. + expect( + await settle([ + sql`SELECT id, doc FROM ${sql(table)} ORDER BY id`.simple(), + sql`SELECT id, doc FROM ${sql(table)} WHERE id = 3`.simple(), + ]), + ).toEqual([refused, [{ id: 3, doc: { b: 2 } }]]); + + // A temporary table lives and dies with its connection. + expect([...(await from(3))]).toEqual([{ id: 3, doc: { b: 2 } }]); + }); + }); +} diff --git a/test/js/sql/wire-frames.ts b/test/js/sql/wire-frames.ts index 22364f1d161d..4de57db65f2f 100644 --- a/test/js/sql/wire-frames.ts +++ b/test/js/sql/wire-frames.ts @@ -542,10 +542,21 @@ export function mysqlHandshakeV10( return mysqlRawPacket(0, payload); } +// Server status flags — SERVER_STATUS_flags_enum in mysql__com_8h.html (subset used by the mocks). +export const MYSQL_SERVER_STATUS_AUTOCOMMIT = 0x0002; +export const MYSQL_SERVER_MORE_RESULTS_EXISTS = 0x0008; // another result set of the same response follows + // MySQL Protocol::OK_Packet — page_protocol_basic_ok_packet.html: Int<1>(header) lenenc(affected_rows) lenenc(last_insert_id) Int<2>(status) Int<2>(warnings) // The header is 0x00, except for the CLIENT_DEPRECATE_EOF result-set terminator, which is an OK packet with a 0xFE header. -export function mysqlOkPacket(seq: number, header: 0x00 | 0xfe = 0x00): Buffer { - return mysqlRawPacket(seq, Buffer.from([header, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00])); +export function mysqlOkPacket( + seq: number, + header: 0x00 | 0xfe = 0x00, + statusFlags: number = MYSQL_SERVER_STATUS_AUTOCOMMIT, +): Buffer { + return mysqlRawPacket( + seq, + Buffer.from([header, 0x00, 0x00, statusFlags & 0xff, (statusFlags >> 8) & 0xff, 0x00, 0x00]), + ); } // MySQL ERR_Packet — page_protocol_basic_err_packet.html: @@ -705,16 +716,70 @@ export function mysqlTextResultSetRow(seq: number, cols: (string | Buffer)[]): B // MySQL Textual Resultset — page_protocol_com_query_response_text_resultset.html, in the // CLIENT_DEPRECATE_EOF framing: lenenc(column_count) packet, one ColumnDefinition41 per // column, one row packet per row, then an OK packet with the 0xFE header as the terminator. +// A response with several result sets (CALL, multi-statement) is these back to back: every +// terminator but the last carries MYSQL_SERVER_MORE_RESULTS_EXISTS in `statusFlags`, and the +// sequence ids run on (this result set takes 2 + columns.length + rows.length of them). export function mysqlTextResultSet( startSeq: number, columns: { name: string; type: number }[], rows: string[][], + statusFlags?: number, ): Buffer { let seq = startSeq; const parts: Buffer[] = [mysqlRawPacket(seq++, mysqlLenencInt(columns.length))]; for (const column of columns) parts.push(mysqlColumnDefinition(seq++, column)); for (const row of rows) parts.push(mysqlTextResultSetRow(seq++, row)); - parts.push(mysqlOkPacket(seq, 0xfe)); + parts.push(mysqlOkPacket(seq, 0xfe, statusFlags)); + return Buffer.concat(parts); +} + +// MySQL Binary Protocol Resultset Row — page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row: +// Int<1>(0x00) Byte<(column_count + 7 + 2) / 8>(NULL bitmap, first column at bit 2) then every non-NULL value. +// Each value is already in its column type's binary encoding (mysqlBinaryLong, mysqlLenencStr, ...), +// so a fault-injection test can put bytes here that no server sends. +export function mysqlBinaryResultSetRow(seq: number, values: (Buffer | null)[]): Buffer { + const nullBitmap = Buffer.alloc((values.length + 7 + 2) >> 3); + values.forEach((value, i) => { + if (value === null) nullBitmap[(i + 2) >> 3] |= 1 << ((i + 2) & 7); + }); + return mysqlRawPacket( + seq, + Buffer.concat([Buffer.from([0x00]), nullBitmap, ...values.filter((value): value is Buffer => value !== null)]), + ); +} + +// Binary protocol value of MYSQL_TYPE_LONG — page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row_value_long: Int<4> +export function mysqlBinaryLong(n: number): Buffer { + const value = Buffer.alloc(4); + value.writeInt32LE(n); + return value; +} + +// Binary protocol value of MYSQL_TYPE_DATE — page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row_value_date: +// Int<1>(length) Int<2>(year) Int<1>(month) Int<1>(day). With a time part (DATETIME, TIMESTAMP) the length is 7 or 11. +// `declaredLength` is the fault-injection escape hatch (mirrors mysqlRawPacket): a server only sends 0, 4, 7 or 11. +export function mysqlBinaryDate(year: number, month: number, day: number, declaredLength: number = 4): Buffer { + const value = Buffer.alloc(5); + value[0] = declaredLength; + value.writeUInt16LE(year, 1); + value[3] = month; + value[4] = day; + return value; +} + +// MySQL Binary Protocol Resultset — page_protocol_binary_resultset.html: what COM_STMT_EXECUTE answers with. +// The framing is mysqlTextResultSet's (see there for `statusFlags`); only the rows differ. +export function mysqlBinaryResultSet( + startSeq: number, + columns: { name: string; type: number }[], + rows: (Buffer | null)[][], + statusFlags?: number, +): Buffer { + let seq = startSeq; + const parts: Buffer[] = [mysqlRawPacket(seq++, mysqlLenencInt(columns.length))]; + for (const column of columns) parts.push(mysqlColumnDefinition(seq++, column)); + for (const row of rows) parts.push(mysqlBinaryResultSetRow(seq++, row)); + parts.push(mysqlOkPacket(seq, 0xfe, statusFlags)); return Buffer.concat(parts); } From b983eca807226601175e69eeecb960285160465b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 4 Oct 2026 23:22:58 -0700 Subject: [PATCH 14/15] test: join fetch leak fixture lifetime before disposal Wait for the Node HTTP listener and all 32 body-validated fetches before disposing the server. Replace the unjoined timer and vacuous Infinity comparison with isolated Response and AbortSignal reclamation checks. The original fixture fails with ERR_SERVER_NOT_RUNNING; a retained-response control fails the finite object bound. --- test/js/web/fetch/fetch-leak.test.ts | 67 +++++++++++++++------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/test/js/web/fetch/fetch-leak.test.ts b/test/js/web/fetch/fetch-leak.test.ts index 0674d650f84f..a6775d404ea9 100644 --- a/test/js/web/fetch/fetch-leak.test.ts +++ b/test/js/web/fetch/fetch-leak.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, tls as COMMON_CERT, gc, isASAN, isCI, isDebug } from "harness"; +import { bunEnv, bunExe, tls as COMMON_CERT, isASAN, isCI, isDebug } from "harness"; import { once } from "node:events"; import { createServer } from "node:http"; import net from "node:net"; @@ -164,40 +164,45 @@ describe.each(["FormData", "Blob", "Buffer", "String", "URLSearchParams", "strea }); test("do not leak", async () => { + let requests = 0; await using server = createServer((req, res) => { - res.end(); + requests++; + res.end("ok"); }); - - let url; - let isDone = false; - server.listen(0, "127.0.0.1", function attack() { - if (isDone) { - return; + await once(server.listen(0, "127.0.0.1"), "listening"); + const url = `http://127.0.0.1:${server.address().port}`; + // Isolate object counts from the other leak tests and join all work before disposing the server. + const script = ` + const { heapStats } = require("bun:jsc"); + async function request() { + const response = await fetch(${JSON.stringify(url)}, { signal: new AbortController().signal }); + if (await response.text() !== "ok") throw new Error("unexpected response body"); } - url ??= new URL(`http://127.0.0.1:${server.address().port}`); - const controller = new AbortController(); - fetch(url, { signal: controller.signal }) - .then(res => res.arrayBuffer()) - .catch(() => {}) - .then(attack); - }); - - let prev = Infinity; - let count = 0; - var interval = setInterval(() => { - isDone = true; - gc(); - const next = process.memoryUsage().heapUsed; - if (next <= prev) { - expect(true).toBe(true); - clearInterval(interval); - } else if (count++ > 20) { - clearInterval(interval); - expect.unreachable(); - } else { - prev = next; + for (let i = 0; i < 32; i++) await request(); + let counts; + for (let i = 0; i < 10; i++) { + Bun.gc(true); + await new Promise(resolve => setImmediate(resolve)); + counts = heapStats().objectTypeCounts; + if ((counts.Response ?? 0) <= 2 && (counts.AbortSignal ?? 0) <= 2) break; } - }, 1e3); + console.log(JSON.stringify({ Response: counts.Response ?? 0, AbortSignal: counts.AbortSignal ?? 0 })); + `; + await using proc = Bun.spawn({ + cmd: [bunExe(), "--smol", "-e", script], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(requests).toBe(32); + const counts = JSON.parse(stdout.trim()); + // Each class's prototype is included; retaining all request objects would report at least 32. + expect(counts.Response).toBeLessThanOrEqual(2); + expect(counts.AbortSignal).toBeLessThanOrEqual(2); + expect(proc.signalCode).toBeNull(); + expect(exitCode).toBe(0); }); test("fetch(data:) with percent-encoding does not leak", async () => { From 6b473b5503b66023e314f49ef1267e8cfaf3a241 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 4 Oct 2026 23:50:23 -0700 Subject: [PATCH 15/15] test: synchronize client readiness before TLS renegotiation Start renegotiation only after the first client handshake callback sends a framed ready message. Keep the framed payload acknowledgement, exact two-handshake assertion and pauseOnConnect regression check. This removes a setup race without changing TLS runtime code. --- test/js/node/tls/renegotiation.test.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/js/node/tls/renegotiation.test.ts b/test/js/node/tls/renegotiation.test.ts index 9514471cbd04..0b7ae7ffae45 100644 --- a/test/js/node/tls/renegotiation.test.ts +++ b/test/js/node/tls/renegotiation.test.ts @@ -147,14 +147,22 @@ it("pauseOnConnect acts on the first handshake only, not on a renegotiation", as "-e", ` const tls = require("tls"); + const { createInterface } = require("readline"); const server = tls.createServer( { cert: process.env.SERVER_CERT, key: process.env.SERVER_KEY, minVersion: "TLSv1.2", maxVersion: "TLSv1.2" }, socket => { socket.on("error", () => {}); - socket.on("data", () => socket.end("second")); - socket.renegotiate({ rejectUnauthorized: false }, err => { - if (err) socket.destroy(err); - else socket.write("first"); + createInterface({ input: socket }).on("line", line => { + if (line === "ready") { + socket.renegotiate({ rejectUnauthorized: false }, err => { + if (err) socket.destroy(err); + else socket.write("first"); + }); + } else if (line === "ack") { + socket.end("second"); + } else { + socket.destroy(new Error("unexpected client message")); + } }); }, ); @@ -179,11 +187,15 @@ it("pauseOnConnect acts on the first handshake only, not on a renegotiation", as pauseOnConnect: true, socket: { handshake(socket) { - if (++handshakes === 1) socket.resume(); + if (++handshakes === 1) { + socket.resume(); + // The server must not coalesce renegotiation with the initial handshake. + socket.write("ready\n"); + } }, data(socket, chunk) { received += chunk.toString(); - if (received === "first") socket.write("ack"); + if (received === "first") socket.write("ack\n"); else if (received === "firstsecond") outcome.resolve({ handshakes, received }); }, error(_socket, error) {